From b4d3246d94fa23482903b82f4a8a991d65ca3851 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Tue, 17 Nov 2015 17:35:38 -0500 Subject: [PATCH 01/37] add GPLV3 logo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 218f121..69d6f71 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

sanoid logo

====== -Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. +
Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From c039f2fff7729f66706a62604ccbf1060e081baa Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Tue, 17 Nov 2015 17:36:20 -0500 Subject: [PATCH 02/37] poking at layout --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69d6f71..d30330a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

sanoid logo

====== -
Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. +Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From 921ec1b3da6274f6ac3cbd38c19625936eb93bcb Mon Sep 17 00:00:00 2001 From: Jason Lawrence Date: Thu, 31 Dec 2015 11:03:59 -0600 Subject: [PATCH 03/37] Also check for 'zfs receive' alias --- sanoid | 2 +- syncoid | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sanoid b/sanoid index 1c43e47..0d6b15d 100755 --- a/sanoid +++ b/sanoid @@ -1006,7 +1006,7 @@ sub iszfsbusy { foreach my $process (@processes) { # if ($args{'debug'}) { print "DEBUG: checking process $process...\n"; } - if ($process =~ /zfs *(send|receive).*$fs/) { + if ($process =~ /zfs *(send|receive|recv).*$fs/) { # there's already a zfs send/receive process for our target filesystem - return true # if ($args{'debug'}) { print "DEBUG: process $process matches target $fs!\n"; } return 1; diff --git a/syncoid b/syncoid index a0a3d20..9129e28 100755 --- a/syncoid +++ b/syncoid @@ -466,7 +466,7 @@ sub iszfsbusy { foreach my $process (@processes) { # if ($debug) { print "DEBUG: checking process $process...\n"; } - if ($process =~ /zfs receive.*$fs/) { + if ($process =~ /zfs *(receive|recv).*$fs/) { # there's already a zfs receive process for our target filesystem - return true if ($debug) { print "DEBUG: process $process matches target $fs!\n"; } return 1; From 48d1de5b0c313a45a2666662b9443adc2764acc2 Mon Sep 17 00:00:00 2001 From: Jason Lawrence Date: Thu, 31 Dec 2015 11:54:05 -0600 Subject: [PATCH 04/37] Use UNIX-style 'ps' options everywhere for compatibility --- sanoid | 8 ++++---- syncoid | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index 1c43e47..dc37342 100755 --- a/sanoid +++ b/sanoid @@ -928,7 +928,7 @@ sub checklock { return 2; } - open PL, "$pscmd p $lockpid -o args= |"; + open PL, "$pscmd -p $lockpid -o args= |"; my @processlist = ; close PL; @@ -980,7 +980,7 @@ sub writelock { my $pid = $$; - open PL, "$pscmd p $$ -o args= |"; + open PL, "$pscmd -p $$ -o args= |"; my @processlist = ; close PL; @@ -998,9 +998,9 @@ sub iszfsbusy { # return true if busy (currently being sent or received), return false if not. my $fs = shift; - # if (args{'debug'}) { print "DEBUG: checking to see if $fs on is already in zfs receive using $pscmd axo args= ...\n"; } + # if (args{'debug'}) { print "DEBUG: checking to see if $fs on is already in zfs receive using $pscmd -Ao args= ...\n"; } - open PL, "$pscmd axo args= |"; + open PL, "$pscmd -Ao args= |"; my @processes = ; close PL; diff --git a/syncoid b/syncoid index a0a3d20..ae3ac7d 100755 --- a/syncoid +++ b/syncoid @@ -458,9 +458,9 @@ sub checkcommands { sub iszfsbusy { my ($rhost,$fs,$isroot) = @_; if ($rhost ne '') { $rhost = "$sshcmd $rhost"; } - if ($debug) { print "DEBUG: checking to see if $fs on $rhost is already in zfs receive using $rhost $pscmd axo args= ...\n"; } + if ($debug) { print "DEBUG: checking to see if $fs on $rhost is already in zfs receive using $rhost $pscmd -Ao args= ...\n"; } - open PL, "$rhost $pscmd axo args= |"; + open PL, "$rhost $pscmd -Ao args= |"; my @processes = ; close PL; From 3ca3f66500dde2e9cba9ca2b3506342d74ebb83b Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Thu, 31 Dec 2015 14:50:54 -0500 Subject: [PATCH 05/37] added --configdir=/path/to/configs CLI option --- sanoid | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sanoid b/sanoid index 8ea102e..891317d 100755 --- a/sanoid +++ b/sanoid @@ -12,15 +12,16 @@ use File::Path; # for rmtree command in use_prune use Data::Dumper; # debugging - print contents of hash use Time::Local; # to parse dates in reverse +# parse CLI arguments +my %args = getargs(@ARGV); + my $pscmd = '/bin/ps'; my $zfs = '/sbin/zfs'; -my $conf_file = '/etc/sanoid/sanoid.conf'; -my $default_conf_file = '/etc/sanoid/sanoid.defaults.conf'; - -# parse CLI arguments -my %args = getargs(@ARGV); +if ($args{'configdir'} eq '') { $args{'configdir'} = '/etc/sanoid'; } +my $conf_file = "$args{'configdir'}/sanoid.conf"; +my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf"; # parse config file my %config = init($conf_file,$default_conf_file); @@ -1030,7 +1031,7 @@ sub getargs { my %validargs; my %novalueargs; - push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly'; + push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly','configdir'; push my @novalueargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly'; foreach my $item (@validargs) { $validargs{$item}=1; } foreach my $item (@novalueargs) { $novalueargs{$item}=1; } From 633c5e237d375689a43bc6de4ab78714dcb8d4e1 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Thu, 31 Dec 2015 14:52:47 -0500 Subject: [PATCH 06/37] jjlawrens' portability pull requests + configdir option for sanoid --- CHANGELIST | 2 ++ VERSION | 2 +- sanoid | 2 +- syncoid | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index ad3378c..924bd9e 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,5 @@ +1.4.4 merged pull requests from jjlawren for OmniOS compatibility, added --configdir=/path/to/configs CLI option to sanoid at jjlawrens' request presumably for same + 1.4.3 added SSH persistence to syncoid - using socket speeds up SSH overhead 300%! =) one extra commit to get rid of the "Exit request sent." SSH noise at the end. diff --git a/VERSION b/VERSION index 428b770..1c99cf0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.3 +1.4.4 diff --git a/sanoid b/sanoid index 891317d..ed53983 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.3'; +my $version = '1.4.4'; use strict; use Config::IniFiles; # read samba-style conf file diff --git a/syncoid b/syncoid index 99981f1..8f67aaa 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.3'; +my $version = '1.4.4'; use strict; use Data::Dumper; From c87279f899fc6f48103e6c8ed0a69edb980b52af Mon Sep 17 00:00:00 2001 From: "Thomas M. Lapp" Date: Tue, 8 Mar 2016 10:29:50 -0500 Subject: [PATCH 07/37] Added -sshkey arg for automated cron runs --- syncoid | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 8f67aaa..aff729a 100755 --- a/syncoid +++ b/syncoid @@ -35,7 +35,12 @@ my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null'; my $lscmd = '/bin/ls'; # figure out if source and/or target are remote. -$sshcmd = "$sshcmd $sshcipher"; +if ( $args{'sshkey'} ) { + $sshcmd = "$sshcmd $sshcipher -i $args{'sshkey'}"; +} +else { + $sshcmd = "$sshcmd $sshcipher"; +} my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs); my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs); @@ -243,7 +248,7 @@ sub getargs { my %novaluearg; my %validarg; - push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r'); + push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey'); foreach my $item (@validargs) { $validarg{$item} = 1; } push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r'); foreach my $item (@novalueargs) { $novaluearg{$item} = 1; } From 50fb0d5733810311b64ea128dde1469cf11abd00 Mon Sep 17 00:00:00 2001 From: "Thomas M. Lapp" Date: Tue, 8 Mar 2016 10:46:36 -0500 Subject: [PATCH 08/37] Added spec file for rpm packaging. --- sanoid.spec | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 sanoid.spec diff --git a/sanoid.spec b/sanoid.spec new file mode 100644 index 0000000..9b6435a --- /dev/null +++ b/sanoid.spec @@ -0,0 +1,48 @@ +Name: sanoid +Version: 1.4.4 +Release: 1%{?dist} +BuildArch: noarch +Summary: A policy-driven snapshot management tool for ZFS filesystems + +Group: Applications/System +License: GPLv3 +URL: https://github.com/jimsalterjrs/sanoid +Source0: https://github.com/jimsalterjrs/sanoid/archive/sanoid-master.zip +Patch0: sanoid-syncoid-sshkey.patch +#BuildRequires: +Requires: perl + +%description +Sanoid is a policy-driven snapshot management +tool for ZFS filesystems. You can use Sanoid +to create, automatically thin, and monitor snapshots +and pool health from a single eminently +human-readable TOML config file. + +%prep +%setup -q -n sanoid-master +%patch0 -p1 + +%build + +%install +%{__install} -D -m 0644 sanoid.defaults.conf %{buildroot}/etc/sanoid/sanoid.defaults.conf +%{__install} -d %{buildroot}%{_sbindir} +%{__install} -m 0755 sanoid syncoid findoid sleepymutex %{buildroot}%{_sbindir} +%{__install} -D -m 0644 sanoid.conf %{buildroot}%{_docdir}/%{name}-%{version}/examples/sanoid.conf +echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name}-%{version}/examples/sanoid.cron + +%files +%doc CHANGELIST LICENSE VERSION README.md +%{_sbindir}/sanoid +%{_sbindir}/syncoid +%{_sbindir}/findoid +%{_sbindir}/sleepymutex +%dir %{_sysconfdir}/%{name} +%config %{_sysconfdir}/%{name}/sanoid.defaults.conf + + + +%changelog +* Sat Feb 13 2016 Tom Lapp - 1.4.4-1 +- Initial RPM Package From dd164bdd6816eb1435415e266f0a2a25ffda68b6 Mon Sep 17 00:00:00 2001 From: "Thomas M. Lapp" Date: Tue, 8 Mar 2016 10:48:40 -0500 Subject: [PATCH 09/37] Updated Change log --- sanoid.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sanoid.spec b/sanoid.spec index 9b6435a..7f7900c 100644 --- a/sanoid.spec +++ b/sanoid.spec @@ -44,5 +44,5 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %changelog -* Sat Feb 13 2016 Tom Lapp - 1.4.4-1 +* Sat Feb 13 2016 Thomas M. Lapp - 1.4.4-1 - Initial RPM Package From 839d65d8080f9b25d3c6e8b72c2f5037cefecaf1 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 23 May 2016 12:35:11 -0400 Subject: [PATCH 10/37] 1.4.5 changed shebang to '#!/usr/bin/env perl' for enhanced FreeBSD compatibility --- CHANGELIST | 2 ++ VERSION | 2 +- findoid | 4 ++-- sanoid | 4 ++-- sleepymutex | 5 +++-- syncoid | 4 ++-- 6 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 924bd9e..d672f28 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,5 @@ +1.4.5 altered shebang to '#!/usr/bin/env perl' for enhanced FreeBSD compatibility + 1.4.4 merged pull requests from jjlawren for OmniOS compatibility, added --configdir=/path/to/configs CLI option to sanoid at jjlawrens' request presumably for same 1.4.3 added SSH persistence to syncoid - using socket speeds up SSH overhead 300%! =) diff --git a/VERSION b/VERSION index 1c99cf0..e516bb9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.4 +1.4.5 diff --git a/findoid b/findoid index 98d4fb7..e49c887 100755 --- a/findoid +++ b/findoid @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this @@ -11,7 +11,7 @@ use warnings; my $zfs = '/sbin/zfs'; my %args = getargs(@ARGV); -my $progversion = '1.4.3'; +my $progversion = '1.4.5'; if ($args{'version'}) { print "$progversion\n"; exit 0; } diff --git a/sanoid b/sanoid index ed53983..8ab0e9e 100755 --- a/sanoid +++ b/sanoid @@ -1,10 +1,10 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.4'; +my $version = '1.4.5'; use strict; use Config::IniFiles; # read samba-style conf file diff --git a/sleepymutex b/sleepymutex index 1eb26c5..1361d8e 100755 --- a/sleepymutex +++ b/sleepymutex @@ -2,7 +2,8 @@ # this is just a cheap way to trigger mutex-based checks for process activity. # -# ie ./sleepymutex zfs receive data/lolz if you want a mutex hanging around as long as necessary that will show up -# to any routine that actively does something like "ps axo | grep 'zfs receive'" or whatever. +# ie ./sleepymutex zfs receive data/lolz if you want a mutex hanging around +# as long as necessary that will show up to any routine that actively does +# something like "ps axo | grep 'zfs receive'" or whatever. sleep 99999 diff --git a/syncoid b/syncoid index aff729a..593ab08 100755 --- a/syncoid +++ b/syncoid @@ -1,10 +1,10 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.4'; +my $version = '1.4.5'; use strict; use Data::Dumper; From 9a6bbb350fe466c0a49f5f3d2fea0bea268fb05e Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 23 May 2016 13:17:22 -0400 Subject: [PATCH 11/37] added INSTALL notes --- INSTALL | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 INSTALL diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000..149edc3 --- /dev/null +++ b/INSTALL @@ -0,0 +1,30 @@ +SYNCOID +------- +Syncoid depends on ssh, pv, gzip, lzop, and mbuffer. It can run with reduced +functionality in the absence of any or all of the above. SSH is only required +for remote synchronization. Arcfour crypto is the default for SSH transport, +and currently (v1.4.5) syncoid runs will fail if arcfour is not available +on either end of the transport. + +On Ubuntu: apt install pv lzop mbuffer +On FreeBSD: pkg install pv lzop + +FreeBSD notes: mbuffer is not currently recommended due to oddities in + FreeBSD's local implementation. Internal network buffering + capability is on the roadmap soon to remove mbuffer dependency + anyway. FreeBSD places pv and lzop in /usr/local/bin instead + of /usr/bin ; syncoid currently does not check path. + + Simplest path workaround is symlinks, eg: + root@bsd:~# ln -s /usr/bin/lzop /usr/local/bin/lzop + + +SANOID +------ +Sanoid depends on the Perl module Config::IniFiles and will not operate +without it. Config::IniFiles may be installed from CPAN, though the project +strongly recommends using your distribution's repositories instead. + +On Ubuntu: apt install libconfig-inifiles-perl +On FreeBSD: pkg install p5-Config-Inifiles + From 546bfabf6d00b2eb17fb67ab2479716826b6d3de Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 23 May 2016 18:55:42 -0400 Subject: [PATCH 12/37] added ==0 or die to all system calls in sanoid and syncoid that didn't already have them --- CHANGELIST | 16 ++++++++++++---- VERSION | 2 +- findoid | 2 +- sanoid | 5 +++-- syncoid | 50 ++++++++++++++++++++++++++++++++++++++------------ 5 files changed, 55 insertions(+), 20 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index d672f28..cc740ba 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,9 @@ +1.4.6 added a mollyguard to syncoid to help newbies who try to zfs create a new target dataset + before doing an initial replication, instead of letting the replication itself create + the target. + + added "==0 or die" to all system() calls in syncoid. + 1.4.5 altered shebang to '#!/usr/bin/env perl' for enhanced FreeBSD compatibility 1.4.4 merged pull requests from jjlawren for OmniOS compatibility, added --configdir=/path/to/configs CLI option to sanoid at jjlawrens' request presumably for same @@ -5,16 +11,18 @@ 1.4.3 added SSH persistence to syncoid - using socket speeds up SSH overhead 300%! =) one extra commit to get rid of the "Exit request sent." SSH noise at the end. -1.4.2 removed -r flag for zfs destroy of pruned snapshots in sanoid, which unintentionally caused same-name child snapshots to be deleted - thank you Lenz Weber! +1.4.2 removed -r flag for zfs destroy of pruned snapshots in sanoid, which unintentionally caused same-name + child snapshots to be deleted - thank you Lenz Weber! 1.4.1 updated check_zpool() in sanoid to parse zpool list properly both pre- and post- ZoL v0.6.4 -1.4.0 added findoid tool - find and list all versions of a given file in all available ZFS snapshots. use: findoid /path/to/file +1.4.0 added findoid tool - find and list all versions of a given file in all available ZFS snapshots. + use: findoid /path/to/file 1.3.1 whoops - prevent process_children_only from getting set from blank value in defaults -1.3.0 changed monitor_children_only to process_children_only. which keeps sanoid from messing around with empty parent datasets at all. - also more thoroughly documented features in default config files. +1.3.0 changed monitor_children_only to process_children_only. which keeps sanoid from messing around with + empty parent datasets at all. also more thoroughly documented features in default config files. 1.2.0 added monitor_children_only parameter to sanoid.conf for use with recursive definitions - in cases where container dataset is kept empty diff --git a/VERSION b/VERSION index e516bb9..c514bd8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.5 +1.4.6 diff --git a/findoid b/findoid index e49c887..b19dfba 100755 --- a/findoid +++ b/findoid @@ -11,7 +11,7 @@ use warnings; my $zfs = '/sbin/zfs'; my %args = getargs(@ARGV); -my $progversion = '1.4.5'; +my $progversion = '1.4.6'; if ($args{'version'}) { print "$progversion\n"; exit 0; } diff --git a/sanoid b/sanoid index 8ab0e9e..d44b576 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.5'; +my $version = '1.4.6'; use strict; use Config::IniFiles; # read samba-style conf file @@ -332,7 +332,8 @@ sub take_snapshots { foreach my $snap ( @newsnaps ) { if ($args{'verbose'}) { print "taking snapshot $snap\n"; } if (!$args{'readonly'}) { - system($zfs, "snapshot", "$snap"); + system($zfs, "snapshot", "$snap") == 0 + or die "CRITICAL ERROR: $zfs snapshot $snap failed, $?"; # make sure we don't end up with multiple snapshots with the same ctime sleep 1; } diff --git a/syncoid b/syncoid index 593ab08..a397c0b 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.5'; +my $version = '1.4.6'; use strict; use Data::Dumper; @@ -56,7 +56,9 @@ my %avail = checkcommands(); my %snaps; -## break here to call replication individually so that we can loop across children separately, for recursive replication ## +## break here to call replication individually so that we ## +## can loop across children separately, for recursive ## +## replication ## if (! $args{'recursive'}) { syncdataset($sourcehost, $sourcefs, $targethost, $targetfs); @@ -167,7 +169,8 @@ sub syncdataset { if (iszfsbusy($targethost,$targetfs,$targetisroot)) { die "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; } - system($synccmd); + system($synccmd) == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; # now do an -I to the new sync snapshot, assuming there were any snapshots # other than the new sync snapshot to begin with, of course @@ -190,7 +193,8 @@ sub syncdataset { print "INFO: Updating new target filesystem with incremental $sourcefs\@$oldestsnap ... $newsyncsnap (~ $disp_pvsize):\n"; if ($debug) { print "DEBUG: $synccmd\n"; } - system($synccmd); + system($synccmd) == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; # restore original readonly value to target after sync complete setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); @@ -202,8 +206,9 @@ sub syncdataset { # get current readonly status of target, then set it to on during sync $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); + my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used'); - my $matchingsnap = getmatchingsnapshot(\%snaps); + my $matchingsnap = getmatchingsnapshot($targetsize, \%snaps); # make sure target is (still) not currently in receive. if (iszfsbusy($targethost,$targetfs,$targetisroot)) { @@ -229,7 +234,8 @@ sub syncdataset { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; if ($debug) { print "DEBUG: $synccmd\n"; } - system("$synccmd"); + system("$synccmd") == 0 + or die "CRITICAL ERROR: $synccmd failed: $?"; # restore original readonly value to target after sync complete setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); @@ -489,7 +495,8 @@ sub setzfsvalue { my $mysudocmd; if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fs\n"; } - system("$rhost $mysudocmd $zfscmd set $property=$value $fs"); + system("$rhost $mysudocmd $zfscmd set $property=$value $fs") == 0 + or die "CRITICAL ERROR: $rhost $mysudocmd $zfscmd set $property=$value $fs died: $?"; return; } @@ -628,7 +635,8 @@ sub pruneoldsyncsnaps { if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } - system("$rhost $prunecmd"); + system("$rhost $prunecmd") == 0 + or die "CRITICAL ERROR: $rhost $prunecmd failed: $?"; $prunecmd = ''; $counter = 0; } @@ -640,19 +648,36 @@ sub pruneoldsyncsnaps { if ($rhost ne '') { $prunecmd = '"' . $prunecmd . '"'; } if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } - system("$rhost $prunecmd"); + system("$rhost $prunecmd") == 0 + or die "CRITICAL ERROR: $rhost $prunecmd failed: $?"; } return; } sub getmatchingsnapshot { - my $snaps = shift; + my ($targetsize, $snaps) = shift; foreach my $snap ( sort { $snaps{'source'}{$b}{'ctime'}<=>$snaps{'source'}{$a}{'ctime'} } keys %{ $snaps{'source'} }) { if ($snaps{'source'}{$snap}{'ctime'} == $snaps{'target'}{$snap}{'ctime'}) { return $snap; } } - print "UNEXPECTED ERROR: target exists but has no matching snapshots!\n"; + + # if we got this far, we failed to find a matching snapshot. + + print "\n"; + print "CRITICAL ERROR: Target exists but has no matching snapshots!\n"; + print " Replication to target would require destroying existing\n"; + print " target. Cowardly refusing to destroy your existing target.\n\n"; + + # experience tells me we need a mollyguard for people who try to + # zfs create targetpool/targetsnap ; syncoid sourcepool/sourcesnap targetpool/targetsnap ... + + if ( $targetsize < (64*1024*1024) ) { + print " NOTE: Target dataset is < 64MB used - did you mistakenly run\n"; + print " \`zfs create $args{'target'}\` on the target? ZFS initial\n"; + print " replication must be to a NON EXISTENT DATASET, which will\n"; + print " then be CREATED BY the initial replication process.\n\n"; + } exit 256; } @@ -665,7 +690,8 @@ sub newsyncsnap { my %date = getdate(); my $snapname = "syncoid\_$hostid\_$date{'stamp'}"; my $snapcmd = "$rhost $mysudocmd $zfscmd snapshot $fs\@$snapname\n"; - system($snapcmd); + system($snapcmd) == 0 + or die "CRITICAL ERROR: $snapcmd failed: $?"; return $snapname; } From ba3e956b08375dcdb16ab2840bee1128f403849d Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 23 May 2016 18:56:28 -0400 Subject: [PATCH 13/37] added ==0 or die to all system() calls that didn't already have them in sanoid and syncoid --- CHANGELIST | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELIST b/CHANGELIST index cc740ba..17ff61b 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -2,7 +2,7 @@ before doing an initial replication, instead of letting the replication itself create the target. - added "==0 or die" to all system() calls in syncoid. + added "==0 or die" to all system() calls in sanoid and syncoid that didn't already have them. 1.4.5 altered shebang to '#!/usr/bin/env perl' for enhanced FreeBSD compatibility From 17759ce659469d67a77854d5e531e7d0a0c0e360 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Wed, 25 May 2016 17:00:19 -0400 Subject: [PATCH 14/37] 1.4.6a - mitigate ZFS on Linux bug #4473 by complaining not dying if set readonly fails. --- CHANGELIST | 4 ++++ findoid | 2 +- sanoid | 2 +- syncoid | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 17ff61b..c32528f 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,7 @@ +1.4.6a due to bug in ZFS on Linux which frequently causes errors to return from `zfs set readonly`, + changed ==0 or die in setzfsvalue() to ==0 or [complain] - it's not worth causing replication + to fail while this ZFS on Linux bug exists. + 1.4.6 added a mollyguard to syncoid to help newbies who try to zfs create a new target dataset before doing an initial replication, instead of letting the replication itself create the target. diff --git a/findoid b/findoid index b19dfba..b00b81a 100755 --- a/findoid +++ b/findoid @@ -11,7 +11,7 @@ use warnings; my $zfs = '/sbin/zfs'; my %args = getargs(@ARGV); -my $progversion = '1.4.6'; +my $progversion = '1.4.6a'; if ($args{'version'}) { print "$progversion\n"; exit 0; } diff --git a/sanoid b/sanoid index d44b576..8c3e013 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6'; +my $version = '1.4.6a'; use strict; use Config::IniFiles; # read samba-style conf file diff --git a/syncoid b/syncoid index a397c0b..c3892b4 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6'; +my $version = '1.4.6a'; use strict; use Data::Dumper; @@ -496,7 +496,7 @@ sub setzfsvalue { if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } if ($debug) { print "$rhost $mysudocmd $zfscmd set $property=$value $fs\n"; } system("$rhost $mysudocmd $zfscmd set $property=$value $fs") == 0 - or die "CRITICAL ERROR: $rhost $mysudocmd $zfscmd set $property=$value $fs died: $?"; + or print "WARNING: $rhost $mysudocmd $zfscmd set $property=$value $fs died: $?, proceeding anyway.\n"; return; } From 5b3693357e79f2d2bdbcb8baadf1a686d093349a Mon Sep 17 00:00:00 2001 From: Albin Gustavsson Date: Thu, 26 May 2016 20:30:24 +0200 Subject: [PATCH 15/37] Added options to set ssh port. ssh port can be set by passing the -sshport variable on the command line or by changing the $sshport variable. --- syncoid | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/syncoid b/syncoid index c3892b4..cd7ac06 100755 --- a/syncoid +++ b/syncoid @@ -26,6 +26,7 @@ my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; my $pscmd = '/bin/ps'; my $sshcipher = '-c arcfour'; +my $sshport = '-p 22'; my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; my $sudocmd = '/usr/bin/sudo'; @@ -34,12 +35,15 @@ my $mbufferoptions = '-q -s 128k -m 16M 2>/dev/null'; # being present on remote machines. my $lscmd = '/bin/ls'; +if ( $args{'sshport'} ) { + $sshport = "-p $args{'sshport'}"; +} # figure out if source and/or target are remote. if ( $args{'sshkey'} ) { - $sshcmd = "$sshcmd $sshcipher -i $args{'sshkey'}"; + $sshcmd = "$sshcmd $sshcipher $sshport -i $args{'sshkey'}"; } else { - $sshcmd = "$sshcmd $sshcipher"; + $sshcmd = "$sshcmd $sshcipher $sshport"; } my ($sourcehost,$sourcefs,$sourceisroot) = getssh($rawsourcefs); my ($targethost,$targetfs,$targetisroot) = getssh($rawtargetfs); @@ -254,7 +258,7 @@ sub getargs { my %novaluearg; my %validarg; - push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey'); + push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport'); foreach my $item (@validargs) { $validarg{$item} = 1; } push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r'); foreach my $item (@novalueargs) { $novaluearg{$item} = 1; } @@ -716,6 +720,7 @@ sub getssh { my $rhost; my $isroot; my $socket; + # if we got passed something with an @ in it, we assume it's an ssh connection, eg root@myotherbox if ($fs =~ /\@/) { $rhost = $fs; @@ -726,7 +731,7 @@ sub getssh { if ($remoteuser eq 'root') { $isroot = 1; } else { $isroot = 0; } # now we need to establish a persistent master SSH connection $socket = "/tmp/syncoid-$remoteuser-$rhost-" . time(); - open FH, "$sshcmd -M -S $socket -o ControlPersist=yes $rhost exit |"; + open FH, "$sshcmd -M -S $socket -o ControlPersist=yes $sshport $rhost exit |"; close FH; $rhost = "-S $socket $rhost"; } else { From 6eaa3033272104b1ac91459cd91c940fcfb2459b Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Fri, 27 May 2016 18:24:27 -0400 Subject: [PATCH 16/37] 1.4.6b - updated cipherlist for syncoid to chacha20-poly1305@openssh.com,arcfour --- CHANGELIST | 5 +++++ findoid | 2 +- sanoid | 2 +- syncoid | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index c32528f..41d6a28 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,8 @@ +1.4.6b updated default cipherlist for syncoid to + chacha20-poly1305@openssh.com,arcfour - arcfour isn't supported on + newer SSH (in Ubuntu Xenial and FreeBSD), chacha20 isn't supported on + some older SSH versions (Ubuntu Precise< I think?) + 1.4.6a due to bug in ZFS on Linux which frequently causes errors to return from `zfs set readonly`, changed ==0 or die in setzfsvalue() to ==0 or [complain] - it's not worth causing replication to fail while this ZFS on Linux bug exists. diff --git a/findoid b/findoid index b00b81a..d975b29 100755 --- a/findoid +++ b/findoid @@ -11,7 +11,7 @@ use warnings; my $zfs = '/sbin/zfs'; my %args = getargs(@ARGV); -my $progversion = '1.4.6a'; +my $progversion = '1.4.6b'; if ($args{'version'}) { print "$progversion\n"; exit 0; } diff --git a/sanoid b/sanoid index 8c3e013..a803654 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6a'; +my $version = '1.4.6b'; use strict; use Config::IniFiles; # read samba-style conf file diff --git a/syncoid b/syncoid index c3892b4..f13b1a7 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6a'; +my $version = '1.4.6b'; use strict; use Data::Dumper; @@ -25,7 +25,7 @@ my $debug = $args{'debug'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; my $pscmd = '/bin/ps'; -my $sshcipher = '-c arcfour'; +my $sshcipher = '-c chacha20-poly1305@openssh.com,arcfour'; my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; my $sudocmd = '/usr/bin/sudo'; From dd4ef0fa533e5d7db42423f1220050ef564bdc3b Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Fri, 27 May 2016 18:57:55 -0400 Subject: [PATCH 17/37] 1.4.6c merge @gusson's --sshport argument for syncoid --- CHANGELIST | 7 +++++++ findoid | 2 +- sanoid | 2 +- syncoid | 4 ++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index c32528f..873c90a 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,10 @@ +1.4.6c merged @gusson's pull request to add -sshport argument + +1.4.6b updated default cipherlist for syncoid to + chacha20-poly1305@openssh.com,arcfour - arcfour isn't supported on + newer SSH (in Ubuntu Xenial and FreeBSD), chacha20 isn't supported on + some older SSH versions (Ubuntu Precise< I think?) + 1.4.6a due to bug in ZFS on Linux which frequently causes errors to return from `zfs set readonly`, changed ==0 or die in setzfsvalue() to ==0 or [complain] - it's not worth causing replication to fail while this ZFS on Linux bug exists. diff --git a/findoid b/findoid index b00b81a..a7121d5 100755 --- a/findoid +++ b/findoid @@ -11,7 +11,7 @@ use warnings; my $zfs = '/sbin/zfs'; my %args = getargs(@ARGV); -my $progversion = '1.4.6a'; +my $progversion = '1.4.6c'; if ($args{'version'}) { print "$progversion\n"; exit 0; } diff --git a/sanoid b/sanoid index 8c3e013..fe50d61 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6a'; +my $version = '1.4.6c'; use strict; use Config::IniFiles; # read samba-style conf file diff --git a/syncoid b/syncoid index cd7ac06..cf3616c 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6a'; +my $version = '1.4.6c'; use strict; use Data::Dumper; @@ -25,7 +25,7 @@ my $debug = $args{'debug'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; my $pscmd = '/bin/ps'; -my $sshcipher = '-c arcfour'; +my $sshcipher = '-c chacha20-poly1305@openssh.com,arcfour'; my $sshport = '-p 22'; my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; From c0f065b0f0191fe207266f5e6b78b7e5a182ef7e Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Fri, 27 May 2016 19:01:51 -0400 Subject: [PATCH 18/37] 1.4.6c - merged @gusson's --sshport argument enhancement for syncoid --- CHANGELIST | 3 --- findoid | 4 ---- sanoid | 4 ---- syncoid | 7 ------- 4 files changed, 18 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 75081d7..873c90a 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,8 +1,5 @@ -<<<<<<< HEAD -======= 1.4.6c merged @gusson's pull request to add -sshport argument ->>>>>>> gusson-ssh_port 1.4.6b updated default cipherlist for syncoid to chacha20-poly1305@openssh.com,arcfour - arcfour isn't supported on newer SSH (in Ubuntu Xenial and FreeBSD), chacha20 isn't supported on diff --git a/findoid b/findoid index 998099e..a7121d5 100755 --- a/findoid +++ b/findoid @@ -11,11 +11,7 @@ use warnings; my $zfs = '/sbin/zfs'; my %args = getargs(@ARGV); -<<<<<<< HEAD -my $progversion = '1.4.6b'; -======= my $progversion = '1.4.6c'; ->>>>>>> gusson-ssh_port if ($args{'version'}) { print "$progversion\n"; exit 0; } diff --git a/sanoid b/sanoid index 2ecff59..fe50d61 100755 --- a/sanoid +++ b/sanoid @@ -4,11 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -<<<<<<< HEAD -my $version = '1.4.6b'; -======= my $version = '1.4.6c'; ->>>>>>> gusson-ssh_port use strict; use Config::IniFiles; # read samba-style conf file diff --git a/syncoid b/syncoid index 61c1a17..cf3616c 100755 --- a/syncoid +++ b/syncoid @@ -4,11 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -<<<<<<< HEAD -my $version = '1.4.6b'; -======= my $version = '1.4.6c'; ->>>>>>> gusson-ssh_port use strict; use Data::Dumper; @@ -30,10 +26,7 @@ my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; my $pscmd = '/bin/ps'; my $sshcipher = '-c chacha20-poly1305@openssh.com,arcfour'; -<<<<<<< HEAD -======= my $sshport = '-p 22'; ->>>>>>> gusson-ssh_port my $pvcmd = '/usr/bin/pv'; my $mbuffercmd = '/usr/bin/mbuffer'; my $sudocmd = '/usr/bin/sudo'; From 95f1ecb159639e3ea7ee686ba70caf22928b6183 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 13 Jun 2016 11:42:17 -0400 Subject: [PATCH 19/37] 1.4.7 reverted shebangs to Linux style, added -F to zfs receive, dyked out set readonly behavior to mitigate upstream bugs in mount/unmount --- CHANGELIST | 10 ++++++++++ VERSION | 2 +- findoid | 4 ++-- sanoid | 4 ++-- syncoid | 31 +++++++++++++++++++++---------- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 873c90a..4e80d00 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,13 @@ +1.4.7 reverted Perl shebangs to #!/usr/bin/perl - sorry FreeBSD folks, shebanged to /usr/bin/env perl bare calls to syncoid + or sanoid (without explicit calls to Perl) don't work on EITHER of our systems. I'm not OK with that, this is going to be + an OS localization issue that can either be addressed with BSD-specific packaging, or you can individually address it + by editing the shebangs on your own systems OR by doing a one-time ln -s /usr/local/bin/perl to /usr/bin/perl, which will + fix the issue for this particular script AND all other Perl scripts developed on non-BSD systems. + + also temporarily dyked out the set readonly functionality in syncoid - it was causing more problems than it prevented, and + using the -F argument with receive prevents incautious writes (including just cd'ing into mounted datasets, if atimes are on) + from interrupting syncoid runs anyway. + 1.4.6c merged @gusson's pull request to add -sshport argument 1.4.6b updated default cipherlist for syncoid to diff --git a/VERSION b/VERSION index c514bd8..be05bba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.4.6 +1.4.7 diff --git a/findoid b/findoid index a7121d5..d98268d 100755 --- a/findoid +++ b/findoid @@ -1,4 +1,4 @@ -#!/usr/bin/env perl +#!/usr/bin/perl # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this @@ -11,7 +11,7 @@ use warnings; my $zfs = '/sbin/zfs'; my %args = getargs(@ARGV); -my $progversion = '1.4.6c'; +my $progversion = '1.4.7'; if ($args{'version'}) { print "$progversion\n"; exit 0; } diff --git a/sanoid b/sanoid index fe50d61..12806e0 100755 --- a/sanoid +++ b/sanoid @@ -1,10 +1,10 @@ -#!/usr/bin/env perl +#!/usr/bin/perl # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6c'; +my $version = '1.4.7'; use strict; use Config::IniFiles; # read samba-style conf file diff --git a/syncoid b/syncoid index cf3616c..5883123 100755 --- a/syncoid +++ b/syncoid @@ -1,10 +1,10 @@ -#!/usr/bin/env perl +#!/usr/bin/perl # this software is licensed for use under the Free Software Foundation's GPL v3.0 license, as retrieved # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -my $version = '1.4.6c'; +my $version = '1.4.7'; use strict; use Data::Dumper; @@ -146,7 +146,9 @@ sub syncdataset { # been turned on... even when it's off... unless and # until the filesystem is zfs umounted and zfs remounted. # we're going to do the right thing anyway. - my $originaltargetreadonly; + # dyking this functionality out for the time being due to buggy mount/unmount behavior + # with ZFS on Linux (possibly OpenZFS in general) when setting/unsetting readonly. + #my $originaltargetreadonly; # sync 'em up. if (! $targetexists) { @@ -181,8 +183,10 @@ sub syncdataset { if ($oldestsnap ne $newsyncsnap) { # get current readonly status of target, then set it to on during sync - $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); - setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); + # dyking this functionality out for the time being due to buggy mount/unmount behavior + # with ZFS on Linux (possibly OpenZFS in general) when setting/unsetting readonly. + # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); + # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); $sendcmd = "$sourcesudocmd $zfscmd send -I $sourcefs\@$oldestsnap $sourcefs\@$newsyncsnap"; $pvsize = getsendsize($sourcehost,"$sourcefs\@$oldestsnap","$sourcefs\@$newsyncsnap",$sourceisroot); @@ -201,15 +205,20 @@ sub syncdataset { or die "CRITICAL ERROR: $synccmd failed: $?"; # restore original readonly value to target after sync complete - setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); + # dyking this functionality out for the time being due to buggy mount/unmount behavior + # with ZFS on Linux (possibly OpenZFS in general) when setting/unsetting readonly. + # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } } else { # find most recent matching snapshot and do an -I # to the new snapshot # get current readonly status of target, then set it to on during sync - $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); - setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); + # dyking this functionality out for the time being due to buggy mount/unmount behavior + # with ZFS on Linux (possibly OpenZFS in general) when setting/unsetting readonly. + # $originaltargetreadonly = getzfsvalue($targethost,$targetfs,$targetisroot,'readonly'); + # setzfsvalue($targethost,$targetfs,$targetisroot,'readonly','on'); + my $targetsize = getzfsvalue($targethost,$targetfs,$targetisroot,'-p used'); my $matchingsnap = getmatchingsnapshot($targetsize, \%snaps); @@ -230,7 +239,7 @@ sub syncdataset { } my $sendcmd = "$sourcesudocmd $zfscmd send -I $sourcefs\@$matchingsnap $sourcefs\@$newsyncsnap"; - my $recvcmd = "$targetsudocmd $zfscmd receive $targetfs"; + my $recvcmd = "$targetsudocmd $zfscmd receive -F $targetfs"; my $pvsize = getsendsize($sourcehost,"$sourcefs\@$matchingsnap","$sourcefs\@$newsyncsnap",$sourceisroot); my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } @@ -242,7 +251,9 @@ sub syncdataset { or die "CRITICAL ERROR: $synccmd failed: $?"; # restore original readonly value to target after sync complete - setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); + # dyking this functionality out for the time being due to buggy mount/unmount behavior + # with ZFS on Linux (possibly OpenZFS in general) when setting/unsetting readonly. + #setzfsvalue($targethost,$targetfs,$targetisroot,'readonly',$originaltargetreadonly); } # prune obsolete sync snaps on source and target. From 4aa923f80cfa64b63112f895c879d7bca36484fa Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 13 Jun 2016 11:48:57 -0400 Subject: [PATCH 20/37] 1.4.7 - whoops, forgot to git add FREEBSD.readme --- FREEBSD.readme | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 FREEBSD.readme diff --git a/FREEBSD.readme b/FREEBSD.readme new file mode 100644 index 0000000..d2d7889 --- /dev/null +++ b/FREEBSD.readme @@ -0,0 +1,13 @@ +FreeBSD users will need to change the Perl shebangs at the top of the executables from #!/usr/bin/perl +to #!/usr/local/bin/perl in most cases. + +Sorry folks, but if I set this with #!/usr/bin/env perl as suggested, then nothing works properly +from a typical cron environment on EITHER operating system, Linux or BSD. I'm mostly using Linux +systems, so I get to set the shebang for my use and give you folks a FREEBSD readme rather than +the other way around. =) + +If you don't want to have to change the shebangs, your other option is to drop a symlink on your system: + +root@bsd:~# ln -s /usr/local/bin/perl /usr/bin/perl + +After putting this symlink in place, ANY perl script shebanged for Linux will work on your system too. From 5dc5b8592e7fd8c24bf878e7a38dafe7ce656a16 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:09:46 -0400 Subject: [PATCH 21/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d30330a..2779388 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

sanoid logo

====== -Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. +Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. (Want an example? Watch a short real time demo of rolling back a full-scale cryptomalware infection in seconds!) More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From 7af7c65825571c6d19c997841c6f0e6e361a7665 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:28:31 -0400 Subject: [PATCH 22/37] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2779388..d77a627 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,11 @@

sanoid logo

====== -Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. (Want an example? Watch a short real time demo of rolling back a full-scale cryptomalware infection in seconds!) +Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. + +Want an example? Watch a short real time demo of rolling back a full-scale cryptomalware infection in seconds: + +[![Sanoid rollback demo](http://openoid.net/sanoid-video-launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From 5789fbd7d7fd8031d0f4d8eba3a6d586d44cc8b7 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:29:09 -0400 Subject: [PATCH 23/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d77a627..6394cd4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Want an example? Watch a short real time demo of rolling back a full-scale cryptomalware infection in seconds: -[![Sanoid rollback demo](http://openoid.net/sanoid-video-launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") +[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From 65adfdd388f5abacf2854aae3b2e3b8459a7f327 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:30:22 -0400 Subject: [PATCH 24/37] Update README.md --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6394cd4..22736ef 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,8 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -Want an example? Watch a short real time demo of rolling back a full-scale cryptomalware infection in seconds: - -[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") +

[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") +Real time demo: rolling back a full-scale cryptomalware infection in seconds

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From 2a4db1d90e3811222df79c1ada7bdc8a64558153 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:30:44 -0400 Subject: [PATCH 25/37] Update README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 22736ef..82c8086 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,10 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") -Real time demo: rolling back a full-scale cryptomalware infection in seconds

+

+[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") +Real time demo: rolling back a full-scale cryptomalware infection in seconds +

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From ddaa41dc09a37667e0338020fd7b3172163bc0c8 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:32:28 -0400 Subject: [PATCH 26/37] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 82c8086..0095453 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,7 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

-[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") +

[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") Real time demo: rolling back a full-scale cryptomalware infection in seconds

From 454389275f63955ee4f94164565ef46f3f68af46 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:32:50 -0400 Subject: [PATCH 27/37] Update README.md --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0095453..eb9f563 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,8 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") -Real time demo: rolling back a full-scale cryptomalware infection in seconds -

+[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") +(Real time demo: rolling back a full-scale cryptomalware infection in seconds) More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From a0e400aaffab8844d097e41b4ba6b94ead946d04 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:34:06 -0400 Subject: [PATCH 28/37] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index eb9f563..d0b3378 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -[![Sanoid rollback demo](http://www.openoid.net/sanoid_video_launcher.png)](https://youtu.be/ZgowLNBsu00 "Sanoid rollback demo") -(Real time demo: rolling back a full-scale cryptomalware infection in seconds) +

+(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From eddfcc4e819796babaa2b03fef6f5f46a4370616 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:34:56 -0400 Subject: [PATCH 29/37] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d0b3378..8f87b1d 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,8 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

-(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

+

sanoid rollback demo

+

(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From bf52340ee0afc6cf3e1f9d74b2414e3943820c61 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:35:36 -0400 Subject: [PATCH 30/37] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f87b1d..cb3bb74 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,7 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

sanoid rollback demo

-

(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

+

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From b5f7d9096cc460c5bd9e533fe7bed83df6c664ee Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:36:01 -0400 Subject: [PATCH 31/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb3bb74..4a2c5ed 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

+

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From e99306094beb7ad1c9218442cc062a39e03ff12e Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Mon, 4 Jul 2016 16:36:51 -0400 Subject: [PATCH 32/37] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a2c5ed..a91146c 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Sanoid is a policy-driven snapshot management tool for ZFS filesystems. When combined with the Linux KVM hypervisor, you can use it to make your systems functionally immortal. -

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

+

sanoid rollback demo
(Real time demo: rolling back a full-scale cryptomalware infection in seconds!)

More prosaically, you can use Sanoid to create, automatically thin, and monitor snapshots and pool health from a single eminently human-readable TOML config file at /etc/sanoid/sanoid.conf. (Sanoid also requires a "defaults" file located at /etc/sanoid/sanoid.defaults.conf, which is not user-editable.) A typical Sanoid system would have a single cron job: From adb14d23434512ff0c39c7cf8f84552a0368302a Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sun, 7 Aug 2016 11:45:46 -0400 Subject: [PATCH 33/37] changed die to warn if syncoid fails to zfs destroy a sync snapshot - no good reason to stop a sync for that --- syncoid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/syncoid b/syncoid index 5883123..c70862d 100755 --- a/syncoid +++ b/syncoid @@ -664,7 +664,7 @@ sub pruneoldsyncsnaps { if ($debug) { print "DEBUG: pruning up to $maxsnapspercmd obsolete sync snapshots...\n"; } if ($debug) { print "DEBUG: $rhost $prunecmd\n"; } system("$rhost $prunecmd") == 0 - or die "CRITICAL ERROR: $rhost $prunecmd failed: $?"; + or warn "WARNING: $rhost $prunecmd failed: $?"; } return; } From 287663765582f36551ba352af9c029b1933ff032 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sun, 7 Aug 2016 12:13:32 -0400 Subject: [PATCH 34/37] fixed use of uninitialized value warnings in syncoid --- syncoid | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/syncoid b/syncoid index c70862d..19109c1 100755 --- a/syncoid +++ b/syncoid @@ -7,6 +7,7 @@ my $version = '1.4.7'; use strict; +use warnings; use Data::Dumper; use Time::Local; use Sys::Hostname; @@ -333,12 +334,14 @@ sub getargs { if ($args{'r'}) { $args{'recursive'} = $args{'r'}; } + if (!defined $args{'compress'}) { $args{'compress'} = 'default'; } + if ($args{'compress'} eq 'gzip') { $args{'rawcompresscmd'} = '/bin/gzip'; $args{'compressargs'} = '-3'; $args{'rawdecompresscmd'} = '/bin/zcat'; $args{'decompressargs'} = ''; - } elsif ( ($args{'compress'} eq 'lzo') || ! (defined $args{'compress'}) ) { + } elsif ( ($args{'compress'} eq 'lzo') || ($args{'compress'} eq 'default') ) { $args{'rawcompresscmd'} = '/usr/bin/lzop'; $args{'compressargs'} = ''; $args{'rawdecompresscmd'} = '/usr/bin/lzop'; @@ -374,8 +377,11 @@ sub checkcommands { return %avail; } - if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; } - if ($targethost ne '') { $targetssh = "$sshcmd $targethost"; } + if (!defined $sourcehost) { $sourcehost = ''; } + if (!defined $targethost) { $targethost = ''; } + + if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; } else { $sourcessh = ''; } + if ($targethost ne '') { $targetssh = "$sshcmd $targethost"; } else { $targetssh = ''; } # if raw compress command is null, we must have specified no compression. otherwise, # make sure that compression is available everywhere we need it @@ -415,6 +421,12 @@ sub checkcommands { $t = "ssh:$t"; } + if (!defined $avail{'sourcecompress'}) { $avail{'sourcecompress'} = ''; } + if (!defined $avail{'targetcompress'}) { $avail{'targetcompress'} = ''; } + if (!defined $avail{'sourcembuffer'}) { $avail{'sourcembuffer'} = ''; } + if (!defined $avail{'targetmbuffer'}) { $avail{'targetmbuffer'} = ''; } + + if ($avail{'sourcecompress'} eq '') { if ($args{'rawcompresscmd'} ne '') { print "WARN: $args{'compresscmd'} not available on source $s- sync will continue without compression.\n"; @@ -566,12 +578,13 @@ sub buildsynccmd { # $synccmd = "$sendcmd | $mbuffercmd | $pvcmd | $recvcmd"; $synccmd = "$sendcmd |"; # avoid confusion - accept either source-bwlimit or target-bwlimit as the bandwidth limiting option here - my $bwlimit; - if ($args{'source-bwlimit'} eq '') { + my $bwlimit = ''; + if (defined $args{'source-bwlimit'}) { + $bwlimit = $args{'source-bwlimit'}; + } elsif (defined $args{'target-bwlimit'}) { $bwlimit = $args{'target-bwlimit'}; - } else { - $bwlimit = $args{'source-bwlimit'}; } + if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $bwlimit $mbufferoptions |"; } if ($avail{'localpv'}) { $synccmd .= " $pvcmd -s $pvsize |"; } $synccmd .= " $recvcmd"; @@ -806,9 +819,7 @@ sub getsendsize { } my $sourcessh; - if ($sourcehost ne '') { - $sourcessh = "$sshcmd $sourcehost"; - } + if ($sourcehost ne '') { $sourcessh = "$sshcmd $sourcehost"; } else { $sourcessh = ''; } my $getsendsizecmd = "$sourcessh $mysudocmd $zfscmd send -nP $snaps"; if ($debug) { print "DEBUG: getting estimated transfer size from source $sourcehost using \"$getsendsizecmd 2>&1 |\"...\n"; } From 7bd2bed91bf82dd526bffbcc1dfb49e96053345f Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sun, 7 Aug 2016 15:22:19 -0400 Subject: [PATCH 35/37] more uninitialized value shushing --- syncoid | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index 19109c1..db329e4 100755 --- a/syncoid +++ b/syncoid @@ -685,8 +685,10 @@ sub pruneoldsyncsnaps { sub getmatchingsnapshot { my ($targetsize, $snaps) = shift; foreach my $snap ( sort { $snaps{'source'}{$b}{'ctime'}<=>$snaps{'source'}{$a}{'ctime'} } keys %{ $snaps{'source'} }) { - if ($snaps{'source'}{$snap}{'ctime'} == $snaps{'target'}{$snap}{'ctime'}) { - return $snap; + if (defined $snaps{'target'}{$snap}{'ctime'}) { + if ($snaps{'source'}{$snap}{'ctime'} == $snaps{'target'}{$snap}{'ctime'}) { + return $snap; + } } } From d84fd9e1ab568c0c941014f3a0d2362595f93f65 Mon Sep 17 00:00:00 2001 From: Jim Salter Date: Sun, 7 Aug 2016 15:26:32 -0400 Subject: [PATCH 36/37] more warning fixing, sigh --- syncoid | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/syncoid b/syncoid index db329e4..cb9385e 100755 --- a/syncoid +++ b/syncoid @@ -329,8 +329,8 @@ sub getargs { } } - if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } - if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } + if (defined $args{'source-bwlimit'}) { $args{'source-bwlimit'} = "-R $args{'source-bwlimit'}"; } else { $args{'source-bwlimit'} = ''; } + if (defined $args{'target-bwlimit'}) { $args{'target-bwlimit'} = "-r $args{'target-bwlimit'}"; } else { $args{'target-bwlimit'} = ''; } if ($args{'r'}) { $args{'recursive'} = $args{'r'}; } From 7eb772fef855b96026428c721414edf6d546fdbf Mon Sep 17 00:00:00 2001 From: Nathan Walp Date: Tue, 16 Aug 2016 22:36:58 -0400 Subject: [PATCH 37/37] add --quiet option --- syncoid | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/syncoid b/syncoid index cb9385e..31c9955 100755 --- a/syncoid +++ b/syncoid @@ -22,6 +22,7 @@ if ($args{'version'}) { my $rawsourcefs = $args{'source'}; my $rawtargetfs = $args{'target'}; my $debug = $args{'debug'}; +my $quiet = $args{'quiet'}; my $zfscmd = '/sbin/zfs'; my $sshcmd = '/usr/bin/ssh'; @@ -169,7 +170,7 @@ sub syncdataset { my $disp_pvsize = readablebytes($pvsize); if ($pvsize == 0) { $disp_pvsize = 'UNKNOWN'; } my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); - print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; + if (!$quiet) { print "INFO: Sending oldest full snapshot $sourcefs\@$oldestsnap (~ $disp_pvsize) to new target filesystem:\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } # make sure target is (still) not currently in receive. @@ -200,7 +201,7 @@ sub syncdataset { die "Cannot sync now: $targetfs is already target of a zfs receive process.\n"; } - print "INFO: Updating new target filesystem with incremental $sourcefs\@$oldestsnap ... $newsyncsnap (~ $disp_pvsize):\n"; + if (!$quiet) { print "INFO: Updating new target filesystem with incremental $sourcefs\@$oldestsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } system($synccmd) == 0 or die "CRITICAL ERROR: $synccmd failed: $?"; @@ -246,7 +247,7 @@ sub syncdataset { if ($pvsize == 0) { $disp_pvsize = "UNKNOWN"; } my $synccmd = buildsynccmd($sendcmd,$recvcmd,$pvsize,$sourceisroot,$targetisroot); - print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; + if (!$quiet) { print "Sending incremental $sourcefs\@$matchingsnap ... $newsyncsnap (~ $disp_pvsize):\n"; } if ($debug) { print "DEBUG: $synccmd\n"; } system("$synccmd") == 0 or die "CRITICAL ERROR: $synccmd failed: $?"; @@ -270,14 +271,14 @@ sub getargs { my %novaluearg; my %validarg; - push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport'); + push my @validargs, ('debug','nocommandchecks','version','monitor-version','compress','source-bwlimit','target-bwlimit','dumpsnaps','recursive','r','sshkey','sshport','quiet'); foreach my $item (@validargs) { $validarg{$item} = 1; } - push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r'); + push my @novalueargs, ('debug','nocommandchecks','version','monitor-version','dumpsnaps','recursive','r','quiet'); foreach my $item (@novalueargs) { $novaluearg{$item} = 1; } while (my $rawarg = shift(@args)) { my $arg = $rawarg; - my $argvalue; + my $argvalue = ''; if ($rawarg =~ /=/) { # user specified the value for a CLI argument with = # instead of with blank space. separate appropriately. @@ -586,13 +587,13 @@ sub buildsynccmd { } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $bwlimit $mbufferoptions |"; } - if ($avail{'localpv'}) { $synccmd .= " $pvcmd -s $pvsize |"; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } $synccmd .= " $recvcmd"; } elsif ($sourcehost eq '') { # local source, remote target. #$synccmd = "$sendcmd | $pvcmd | $args{'compresscmd'} | $mbuffercmd | $sshcmd $targethost '$args{'decompresscmd'} | $mbuffercmd | $recvcmd'"; $synccmd = "$sendcmd |"; - if ($avail{'localpv'}) { $synccmd .= " $pvcmd -s $pvsize |"; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= " $pvcmd -s $pvsize |"; } if ($avail{'compress'}) { $synccmd .= " $args{'compresscmd'} |"; } if ($avail{'sourcembuffer'}) { $synccmd .= " $mbuffercmd $args{'source-bwlimit'} $mbufferoptions |"; } $synccmd .= " $sshcmd $targethost '"; @@ -608,7 +609,7 @@ sub buildsynccmd { $synccmd .= "' | "; if ($avail{'targetmbuffer'}) { $synccmd .= "$mbuffercmd $args{'target-bwlimit'} $mbufferoptions | "; } if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } - if ($avail{'localpv'}) { $synccmd .= "$pvcmd -s $pvsize | "; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } $synccmd .= "$recvcmd"; } else { #remote source, remote target... weird, but whatever, I'm not here to judge you. @@ -618,7 +619,7 @@ sub buildsynccmd { if ($avail{'sourcembuffer'}) { $synccmd .= " | $mbuffercmd $args{'source-bwlimit'} $mbufferoptions"; } $synccmd .= "' | "; if ($avail{'compress'}) { $synccmd .= "$args{'decompresscmd'} | "; } - if ($avail{'localpv'}) { $synccmd .= "$pvcmd -s $pvsize | "; } + if ($avail{'localpv'} && !$quiet) { $synccmd .= "$pvcmd -s $pvsize | "; } if ($avail{'compress'}) { $synccmd .= "$args{'compresscmd'} | "; } if ($avail{'localmbuffer'}) { $synccmd .= "$mbuffercmd $mbufferoptions | "; } $synccmd .= "$sshcmd $targethost '";