Merge cf0ecb30ae into a7e6c2db68

[sanoid] implemented dataset cache and fix race conditions

README.md

@@ -80,10 +80,6 @@ For more full details on sanoid.conf settings see [Wiki page](https://github.com
 
 	This will process your sanoid.conf file, it will NOT create snapshots, but it will purge expired ones.
 
-+ --force-prune
-
-	Purges expired snapshots even if a send/recv is in progress
-
 + --monitor-snapshots
 
 	This option is designed to be run by a Nagios monitoring system. It reports on the health of your snapshots.

sanoid

@@ -35,17 +35,6 @@ if (keys %args < 4) {
 	$args{'verbose'} = 1;
 }
 
-
-my $cacheTTL = 900; # 15 minutes
-
-# Allow a much older snapshot cache file than default if _only_ "--monitor-*" action commands are given
-# (ignore "--verbose", "--configdir" etc)
-if (($args{'monitor-snapshots'} || $args{'monitor-health'} || $args{'monitor-capacity'}) && ! ($args{'cron'} || $args{'force-update'} || $args{'take-snapshots'} || $args{'prune-snapshots'} || $args{'force-prune'})) {
-	# The command combination above must not assert true for any command that takes or prunes snapshots
-	$cacheTTL = 18000; # 5 hours
-	if ($args{'debug'}) { print "DEBUG: command combo means that the cache file (provided it exists) will be allowed to be older than default.\n"; }
-}
-
 # for compatibility reasons, older versions used hardcoded command paths
 $ENV{'PATH'} = $ENV{'PATH'} . ":/bin:/sbin";
 
@@ -57,25 +46,73 @@ my $zpool = 'zpool';
 my $conf_file = "$args{'configdir'}/sanoid.conf";
 my $default_conf_file = "$args{'configdir'}/sanoid.defaults.conf";
 
-# parse config file
-my %config = init($conf_file,$default_conf_file);
-
 my $cache_dir = $args{'cache-dir'};
 my $run_dir = $args{'run-dir'};
 
 make_path($cache_dir);
 make_path($run_dir);
 
-# if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL
-my $forcecacheupdate = 0;
+my $cacheTTL = 1200; # 20 minutes
+
+if ($args{'force-prune'}) {
+	warn "WARN: --force-prune argument is deprecated and its behavior is now standard";
+}
+
+# Allow a much older snapshot cache file than default if _only_ "--monitor-*" action commands are given
+# (ignore "--verbose", "--configdir" etc)
+if (
+	(
+		$args{'monitor-snapshots'}
+		|| $args{'monitor-health'}
+		|| $args{'monitor-capacity'}
+	) && ! (
+		$args{'cron'}
+		|| $args{'force-update'}
+		|| $args{'take-snapshots'}
+		|| $args{'prune-snapshots'}
+	)
+) {
+	# The command combination above must not assert true for any command that takes or prunes snapshots
+	$cacheTTL = 18000; # 5 hours
+	if ($args{'debug'}) { print "DEBUG: command combo means that the cache file (provided it exists) will be allowed to be older than default.\n"; }
+}
+
+# snapshot cache
 my $cache = "$cache_dir/snapshots.txt";
-my %snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate );
+
+# configured dataset cache
+my $cachedatasetspath = "$cache_dir/datasets.txt";
+my @cachedatasets;
+
+# parse config file
+my %config = init($conf_file,$default_conf_file);
+
 my %pruned;
 my %capacitycache;
 
-my %snapsbytype = getsnapsbytype( \%config, \%snaps );
+my %snaps;
+my %snapsbytype;
+my %snapsbypath;
 
-my %snapsbypath = getsnapsbypath( \%config, \%snaps );
+# get snapshot list only if needed
+if ($args{'monitor-snapshots'}
+	|| $args{'monitor-health'}
+	|| $args{'cron'}
+	|| $args{'take-snapshots'}
+	|| $args{'prune-snapshots'}
+	|| $args{'force-update'}
+	|| $args{'debug'}
+) {
+	my $forcecacheupdate = 0;
+	if ($args{'force-update'}) 	{
+		$forcecacheupdate = 1;
+	}
+
+	%snaps = getsnaps( \%config, $cacheTTL, $forcecacheupdate);
+
+	%snapsbytype = getsnapsbytype( \%config, \%snaps );
+	%snapsbypath = getsnapsbypath( \%config, \%snaps );
+}
 
 # let's make it a little easier to be consistent passing these hashes in the same order to each sub
 my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath );
@@ -84,7 +121,6 @@ if ($args{'debug'})		{ $args{'verbose'}=1; blabber (@params); }
 if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); }
 if ($args{'monitor-health'}) 	{ monitor_health(@params); }
 if ($args{'monitor-capacity'})  { monitor_capacity(@params); }
-if ($args{'force-update'}) 	{ my $snaps = getsnaps( \%config, $cacheTTL, 1 ); }
 
 if ($args{'cron'}) {
 	if ($args{'quiet'}) { $args{'verbose'} = 0; }
@@ -275,7 +311,6 @@ sub prune_snapshots {
 	my ($config, $snaps, $snapsbytype, $snapsbypath) = @_;
 
 	my %datestamp = get_date();
-	my $forcecacheupdate = 0;
 
 	foreach my $section (keys %config) {
 		if ($section =~ /^template/) { next; }
@@ -349,26 +384,23 @@ sub prune_snapshots {
 							}
 
 							if ($args{'verbose'}) { print "INFO: pruning $snap ... \n"; }
-							if (!$args{'force-prune'} && iszfsbusy($path)) {
-								if ($args{'verbose'}) { print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n"; }
-							} else {
-								if (! $args{'readonly'}) {
-									if (system($zfs, "destroy", $snap) == 0) {
-										$pruned{$snap} = 1;
-										if ($config{$dataset}{'pruning_script'}) {
-											$ENV{'SANOID_TARGET'} = $dataset;
-											$ENV{'SANOID_SNAPNAME'} = $snapname;
-											$ENV{'SANOID_SCRIPT'} = 'prune';
-											if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; }
-											my $ret = runscript('pruning_script',$dataset);
 
-											delete $ENV{'SANOID_TARGET'};
-											delete $ENV{'SANOID_SNAPNAME'};
-											delete $ENV{'SANOID_SCRIPT'};
-										}
-									} else {
-										warn "could not remove $snap : $?";
+							if (! $args{'readonly'}) {
+								if (system($zfs, "destroy", $snap) == 0) {
+									$pruned{$snap} = 1;
+									if ($config{$dataset}{'pruning_script'}) {
+										$ENV{'SANOID_TARGET'} = $dataset;
+										$ENV{'SANOID_SNAPNAME'} = $snapname;
+										$ENV{'SANOID_SCRIPT'} = 'prune';
+										if ($args{'verbose'}) { print "executing pruning_script '".$config{$dataset}{'pruning_script'}."' on dataset '$dataset'\n"; }
+										my $ret = runscript('pruning_script',$dataset);
+
+										delete $ENV{'SANOID_TARGET'};
+										delete $ENV{'SANOID_SNAPNAME'};
+										delete $ENV{'SANOID_SCRIPT'};
 									}
+								} else {
+									warn "could not remove $snap : $?";
 								}
 							}
 						}
@@ -826,7 +858,7 @@ sub getsnaps {
 		if (checklock('sanoid_cacheupdate')) {
 			writelock('sanoid_cacheupdate');
 			if ($args{'verbose'}) {
-				if ($args{'force-update'}) {
+				if ($forcecacheupdate) {
 					print "INFO: cache forcibly expired - updating from zfs list.\n";
 				} else {
 					print "INFO: cache expired - updating from zfs list.\n";
@@ -836,9 +868,10 @@ sub getsnaps {
 			@rawsnaps = <FH>;
 			close FH;
 
-			open FH, "> $cache" or die 'Could not write to $cache!\n';
+			open FH, "> $cache.tmp" or die 'Could not write to $cache.tmp!\n';
 			print FH @rawsnaps;
 			close FH;
+			rename("$cache.tmp", "$cache") or die 'Could not rename to $cache!\n';
 			removelock('sanoid_cacheupdate');
 		} else {
 			if ($args{'verbose'}) { print "INFO: deferring cache update - valid cache update lock held by another sanoid process.\n"; }
@@ -901,6 +934,20 @@ sub init {
 		die "FATAL: you're using sanoid.defaults.conf v$defaults_version, this version of sanoid requires a minimum sanoid.defaults.conf v$MINIMUM_DEFAULTS_VERSION";
 	}
 
+	my @updatedatasets;
+
+	# load dataset cache if valid
+	if (!$args{'force-update'} && -f $cachedatasetspath) {
+		my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat($cachedatasetspath);
+
+		if ((time() - $mtime) <= $cacheTTL) {
+			if ($args{'debug'}) { print "DEBUG: dataset cache not expired (" . (time() - $mtime) . " seconds old with TTL of $cacheTTL): pulling dataset list from cache.\n"; }
+			open FH, "< $cachedatasetspath";
+			@cachedatasets = <FH>;
+			close FH;
+		}
+	}
+
 	foreach my $section (keys %ini) {
 
 		# first up - die with honor if unknown parameters are set in any modules or templates by the user.
@@ -990,6 +1037,10 @@ sub init {
 			$config{$section}{'path'} = $section;
 		}
 
+		if (! @cachedatasets) {
+			push (@updatedatasets, "$config{$section}{'path'}\n");
+		}
+
 		# how 'bout some recursion? =)
 		if ($config{$section}{'zfs_recursion'} && $config{$section}{'zfs_recursion'} == 1 && $config{$section}{'autosnap'} == 1) {
 			warn "ignored autosnap configuration for '$section' because it's part of a zfs recursion.\n";
@@ -1007,6 +1058,10 @@ sub init {
 
 			@datasets = getchilddatasets($config{$section}{'path'});
 			DATASETS: foreach my $dataset(@datasets) {
+				if (! @cachedatasets) {
+					push (@updatedatasets, $dataset);
+				}
+
 				chomp $dataset;
 
 				if ($zfsRecursive) {
@@ -1038,9 +1093,27 @@ sub init {
 				$config{$dataset}{'initialized'} = 1;
 			}
 		}
+	}
 
-
-
+	# update dataset cache if it was unused
+	if (! @cachedatasets) {
+		if (checklock('sanoid_cachedatasetupdate')) {
+			writelock('sanoid_cachedatasetupdate');
+			if ($args{'verbose'}) {
+				if ($args{'force-update'}) {
+					print "INFO: dataset cache forcibly expired - updating from zfs list.\n";
+				} else {
+					print "INFO: dataset cache expired - updating from zfs list.\n";
+				}
+			}
+			open FH, "> $cachedatasetspath.tmp" or die 'Could not write to $cachedatasetspath.tmp!\n';
+			print FH @updatedatasets;
+			close FH;
+			rename("$cachedatasetspath.tmp", "$cachedatasetspath") or die 'Could not rename to $cachedatasetspath!\n';
+			removelock('sanoid_cachedatasetupdate');
+		} else {
+			if ($args{'verbose'}) { print "INFO: deferring dataset cache update - valid cache update lock held by another sanoid process.\n"; }
+		}
 	}
 
 	return %config;
@@ -1557,30 +1630,6 @@ sub writelock {
 	close FH;
 }
 
-sub iszfsbusy {
-	# check to see if ZFS filesystem passed in as argument currently has a zfs send or zfs receive process referencing it.
-	# return true if busy (currently being sent or received), return false if not.
-
-        my $fs = shift;
-        # if (args{'debug'}) { print "DEBUG: checking to see if $fs on is already in zfs receive using $pscmd -Ao args= ...\n"; }
-
-        open PL, "$pscmd -Ao args= |";
-        my @processes = <PL>;
-        close PL;
-
-        foreach my $process (@processes) {
-		# if ($args{'debug'}) { print "DEBUG: checking process $process...\n"; }
-                if ($process =~ /zfs *(send|receive|recv).*$fs/) {
-                        # there's already a zfs send/receive process for our target filesystem - return true
-                        # if ($args{'debug'}) { print "DEBUG: process $process matches target $fs!\n"; }
-                        return 1;
-                }
-        }
-
-        # no zfs receive processes for our target filesystem found - return false
-        return 0;
-}
-
 #######################################################################################################################3
 #######################################################################################################################3
 #######################################################################################################################3
@@ -1590,6 +1639,30 @@ sub getchilddatasets {
 	my $fs = shift;
 	my $mysudocmd = '';
 
+	# use dataset cache if available
+	if (@cachedatasets) {
+		my $foundparent = 0;
+		my @cachechildren = ();
+		foreach my $dataset (@cachedatasets) {
+			chomp $dataset;
+			my $ret = rindex $dataset, "${fs}/", 0;
+			if ($ret == 0) {
+				push (@cachechildren, $dataset);
+			} else {
+				if ($dataset eq $fs) {
+					$foundparent = 1;
+				}
+			}
+		}
+
+		# sanity check
+		if ($foundparent) {
+			return @cachechildren;
+		}
+
+		# fallback if cache misses items for whatever reason
+	}
+
 	my $getchildrencmd = "$mysudocmd $zfs list -o name -t filesystem,volume -Hr $fs |";
 	 if ($args{'debug'}) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; }
 	open FH, $getchildrencmd;
@@ -1636,16 +1709,17 @@ sub removecachedsnapshots {
 	my @rawsnaps = <FH>;
 	close FH;
 
-	open FH, "> $cache" or die 'Could not write to $cache!\n';
+	open FH, "> $cache.tmp" or die 'Could not write to $cache.tmp!\n';
 	foreach my $snapline ( @rawsnaps ) {
 		my @columns = split("\t", $snapline);
 		my $snap = $columns[0];
 		print FH $snapline unless ( exists($pruned{$snap}) );
 	}
 	close FH;
+	rename("$cache.tmp", "$cache") or die 'Could not rename to $cache!\n';
 
 	removelock('sanoid_cacheupdate');
-	%snaps = getsnaps(\%config,$cacheTTL,$forcecacheupdate);
+	%snaps = getsnaps(\%config,$cacheTTL,0);
 
 	# clear hash
 	undef %pruned;
@@ -1752,7 +1826,6 @@ Options:
   --monitor-snapshots   Reports on snapshot "health", in a Nagios compatible format
   --take-snapshots      Creates snapshots as specified in sanoid.conf
   --prune-snapshots     Purges expired snapshots as specified in sanoid.conf
-  --force-prune         Purges expired snapshots even if a send/recv is in progress
 
   --help                Prints this helptext
   --version             Prints the version number