Merge branch 'dev'

This commit is contained in:
Jim Salter 2015-04-01 18:03:02 -04:00
commit a030aa9903
6 changed files with 191 additions and 163 deletions

View File

@ -1,3 +1,7 @@
1.1.0 woooo - working recursive definitions in Sanoid! Also intelligent config errors in Sanoid; will die with errors if unknown config value is set.
1.0.20 greatly cleaned up config parsing in sanoid, got rid of 'hardcoded defaults' in favor of /etc/sanoid/sanoid.defaults.conf
1.0.19 working recursive sync (sync specified dataset and all child datasets, ie pool/ds, pool/ds/1, pool, ds/1/a, pool/ds/2 ...) with --recursive or -r in syncoid!
1.0.18 updated syncoid to break sync out of main routine and into syncdataset(). this will allow doing recursive sync, in next update :)

View File

@ -1 +1 @@
1.0.19
1.1.0

198
sanoid
View File

@ -4,7 +4,7 @@
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
my $version = '1.0.16';
my $version = '1.1.0';
use strict;
use Config::IniFiles; # read samba-style conf file
@ -15,10 +15,15 @@ use Time::Local; # to parse dates in reverse
my $pscmd = '/bin/ps';
my $zfs = '/sbin/zfs';
my $conf_file = '/etc/sanoid/sanoid.conf';
my $default_conf_file = '/etc/sanoid/sanoid.defaults.conf';
# parse CLI arguments
my %args = getargs(@ARGV);
# parse config file
my %config = init($conf_file);
my %config = init($conf_file,$default_conf_file);
# if we call getsnaps(%config,1) it will forcibly update the cache, TTL or no TTL
my $forcecacheupdate = 0;
@ -32,8 +37,6 @@ my %snapsbypath = getsnapsbypath( \%config, \%snaps );
# let's make it a little easier to be consistent passing these hashes in the same order to each sub
my @params = ( \%config, \%snaps, \%snapsbytype, \%snapsbypath );
my %args = getargs(@ARGV);
if ($args{'debug'}) { $args{'verbose'}=1; blabber (@params); }
if ($args{'monitor-snapshots'}) { monitor_snapshots(@params); }
if ($args{'monitor-health'}) { monitor_health(@params); }
@ -218,7 +221,7 @@ sub prune_snapshots {
if (iszfsbusy($path)) {
print "INFO: deferring pruning of $snap - $path is currently in zfs send or receive.\n";
} else {
system($zfs, "destroy","-Rr",$snap) == 0 or die "could not remove $snap : $?";
if (! $args{'readonly'}) { system($zfs, "destroy","-Rr",$snap) == 0 or die "could not remove $snap : $?"; }
}
}
removelock('sanoid_pruning');
@ -324,9 +327,11 @@ sub take_snapshots {
if ( (scalar(@newsnaps)) > 0) {
foreach my $snap ( @newsnaps ) {
if ($args{'verbose'}) { print "taking snapshot $snap\n"; }
system($zfs, "snapshot", "$snap");
# make sure we don't end up with multiple snapshots with the same ctime
sleep 1;
if (!$args{'readonly'}) {
system($zfs, "snapshot", "$snap");
# make sure we don't end up with multiple snapshots with the same ctime
sleep 1;
}
}
$forcecacheupdate = 1;
%snaps = getsnaps(%config,$cacheTTL,$forcecacheupdate);
@ -341,9 +346,9 @@ sub blabber {
my ($config, $snaps, $snapsbytype, $snapsbypath) = @_;
#$Data::Dumper::Sortkeys = 1;
#print "****** CONFIGS ******\n";
#print Dumper(\%config);
$Data::Dumper::Sortkeys = 1;
print "****** CONFIGS ******\n";
print Dumper(\%config);
#print "****** SNAPSHOTS ******\n";
#print Dumper(\%snaps);
#print "****** SNAPSBYTYPE ******\n";
@ -515,93 +520,71 @@ sub getsnaps {
####################################################################################
sub init {
my ($conf_file) = @_;
my ($conf_file, $default_conf_file) = @_;
my %config;
tie my %defaults, 'Config::IniFiles', ( -file => $default_conf_file ) or die 'cannot load $conf_file - please restore a clean copy, this is not a user-editable file!';
tie my %ini, 'Config::IniFiles', ( -file => $conf_file );
# we'll use these later to normalize potentially true and false values on any toggle keys
my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor');
my @toggles = ('autosnap','autoprune','monitor_dont_warn','monitor_dont_crit','monitor','recursive');
my @istrue=(1,"true","True","TRUE","yes","Yes","YES","on","On","ON");
my @isfalse=(0,"false","False","FALSE","no","No","NO","off","Off","OFF");
foreach my $section (keys %ini) {
if ($section =~ /^template_/) { next; } # don't process templates directly
#
# set hardcoded default values (overridden by template_default, local use_template, then local settings)
#
# these are the ages (in periodicity) after which we want to automatically prune snapshots
# (assuming autoprune is on). For example if hourly=48, any hourly snapshots 49+ hours old
# will be automatically pruned.
#
# we will not take (and will immediately prune, should any exist) any snapshot types set to 0.
#
$config{$section}{'autoprune'} = 1;
$config{$section}{'hourly'} = 48;
$config{$section}{'daily'} = 90;
$config{$section}{'monthly'} = 6;
$config{$section}{'yearly'} = 0;
# if still less than min_percent_free space is free after normal pruning, prune from
# oldest to newest until min_percent_free is achieved
$config{$section}{'min_percent_free'} = 10;
# We will automatically take snapshots if autosnap is on, at the desired times configured
# below (or immediately, if we don't have one since the last preferred time for that type).
$config{$section}{'autosnap'} = 1;
# these are preferred times to take snapshots
# hourly - top of the hour
$config{$section}{'hourly_min'} = 0;
# daily - at 23:59 (most people expect a daily to contain everything done DURING that day)
$config{$section}{'daily_hour'} = 23;
$config{$section}{'daily_min'} = 59;
# monthly - immediately at the beginning of the month (ie 00:00 of day 1)
$config{$section}{'monthly_mday'} = 1;
$config{$section}{'monthly_hour'} = 0;
$config{$section}{'monthly_min'} = 0;
# yearly - immediately at the beginning of the year (ie 00:00 on Jan 1)
$config{$section}{'yearly_mday'} = 1;
$config{$section}{'yearly_mon'} = 1;
$config{$section}{'yearly_hour'} = 0;
$config{$section}{'yearly_min'} = 0;
$config{$section}{'monitor_dont_warn'} = 0;
$config{$section}{'monitor_dont_crit'} = 0;
$config{$section}{'hourly_warn'} = 90;
$config{$section}{'hourly_crit'} = 360;
$config{$section}{'daily_warn'} = 28;
$config{$section}{'daily_crit'} = 32;
$config{$section}{'monthly_warn'} = 32;
$config{$section}{'monthly_crit'} = 35;
$config{$section}{'yearly_warn'} = 0;
$config{$section}{'yearly_crit'} = 0;
# set $config{$section}{'template'} to deepest template level existent for this $section
my $template = 'hardcoded';
if (defined $ini{'template_default'}) {$template = 'default'; }
# we've already set everything for the section to hardcoded default values.
# now, we push the section itself, its use_template setting, and then the default template
# (if present) into an array, so that we can pop them in order and use all the values
# defined in each section to override the hardcoded defaults:
#
# hardcoded -> default template -> use_template -> section local
#
push my @templates, $section;
if (defined $ini{$section}{'use_template'}) {
$template = 'template_'.$ini{$section}{'use_template'};
push @templates, $template;
# first up - die with honor if unknown parameters are set in any modules or templates by the user.
foreach my $key (keys %{$ini{$section}}) {
if (! defined ($defaults{'template_default'}{$key})) {
die "FATAL ERROR: I don't understand the setting $key you've set in \[$section\] in $conf_file.\n";
}
}
push @templates, 'template_default';
# override as appropriate: hardcoded -> default template -> use_template -> local settings
while (my $template = pop(@templates)) {
if (defined $ini{$template}) {
foreach my $key (keys %{ $ini{$template} }) {
$config{$section}{$key} = $ini{$template}{$key};
if ($section =~ /^template_/) { next; } # don't process templates directly
# only set defaults on sections that haven't already been initialized - this allows us to override values
# for sections directly when they've already been defined recursively, without starting them over from scratch.
if (! defined ($config{$section}{'initialized'})) {
if ($args{'debug'}) { print "DEBUG: initializing \$config\{$section\} with default values from $default_conf_file.\n"; }
# set default values from %defaults, which can then be overriden by template
# and/or local settings within the module.
foreach my $key (keys %{$defaults{'template_default'}}) {
if (! ($key =~ /template|recursive/)) {
$config{$section}{$key} = $defaults{'template_default'}{$key};
}
}
# override with values from user-defined default template, if any
foreach my $key (keys %{$ini{'template_default'}}) {
if (! ($key =~ /template|recursive/)) {
if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined default template.\n"; }
$config{$section}{$key} = $ini{'template_default'}{$key};
}
}
}
# override with values from user-defined templates applied to this module,
# in the order they were specified (ie use_template = default,production,mytemplate)
if (defined $ini{$section}{'use_template'}) {
my @templates = split (' *, *',$ini{$section}{'use_template'});
foreach my $rawtemplate (@templates) {
my $template = 'template_'.$rawtemplate;
foreach my $key (keys %{$ini{$template}}) {
if (! ($key =~ /template|recursive/)) {
if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value from user-defined template $template.\n"; }
$config{$section}{$key} = $ini{$template}{$key};
}
}
}
}
# override with any locally set values in the module itself
foreach my $key (keys %{$ini{$section}} ) {
if (! ($key =~ /template|recursive/)) {
if ($args{'debug'}) { print "DEBUG: overriding $key on $section with value directly set in module.\n"; }
$config{$section}{$key} = $ini{$section}{$key};
}
}
# make sure that true values are true and false values are false for any toggled values
@ -615,8 +598,31 @@ sub init {
}
# section path is the section name, unless section path has been explicitly defined
$config{$section}{'path'} = $section;
if (defined $ini{$section}{'path'}) { $config{$section}{'path'} = $ini{$section}{'path'}; }
if (defined ($ini{$section}{'path'})) {
$config{$section}{'path'} = $ini{$section}{'path'};
} else {
$config{$section}{'path'} = $section;
}
# how 'bout some recursion? =)
my @datasets;
if ($ini{$section}{'recursive'}) {
@datasets = getchilddatasets($config{$section}{'path'});
foreach my $dataset(@datasets) {
chomp $dataset;
foreach my $key (keys %{$config{$section}} ) {
if (! ($key =~ /template|recursive/)) {
if ($args{'debug'}) { print "DEBUG: recursively setting $key from $section to $dataset.\n"; }
$config{$dataset}{$key} = $config{$section}{$key};
}
}
$config{$dataset}{'path'} = $dataset;
$config{$dataset}{'initialized'} = 1;
}
}
}
return %config;
@ -1013,8 +1019,8 @@ sub getargs {
my %validargs;
my %novalueargs;
push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots';
push my @novalueargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots';
push my @validargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly';
push my @novalueargs, 'verbose','debug','version','monitor-health','monitor-snapshots','force-update','cron','take-snapshots','prune-snapshots','readonly';
foreach my $item (@validargs) { $validargs{$item}=1; }
foreach my $item (@novalueargs) { $novalueargs{$item}=1; }
@ -1069,3 +1075,17 @@ sub getargs {
return %args;
}
sub getchilddatasets {
# for later, if we make sanoid itself support sudo use
my $fs = shift;
my $mysudocmd;
my $getchildrencmd = "$mysudocmd $zfs list -o name -Hr $fs |";
if ($args{'debug'}) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; }
open FH, $getchildrencmd;
my @children = <FH>;
close FH;
return @children;
}

View File

@ -5,8 +5,10 @@
# name your backup modules with the path to their ZFS dataset - no leading slash.
[zpoolname/datasetname]
# pick a template - they're defined (and editable) below.
use_template = production
# pick one or more templates - they're defined (and editable) below. Comma separated, processed in order.
# in this example, template_demo's daily value overrides template_production's daily value.
use_template = production,demo
# if you want to, you can override settings in the template directly inside module definitions like this.
# in this example, we override the template to only keep 12 hourly and 1 monthly snapshot for this dataset.
hourly = 12
@ -18,8 +20,11 @@
#############################
# name your templates template_templatename. you can create your own, and use them in your module definitions above.
[template_demo]
daily = 60
[template_production]
template = yes
hourly = 36
daily = 30
monthly = 3
@ -28,13 +33,11 @@
autoprune = yes
[template_backup]
template = yes
autoprune = yes
hourly = 30
daily = 90
monthly = 12
yearlies = 0
yearly = 0
### don't take new snapshots - snapshots on backup
### datasets are replicated in from source, not
@ -50,68 +53,3 @@
daily_crit = 60
###################################################################################
# default template - contains same values as hardcoded, unless you override here. #
# ALL values set here, so useful as documentation even if #
# nothing is actually overridden. #
###################################################################################
[template_default]
template = yes
# If any snapshot type is set to 0, we will not take snapshots for it - and will immediately
# prune any of those type snapshots already present.
#
# Otherwise, if autoprune is set, we will prune any snapshots of that type which are older
# than (setting * periodicity) - so if daily = 90, we'll prune any dailies older than 90 days.
autoprune = yes
hourly = 48
daily = 90
monthly = 6
yearly = 0
min_percent_free = 10
# We will automatically take snapshots if autosnap is on, at the desired times configured
# below (or immediately, if we don't have one since the last preferred time for that type).
#
# Note that we will not take snapshots for a given type if that type is set to 0 above,
# regardless of the autosnap setting - for example, if yearly=0 we will not take yearlies
# even if we've defined a preferred time for yearlies and autosnap is on.
autosnap = 1;
# hourly - top of the hour
hourly_min = 0;
# daily - at 23:59 (most people expect a daily to contain everything done DURING that day)
daily_hour = 23;
daily_min = 59;
# monthly - immediately at the beginning of the month (ie 00:00 of day 1)
monthly_mday = 1;
monthly_hour = 0;
monthly_min = 0;
# yearly - immediately at the beginning of the year (ie 00:00 on Jan 1)
yearly_mon = 1;
yearly_mday = 1;
yearly_hour = 0;
yearly_min = 0;
# monitoring plugin - define warn / crit levels for each snapshot type by age, in units of one period down
# example hourly_warn = 90 means issue WARNING if most recent hourly snapshot is not less than 90 minutes old,
# daily_crit = 36 means issue CRITICAL if most recent daily snapshot is not less than 36 hours old,
# monthly_warn = 36 means issue WARNING if most recent monthly snapshot is not less than 36 days old... etc.
#
# monitor_dont_warn = yes will cause the monitoring service to report warnings as text, but with status OK.
# monitor_dont_crit = yes will cause the monitoring service to report criticals as text, but with status OK.
#
# setting any value to 0 will keep the monitoring service from monitoring that snapshot type on that section at all.
monitor = yes
monitor_dont_warn = no
monitor_dont_crit = no
hourly_warn = 90
hourly_crit = 360
daily_warn = 28
daily_crit = 32
monthly_warn = 32
monthly_crit = 35
yearly_warn = 0
yearly_crit = 0

66
sanoid.defaults.conf Normal file
View File

@ -0,0 +1,66 @@
###################################################################################
# default template - DO NOT EDIT THIS FILE DIRECTLY. #
# If you wish to override default values, you can create your #
# own [template_default] in /etc/sanoid/sanoid.conf. #
# #
# you have been warned. #
###################################################################################
[template_default]
# If any snapshot type is set to 0, we will not take snapshots for it - and will immediately
# prune any of those type snapshots already present.
#
# Otherwise, if autoprune is set, we will prune any snapshots of that type which are older
# than (setting * periodicity) - so if daily = 90, we'll prune any dailies older than 90 days.
autoprune = yes
hourly = 48
daily = 90
monthly = 6
yearly = 0
min_percent_free = 10
# We will automatically take snapshots if autosnap is on, at the desired times configured
# below (or immediately, if we don't have one since the last preferred time for that type).
#
# Note that we will not take snapshots for a given type if that type is set to 0 above,
# regardless of the autosnap setting - for example, if yearly=0 we will not take yearlies
# even if we've defined a preferred time for yearlies and autosnap is on.
autosnap = 1;
# hourly - top of the hour
hourly_min = 0;
# daily - at 23:59 (most people expect a daily to contain everything done DURING that day)
daily_hour = 23;
daily_min = 59;
# monthly - immediately at the beginning of the month (ie 00:00 of day 1)
monthly_mday = 1;
monthly_hour = 0;
monthly_min = 0;
# yearly - immediately at the beginning of the year (ie 00:00 on Jan 1)
yearly_mon = 1;
yearly_mday = 1;
yearly_hour = 0;
yearly_min = 0;
# monitoring plugin - define warn / crit levels for each snapshot type by age, in units of one period down
# example hourly_warn = 90 means issue WARNING if most recent hourly snapshot is not less than 90 minutes old,
# daily_crit = 36 means issue CRITICAL if most recent daily snapshot is not less than 36 hours old,
# monthly_warn = 36 means issue WARNING if most recent monthly snapshot is not less than 36 days old... etc.
#
# monitor_dont_warn = yes will cause the monitoring service to report warnings as text, but with status OK.
# monitor_dont_crit = yes will cause the monitoring service to report criticals as text, but with status OK.
#
# setting any value to 0 will keep the monitoring service from monitoring that snapshot type on that section at all.
monitor = yes
monitor_dont_warn = no
monitor_dont_crit = no
hourly_warn = 90
hourly_crit = 360
daily_warn = 28
daily_crit = 32
monthly_warn = 32
monthly_crit = 35
yearly_warn = 0
yearly_crit = 0

View File

@ -4,7 +4,7 @@
# from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this
# project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
my $version = '1.0.19';
my $version = '1.1.0';
use strict;
use Data::Dumper;
@ -82,7 +82,7 @@ sub getchilddatasets {
if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; }
if ($rhost ne '') { $rhost = "$sshcmd $rhost"; }
my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -Hpr $fs |";
my $getchildrencmd = "$rhost $mysudocmd $zfscmd list -o name -Hr $fs |";
if ($debug) { print "DEBUG: getting list of child datasets on $fs using $getchildrencmd...\n"; }
open FH, $getchildrencmd;
my @children = <FH>;