Merge pull request #832 from phreaker0/prepare-2.2.0

prepare 2.2.0

CHANGELIST

@@ -1,3 +1,17 @@
+2.2.0  [overall]   documentation updates, small fixes (@azmodude, @deviantintegral, @jimsalterjrs, @alexhaydock, @cbreak-black, @kd8bny, @JavaScriptDude, @veeableful, @rsheasby, @Topslakr, @mavhc, @adam-stamand, @joelishness, @jsoref, @dodexahedron, @phreaker0)
+       [syncoid]   implemented flag for preserving properties without the zfs -p flag (@phreaker0)
+       [syncoid]   implemented target snapshot deletion (@mat813)
+       [syncoid]   support bookmarks which are taken in the same second (@delxg, @phreaker0)
+       [syncoid]   exit with an error if the specified src dataset doesn't exist (@phreaker0)
+       [syncoid]   rollback is now done implicitly instead of explicit (@jimsalterjrs, @phreaker0)
+       [syncoid]   append a rand int to the socket name to prevent collisions with parallel invocations (@Gryd3)
+       [syncoid]   implemented support for ssh_config(5) files (@endreszabo)
+       [syncoid]   snapshot hold/unhold support (@rbike)
+       [sanoid]    handle duplicate key definitions gracefully (@phreaker0)
+       [syncoid]   implemented removal of conflicting snapshots with force-delete option (@phreaker0)
+       [sanoid]    implemented pre pruning script hook (@phreaker0)
+       [syncoid]   implemented direct connection support (bypass ssh) for the actual data transfer (@phreaker0)
+
 2.1.0  [overall]   documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0)
        [syncoid]   do not require user to be specified for syncoid (@aerusso)
        [syncoid]   implemented option for keeping sync snaps (@phreaker0)

README.md

@@ -323,6 +323,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup
 
 	This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends.
 
++ --preserve-properties
+
+	This argument tells syncoid to get all locally set dataset properties from the source and apply all supported ones on the target before writing any data. It's similar to the '-p' flag for zfs send but also works for encrypted datasets in non raw sends. This only applies to initial sends.
+
 + --delete-target-snapshots
 
 	With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source.

VERSION

@@ -1 +1 @@
-2.1.0
+2.2.0

findoid

@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '2.1.0';
+$::VERSION = '2.2.0';
 
 use strict;
 use warnings;

packages/debian/changelog

@@ -1,3 +1,21 @@
+sanoid (2.2.0) unstable; urgency=medium
+
+  [overall]   documentation updates, small fixes (@azmodude, @deviantintegral, @jimsalterjrs, @alexhaydock, @cbreak-black, @kd8bny, @JavaScriptDude, @veeableful, @rsheasby, @Topslakr, @mavhc, @adam-stamand, @joelishness, @jsoref, @dodexahedron, @phreaker0)
+  [syncoid]   implemented flag for preserving properties without the zfs -p flag (@phreaker0)
+  [syncoid]   implemented target snapshot deletion (@mat813)
+  [syncoid]   support bookmarks which are taken in the same second (@delxg, @phreaker0)
+  [syncoid]   exit with an error if the specified src dataset doesn't exist (@phreaker0)
+  [syncoid]   rollback is now done implicitly instead of explicit (@jimsalterjrs, @phreaker0)
+  [syncoid]   append a rand int to the socket name to prevent collisions with parallel invocations (@Gryd3)
+  [syncoid]   implemented support for ssh_config(5) files (@endreszabo)
+  [syncoid]   snapshot hold/unhold support (@rbike)
+  [sanoid]    handle duplicate key definitions gracefully (@phreaker0)
+  [syncoid]   implemented removal of conflicting snapshots with force-delete option (@phreaker0)
+  [sanoid]    implemented pre pruning script hook (@phreaker0)
+  [syncoid]   implemented direct connection support (bypass ssh) for the actual data transfer (@phreaker0)
+
+ -- Jim Salter <github@jrs-s.net>  Tue, 18 Jul 2023 10:04:00 +0200
+
 sanoid (2.1.0) unstable; urgency=medium
 
   [overall]   documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0)

packages/rhel/sanoid.spec

@@ -1,4 +1,4 @@
-%global version 2.1.0
+%global version 2.2.0
 %global git_tag v%{version}
 
 # Enable with systemctl "enable sanoid.timer"
@@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name}
 %endif
 
 %changelog
+* Tue Jul 18 2023 Christoph Klaffl <christoph@phreaker.eu> - 2.2.0
+- Bump to 2.2.0
 * Tue Nov 24 2020 Christoph Klaffl <christoph@phreaker.eu> - 2.1.0
 - Bump to 2.1.0
 * Wed Oct 02 2019 Christoph Klaffl <christoph@phreaker.eu> - 2.0.3

sanoid

@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '2.1.0';
+$::VERSION = '2.2.0';
 my $MINIMUM_DEFAULTS_VERSION = 2;
 
 use strict;

syncoid

@@ -4,7 +4,7 @@
 # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17.  A copy should also be available in this
 # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE.
 
-$::VERSION = '2.1.0';
+$::VERSION = '2.2.0';
 
 use strict;
 use warnings;
@@ -26,7 +26,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn
                    "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s",
                    "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark", "use-hold",
                    "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size,
-                   "delete-target-snapshots", "insecure-direct-connection=s")
+                   "delete-target-snapshots", "insecure-direct-connection=s", "preserve-properties")
                    or pod2usage(2);
 
 my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set
@@ -487,11 +487,19 @@ sub syncdataset {
 		}
 		my $oldestsnapescaped = escapeshellparam($oldestsnap);
 
-		if (defined $args{'preserve-recordsize'}) {
+		if (defined $args{'preserve-properties'}) {
+			my %properties = getlocalzfsvalues($sourcehost,$sourcefs,$sourceisroot);
+
+			foreach my $key (keys %properties) {
+				my $value = $properties{$key};
+				if ($debug) { print "DEBUG: will set $key to $value ...\n"; }
+				$recvoptions .= " -o $key=$value";
+			}
+		} elsif (defined $args{'preserve-recordsize'}) {
 			my $type = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'type');
 			if ($type eq "filesystem") {
 				my $recordsize = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'recordsize');
-				$recvoptions .= "-o recordsize=$recordsize"
+				$recvoptions .= "-o recordsize=$recordsize";
 			}
 		}
 
@@ -1335,6 +1343,55 @@ sub getzfsvalue {
 	return $wantarray ? ($value, $error) : $value;
 }
 
+sub getlocalzfsvalues {
+	my ($rhost,$fs,$isroot) = @_;
+
+	my $fsescaped = escapeshellparam($fs);
+
+	if ($rhost ne '') {
+		$rhost = "$sshcmd $rhost";
+		# double escaping needed
+		$fsescaped = escapeshellparam($fsescaped);
+	}
+
+	if ($debug) { print "DEBUG: getting locally set values of properties on $fs...\n"; }
+	my $mysudocmd;
+	if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; }
+	if ($debug) { print "$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped\n"; }
+	my ($values, $error, $exit) = capture {
+		system("$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped");
+	};
+
+	my %properties=();
+
+	if ($exit != 0) {
+		warn "WARNING: getlocalzfsvalues failed for $fs: $error";
+		if ($exitcode < 1) { $exitcode = 1; }
+		return %properties;
+	}
+
+	my @blacklist = (
+		"available", "compressratio", "createtxg", "creation", "clones",
+		"defer_destroy", "encryptionroot", "filesystem_count", "keystatus", "guid",
+		"logicalreferenced", "logicalused", "mounted", "objsetid", "origin",
+		"receive_resume_token", "redact_snaps", "referenced", "refcompressratio", "snapshot_count",
+		"type", "used", "usedbychildren", "usedbydataset", "usedbyrefreservation",
+		"usedbysnapshots", "userrefs", "snapshots_changed", "volblocksize", "written",
+		"version", "volsize", "casesensitivity", "normalization", "utf8only"
+	);
+	my %blacklisthash = map {$_ => 1} @blacklist;
+
+	foreach (split(/\n/,$values)) {
+		my @parts = split(/\t/, $_);
+		if (exists $blacklisthash{$parts[1]}) {
+			next;
+		}
+		$properties{$parts[1]} = $parts[2];
+	}
+
+	return %properties;
+}
+
 sub readablebytes {
 	my $bytes = shift;
 	my $disp;
@@ -2153,6 +2210,7 @@ Options:
   --create-bookmark     Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap)
   --use-hold            Adds a hold to the newest snapshot on the source and target after replication succeeds and removes the hold after the next succesful replication. The hold name incldues the identifier if set. This allows for separate holds in case of multiple targets
   --preserve-recordsize Preserves the recordsize on initial sends to the target
+  --preserve-properties Preserves locally set dataset properties similiar to the zfs send -p flag but this one will also work for encrypted datasets in non raw sends
   --no-rollback         Does not rollback snapshots on target (it probably requires a readonly target)
   --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source.
   --exclude=REGEX       Exclude specific datasets which match the given regular expression. Can be specified multiple times

tests/syncoid/9_preserve_properties/run.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# test preserving locally set properties from the src dataset to the target one
+
+set -x
+set -e
+
+. ../../common/lib.sh
+
+POOL_IMAGE="/tmp/syncoid-test-9.zpool"
+MOUNT_TARGET="/tmp/syncoid-test-9.mount"
+POOL_SIZE="1000M"
+POOL_NAME="syncoid-test-9"
+
+truncate -s "${POOL_SIZE}" "${POOL_IMAGE}"
+
+zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}"
+
+function cleanUp {
+  zpool export "${POOL_NAME}"
+}
+
+# export pool in any case
+trap cleanUp EXIT
+
+zfs create -o recordsize=16k -o xattr=on -o mountpoint=none -o primarycache=none "${POOL_NAME}"/src
+zfs create -V 100M -o volblocksize=8k "${POOL_NAME}"/src/zvol8
+zfs create -V 100M -o volblocksize=16k -o primarycache=all "${POOL_NAME}"/src/zvol16
+zfs create -V 100M -o volblocksize=64k "${POOL_NAME}"/src/zvol64
+zfs create -o recordsize=16k -o primarycache=none "${POOL_NAME}"/src/16
+zfs create -o recordsize=32k -o acltype=posixacl "${POOL_NAME}"/src/32
+
+../../../syncoid --preserve-properties --recursive --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst
+
+
+if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst)" != "16K" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get mountpoint -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get xattr -H -o value -t filesystem "${POOL_NAME}"/dst)" != "on" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "16K" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "none" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "32K" ]; then
+	exit 1
+fi
+
+if [ "$(zfs get acltype -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "posix" ]; then
+	exit 1
+fi