From f3d4d309b5a6ccba11512b698041e80a0c8518fa Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 18 Jul 2023 08:38:40 +0200 Subject: [PATCH] implemented flag for preserving properties without the zfs -p flag --- README.md | 4 ++ syncoid | 64 ++++++++++++++++++++- tests/syncoid/9_preserve_properties/run.sh | 66 ++++++++++++++++++++++ 3 files changed, 131 insertions(+), 3 deletions(-) create mode 100755 tests/syncoid/9_preserve_properties/run.sh diff --git a/README.md b/README.md index 7a10dac..6549617 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends. ++ --preserve-properties + + This argument tells syncoid to get all locally set dataset properties from the source and apply all supported ones on the target before writing any data. It's similar to the '-p' flag for zfs send but also works for encrypted datasets in non raw sends. This only applies to initial sends. + + --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source. diff --git a/syncoid b/syncoid index eb38539..6cde9f9 100755 --- a/syncoid +++ b/syncoid @@ -26,7 +26,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark", "use-hold", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size, - "delete-target-snapshots", "insecure-direct-connection=s") + "delete-target-snapshots", "insecure-direct-connection=s", "preserve-properties") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -487,11 +487,19 @@ sub syncdataset { } my $oldestsnapescaped = escapeshellparam($oldestsnap); - if (defined $args{'preserve-recordsize'}) { + if (defined $args{'preserve-properties'}) { + my %properties = getlocalzfsvalues($sourcehost,$sourcefs,$sourceisroot); + + foreach my $key (keys %properties) { + my $value = $properties{$key}; + if ($debug) { print "DEBUG: will set $key to $value ...\n"; } + $recvoptions .= " -o $key=$value"; + } + } elsif (defined $args{'preserve-recordsize'}) { my $type = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'type'); if ($type eq "filesystem") { my $recordsize = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'recordsize'); - $recvoptions .= "-o recordsize=$recordsize" + $recvoptions .= "-o recordsize=$recordsize"; } } @@ -1335,6 +1343,55 @@ sub getzfsvalue { return $wantarray ? ($value, $error) : $value; } +sub getlocalzfsvalues { + my ($rhost,$fs,$isroot) = @_; + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + + if ($debug) { print "DEBUG: getting locally set values of properties on $fs...\n"; } + my $mysudocmd; + if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + if ($debug) { print "$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped\n"; } + my ($values, $error, $exit) = capture { + system("$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped"); + }; + + my %properties=(); + + if ($exit != 0) { + warn "WARNING: getlocalzfsvalues failed for $fs: $error"; + if ($exitcode < 1) { $exitcode = 1; } + return %properties; + } + + my @blacklist = ( + "available", "compressratio", "createtxg", "creation", "clones", + "defer_destroy", "encryptionroot", "filesystem_count", "keystatus", "guid", + "logicalreferenced", "logicalused", "mounted", "objsetid", "origin", + "receive_resume_token", "redact_snaps", "referenced", "refcompressratio", "snapshot_count", + "type", "used", "usedbychildren", "usedbydataset", "usedbyrefreservation", + "usedbysnapshots", "userrefs", "snapshots_changed", "volblocksize", "written", + "version", "volsize", "casesensitivity", "normalization", "utf8only" + ); + my %blacklisthash = map {$_ => 1} @blacklist; + + foreach (split(/\n/,$values)) { + my @parts = split(/\t/, $_); + if (exists $blacklisthash{$parts[1]}) { + next; + } + $properties{$parts[1]} = $parts[2]; + } + + return %properties; +} + sub readablebytes { my $bytes = shift; my $disp; @@ -2153,6 +2210,7 @@ Options: --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) --use-hold Adds a hold to the newest snapshot on the source and target after replication succeeds and removes the hold after the next succesful replication. The hold name incldues the identifier if set. This allows for separate holds in case of multiple targets --preserve-recordsize Preserves the recordsize on initial sends to the target + --preserve-properties Preserves locally set dataset properties similiar to the zfs send -p flag but this one will also work for encrypted datasets in non raw sends --no-rollback Does not rollback snapshots on target (it probably requires a readonly target) --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source. --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times diff --git a/tests/syncoid/9_preserve_properties/run.sh b/tests/syncoid/9_preserve_properties/run.sh new file mode 100755 index 0000000..497ce9a --- /dev/null +++ b/tests/syncoid/9_preserve_properties/run.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# test preserving locally set properties from the src dataset to the target one + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-9.zpool" +MOUNT_TARGET="/tmp/syncoid-test-9.mount" +POOL_SIZE="1000M" +POOL_NAME="syncoid-test-9" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create -o recordsize=16k -o xattr=on -o mountpoint=none -o primarycache=none "${POOL_NAME}"/src +zfs create -V 100M -o volblocksize=8k "${POOL_NAME}"/src/zvol8 +zfs create -V 100M -o volblocksize=16k -o primarycache=all "${POOL_NAME}"/src/zvol16 +zfs create -V 100M -o volblocksize=64k "${POOL_NAME}"/src/zvol64 +zfs create -o recordsize=16k -o primarycache=none "${POOL_NAME}"/src/16 +zfs create -o recordsize=32k -o acltype=posixacl "${POOL_NAME}"/src/32 + +../../../syncoid --preserve-properties --recursive --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst + + +if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst)" != "16K" ]; then + exit 1 +fi + +if [ "$(zfs get mountpoint -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then + exit 1 +fi + +if [ "$(zfs get xattr -H -o value -t filesystem "${POOL_NAME}"/dst)" != "on" ]; then + exit 1 +fi + +if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then + exit 1 +fi + +if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "16K" ]; then + exit 1 +fi + +if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "none" ]; then + exit 1 +fi + +if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "32K" ]; then + exit 1 +fi + +if [ "$(zfs get acltype -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "posix" ]; then + exit 1 +fi