From f3d4d309b5a6ccba11512b698041e80a0c8518fa Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 18 Jul 2023 08:38:40 +0200 Subject: [PATCH 1/2] implemented flag for preserving properties without the zfs -p flag --- README.md | 4 ++ syncoid | 64 ++++++++++++++++++++- tests/syncoid/9_preserve_properties/run.sh | 66 ++++++++++++++++++++++ 3 files changed, 131 insertions(+), 3 deletions(-) create mode 100755 tests/syncoid/9_preserve_properties/run.sh diff --git a/README.md b/README.md index 7a10dac3..6549617b 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,10 @@ As of 1.4.18, syncoid also automatically supports and enables resume of interrup This argument tells syncoid to set the recordsize on the target before writing any data to it matching the one set on the replication src. This only applies to initial sends. ++ --preserve-properties + + This argument tells syncoid to get all locally set dataset properties from the source and apply all supported ones on the target before writing any data. It's similar to the '-p' flag for zfs send but also works for encrypted datasets in non raw sends. This only applies to initial sends. + + --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source. diff --git a/syncoid b/syncoid index eb385393..6cde9f99 100755 --- a/syncoid +++ b/syncoid @@ -26,7 +26,7 @@ GetOptions(\%args, "no-command-checks", "monitor-version", "compress=s", "dumpsn "debug", "quiet", "no-stream", "no-sync-snap", "no-resume", "exclude=s@", "skip-parent", "identifier=s", "no-clone-handling", "no-privilege-elevation", "force-delete", "no-rollback", "create-bookmark", "use-hold", "pv-options=s" => \$pvoptions, "keep-sync-snap", "preserve-recordsize", "mbuffer-size=s" => \$mbuffer_size, - "delete-target-snapshots", "insecure-direct-connection=s") + "delete-target-snapshots", "insecure-direct-connection=s", "preserve-properties") or pod2usage(2); my %compressargs = %{compressargset($args{'compress'} || 'default')}; # Can't be done with GetOptions arg, as default still needs to be set @@ -487,11 +487,19 @@ sub syncdataset { } my $oldestsnapescaped = escapeshellparam($oldestsnap); - if (defined $args{'preserve-recordsize'}) { + if (defined $args{'preserve-properties'}) { + my %properties = getlocalzfsvalues($sourcehost,$sourcefs,$sourceisroot); + + foreach my $key (keys %properties) { + my $value = $properties{$key}; + if ($debug) { print "DEBUG: will set $key to $value ...\n"; } + $recvoptions .= " -o $key=$value"; + } + } elsif (defined $args{'preserve-recordsize'}) { my $type = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'type'); if ($type eq "filesystem") { my $recordsize = getzfsvalue($sourcehost,$sourcefs,$sourceisroot,'recordsize'); - $recvoptions .= "-o recordsize=$recordsize" + $recvoptions .= "-o recordsize=$recordsize"; } } @@ -1335,6 +1343,55 @@ sub getzfsvalue { return $wantarray ? ($value, $error) : $value; } +sub getlocalzfsvalues { + my ($rhost,$fs,$isroot) = @_; + + my $fsescaped = escapeshellparam($fs); + + if ($rhost ne '') { + $rhost = "$sshcmd $rhost"; + # double escaping needed + $fsescaped = escapeshellparam($fsescaped); + } + + if ($debug) { print "DEBUG: getting locally set values of properties on $fs...\n"; } + my $mysudocmd; + if ($isroot) { $mysudocmd = ''; } else { $mysudocmd = $sudocmd; } + if ($debug) { print "$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped\n"; } + my ($values, $error, $exit) = capture { + system("$rhost $mysudocmd $zfscmd get all -s local -H $fsescaped"); + }; + + my %properties=(); + + if ($exit != 0) { + warn "WARNING: getlocalzfsvalues failed for $fs: $error"; + if ($exitcode < 1) { $exitcode = 1; } + return %properties; + } + + my @blacklist = ( + "available", "compressratio", "createtxg", "creation", "clones", + "defer_destroy", "encryptionroot", "filesystem_count", "keystatus", "guid", + "logicalreferenced", "logicalused", "mounted", "objsetid", "origin", + "receive_resume_token", "redact_snaps", "referenced", "refcompressratio", "snapshot_count", + "type", "used", "usedbychildren", "usedbydataset", "usedbyrefreservation", + "usedbysnapshots", "userrefs", "snapshots_changed", "volblocksize", "written", + "version", "volsize", "casesensitivity", "normalization", "utf8only" + ); + my %blacklisthash = map {$_ => 1} @blacklist; + + foreach (split(/\n/,$values)) { + my @parts = split(/\t/, $_); + if (exists $blacklisthash{$parts[1]}) { + next; + } + $properties{$parts[1]} = $parts[2]; + } + + return %properties; +} + sub readablebytes { my $bytes = shift; my $disp; @@ -2153,6 +2210,7 @@ Options: --create-bookmark Creates a zfs bookmark for the newest snapshot on the source after replication succeeds (only works with --no-sync-snap) --use-hold Adds a hold to the newest snapshot on the source and target after replication succeeds and removes the hold after the next succesful replication. The hold name incldues the identifier if set. This allows for separate holds in case of multiple targets --preserve-recordsize Preserves the recordsize on initial sends to the target + --preserve-properties Preserves locally set dataset properties similiar to the zfs send -p flag but this one will also work for encrypted datasets in non raw sends --no-rollback Does not rollback snapshots on target (it probably requires a readonly target) --delete-target-snapshots With this argument snapshots which are missing on the source will be destroyed on the target. Use this if you only want to handle snapshots on the source. --exclude=REGEX Exclude specific datasets which match the given regular expression. Can be specified multiple times diff --git a/tests/syncoid/9_preserve_properties/run.sh b/tests/syncoid/9_preserve_properties/run.sh new file mode 100755 index 00000000..497ce9a5 --- /dev/null +++ b/tests/syncoid/9_preserve_properties/run.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# test preserving locally set properties from the src dataset to the target one + +set -x +set -e + +. ../../common/lib.sh + +POOL_IMAGE="/tmp/syncoid-test-9.zpool" +MOUNT_TARGET="/tmp/syncoid-test-9.mount" +POOL_SIZE="1000M" +POOL_NAME="syncoid-test-9" + +truncate -s "${POOL_SIZE}" "${POOL_IMAGE}" + +zpool create -m none -f "${POOL_NAME}" "${POOL_IMAGE}" + +function cleanUp { + zpool export "${POOL_NAME}" +} + +# export pool in any case +trap cleanUp EXIT + +zfs create -o recordsize=16k -o xattr=on -o mountpoint=none -o primarycache=none "${POOL_NAME}"/src +zfs create -V 100M -o volblocksize=8k "${POOL_NAME}"/src/zvol8 +zfs create -V 100M -o volblocksize=16k -o primarycache=all "${POOL_NAME}"/src/zvol16 +zfs create -V 100M -o volblocksize=64k "${POOL_NAME}"/src/zvol64 +zfs create -o recordsize=16k -o primarycache=none "${POOL_NAME}"/src/16 +zfs create -o recordsize=32k -o acltype=posixacl "${POOL_NAME}"/src/32 + +../../../syncoid --preserve-properties --recursive --debug --compress=none "${POOL_NAME}"/src "${POOL_NAME}"/dst + + +if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst)" != "16K" ]; then + exit 1 +fi + +if [ "$(zfs get mountpoint -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then + exit 1 +fi + +if [ "$(zfs get xattr -H -o value -t filesystem "${POOL_NAME}"/dst)" != "on" ]; then + exit 1 +fi + +if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst)" != "none" ]; then + exit 1 +fi + +if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "16K" ]; then + exit 1 +fi + +if [ "$(zfs get primarycache -H -o value -t filesystem "${POOL_NAME}"/dst/16)" != "none" ]; then + exit 1 +fi + +if [ "$(zfs get recordsize -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "32K" ]; then + exit 1 +fi + +if [ "$(zfs get acltype -H -o value -t filesystem "${POOL_NAME}"/dst/32)" != "posix" ]; then + exit 1 +fi From 538416879d45baa981307aeadd3abe34adae3874 Mon Sep 17 00:00:00 2001 From: Christoph Klaffl Date: Tue, 18 Jul 2023 18:09:51 +0200 Subject: [PATCH 2/2] prepare 2.2.0 --- CHANGELIST | 14 ++++++++++++++ VERSION | 2 +- findoid | 2 +- packages/debian/changelog | 18 ++++++++++++++++++ packages/rhel/sanoid.spec | 4 +++- sanoid | 2 +- syncoid | 2 +- 7 files changed, 39 insertions(+), 5 deletions(-) diff --git a/CHANGELIST b/CHANGELIST index 6ddc5652..314e431f 100644 --- a/CHANGELIST +++ b/CHANGELIST @@ -1,3 +1,17 @@ +2.2.0 [overall] documentation updates, small fixes (@azmodude, @deviantintegral, @jimsalterjrs, @alexhaydock, @cbreak-black, @kd8bny, @JavaScriptDude, @veeableful, @rsheasby, @Topslakr, @mavhc, @adam-stamand, @joelishness, @jsoref, @dodexahedron, @phreaker0) + [syncoid] implemented flag for preserving properties without the zfs -p flag (@phreaker0) + [syncoid] implemented target snapshot deletion (@mat813) + [syncoid] support bookmarks which are taken in the same second (@delxg, @phreaker0) + [syncoid] exit with an error if the specified src dataset doesn't exist (@phreaker0) + [syncoid] rollback is now done implicitly instead of explicit (@jimsalterjrs, @phreaker0) + [syncoid] append a rand int to the socket name to prevent collisions with parallel invocations (@Gryd3) + [syncoid] implemented support for ssh_config(5) files (@endreszabo) + [syncoid] snapshot hold/unhold support (@rbike) + [sanoid] handle duplicate key definitions gracefully (@phreaker0) + [syncoid] implemented removal of conflicting snapshots with force-delete option (@phreaker0) + [sanoid] implemented pre pruning script hook (@phreaker0) + [syncoid] implemented direct connection support (bypass ssh) for the actual data transfer (@phreaker0) + 2.1.0 [overall] documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0) [syncoid] do not require user to be specified for syncoid (@aerusso) [syncoid] implemented option for keeping sync snaps (@phreaker0) diff --git a/VERSION b/VERSION index 7ec1d6db..ccbccc3d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 +2.2.0 diff --git a/findoid b/findoid index 98ad581e..0bb5e5f4 100755 --- a/findoid +++ b/findoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.1.0'; +$::VERSION = '2.2.0'; use strict; use warnings; diff --git a/packages/debian/changelog b/packages/debian/changelog index 4cab69ba..00c0c074 100644 --- a/packages/debian/changelog +++ b/packages/debian/changelog @@ -1,3 +1,21 @@ +sanoid (2.2.0) unstable; urgency=medium + + [overall] documentation updates, small fixes (@azmodude, @deviantintegral, @jimsalterjrs, @alexhaydock, @cbreak-black, @kd8bny, @JavaScriptDude, @veeableful, @rsheasby, @Topslakr, @mavhc, @adam-stamand, @joelishness, @jsoref, @dodexahedron, @phreaker0) + [syncoid] implemented flag for preserving properties without the zfs -p flag (@phreaker0) + [syncoid] implemented target snapshot deletion (@mat813) + [syncoid] support bookmarks which are taken in the same second (@delxg, @phreaker0) + [syncoid] exit with an error if the specified src dataset doesn't exist (@phreaker0) + [syncoid] rollback is now done implicitly instead of explicit (@jimsalterjrs, @phreaker0) + [syncoid] append a rand int to the socket name to prevent collisions with parallel invocations (@Gryd3) + [syncoid] implemented support for ssh_config(5) files (@endreszabo) + [syncoid] snapshot hold/unhold support (@rbike) + [sanoid] handle duplicate key definitions gracefully (@phreaker0) + [syncoid] implemented removal of conflicting snapshots with force-delete option (@phreaker0) + [sanoid] implemented pre pruning script hook (@phreaker0) + [syncoid] implemented direct connection support (bypass ssh) for the actual data transfer (@phreaker0) + + -- Jim Salter Tue, 18 Jul 2023 10:04:00 +0200 + sanoid (2.1.0) unstable; urgency=medium [overall] documentation updates, small fixes (@HavardLine, @croadfeldt, @jimsalterjrs, @jim-perkins, @kr4z33, @phreaker0) diff --git a/packages/rhel/sanoid.spec b/packages/rhel/sanoid.spec index 376f58a4..218f52de 100644 --- a/packages/rhel/sanoid.spec +++ b/packages/rhel/sanoid.spec @@ -1,4 +1,4 @@ -%global version 2.1.0 +%global version 2.2.0 %global git_tag v%{version} # Enable with systemctl "enable sanoid.timer" @@ -111,6 +111,8 @@ echo "* * * * * root %{_sbindir}/sanoid --cron" > %{buildroot}%{_docdir}/%{name} %endif %changelog +* Tue Jul 18 2023 Christoph Klaffl - 2.2.0 +- Bump to 2.2.0 * Tue Nov 24 2020 Christoph Klaffl - 2.1.0 - Bump to 2.1.0 * Wed Oct 02 2019 Christoph Klaffl - 2.0.3 diff --git a/sanoid b/sanoid index 12a184ab..5150f3b7 100755 --- a/sanoid +++ b/sanoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.1.0'; +$::VERSION = '2.2.0'; my $MINIMUM_DEFAULTS_VERSION = 2; use strict; diff --git a/syncoid b/syncoid index 6cde9f99..5ed00f07 100755 --- a/syncoid +++ b/syncoid @@ -4,7 +4,7 @@ # from http://www.gnu.org/licenses/gpl-3.0.html on 2014-11-17. A copy should also be available in this # project's Git repository at https://github.com/jimsalterjrs/sanoid/blob/master/LICENSE. -$::VERSION = '2.1.0'; +$::VERSION = '2.2.0'; use strict; use warnings;