diff --git a/application-src/Makefile.am b/application-src/Makefile.am index 4c1d34343a..82d7d29ef5 100644 --- a/application-src/Makefile.am +++ b/application-src/Makefile.am @@ -20,8 +20,17 @@ AM_LDFLAGS = $(AMANDA_STATIC_LDFLAGS) $(AS_NEEDED_FLAGS) applicationexec_SCRIPTS_SHELL = script-fail applicationexec_SCRIPTS_PERL = script-email \ + am389bak \ + amgrowingfile \ + amgrowingzip \ + amlibvirtfsfreeze \ amlog-script \ + amlvmsnapshot \ + amooraw \ + amopaquetree \ ampgsql \ + amsvnmakehotcopy \ + amzfs-holdsend \ amzfs-sendrecv \ amzfs-snapshot \ amrandom \ diff --git a/application-src/am389bak.pl b/application-src/am389bak.pl new file mode 100644 index 0000000000..d8f409c746 --- /dev/null +++ b/application-src/am389bak.pl @@ -0,0 +1,95 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Script::Am389Bak; + +use base 'Amanda::Script::Abstract'; + +use File::Spec; +use File::Path qw(make_path remove_tree); +use IO::File; + +sub new { + my ( $class, $execute_where, $refopthash ) = @_; + my $self = $class->SUPER::new($execute_where, $refopthash); + + return $self; +} + +sub check_properties { + my ( $self ) = @_; + + $self->{'db2bakexecutable'} = $self->{'options'}->{'db2bakexecutable'}; + + $self->{'instance'} = $self->{'options'}->{'instance'}; + if ( !defined $self->{'instance'} ) { + die Amanda::Script::InvocationError->transitionalError( + item => 'property', value => 'instance', problem => 'missing'); + } +} + +sub declare_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'db2bakexecutable=s', 'instance=s' ); + + $class->store_option($refopthash, 'db2bakexecutable', 'db2bak'); +} + +sub command_pre_dle_estimate { + my ( $self ) = @_; + + my $repo = $self->{'instance'}; + my $dst = $self->{'options'}->{'device'}; + + make_path($dst); + remove_tree($dst, {keep_root => 1}); + + my $rslt = system {$self->{'db2bakexecutable'}} ( + 'db2bak', $dst, '-qZ', $repo ); + + unless ( 0 == $rslt ) { + die Amanda::Script::CalledProcessError->transitionalError( + cmd => 'db2bak', returncode => $rslt); + } +} + +package main; + +Amanda::Script::Am389Bak->run(); diff --git a/application-src/amgrowingfile.pl b/application-src/amgrowingfile.pl new file mode 100644 index 0000000000..d0856504fc --- /dev/null +++ b/application-src/amgrowingfile.pl @@ -0,0 +1,204 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Application::AmGrowingFile; + +use base 'Amanda::Application::Abstract'; + +use Data::Dumper; +use File::Spec; +use File::Path qw(make_path); + +sub supports_host { my ( $class ) = @_; return 1; } +sub supports_disk { my ( $class ) = @_; return 1; } +sub supports_index_line { my ( $class ) = @_; return 1; } +sub supports_message_line { my ( $class ) = @_; return 1; } +sub supports_record { my ( $class ) = @_; return 1; } +sub supports_client_estimate { my ( $class ) = @_; return 1; } +sub supports_multi_estimate { my ( $class ) = @_; return 1; } + +sub max_level { my ( $class ) = @_; return 'DEFAULT'; } + +sub new { + my ( $class, $refopthash ) = @_; + my $self = $class->SUPER::new($refopthash); + $self->{'localstate'} = + $self->read_local_state(['level=i', 'byteoffset=s', 'bytes=s']); + return $self; +} + +sub declare_restore_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_restore_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'filename=s' ); +} + +sub inner_estimate { + my ( $self, $level ) = @_; + my $fn = $self->target(); + my $sz = $self->int2big(-s $fn); + return $sz if 0 == $level; + + my $mxl = $self->{'localstate'}->{'maxlevel'}; + + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level) if $level > $mxl; + + my $lowerstate = $self->{'localstate'}->{$level - 1}; + my $loweroffset = Math::BigInt->new($lowerstate->{'byteoffset'}); + my $lowersize = Math::BigInt->new($lowerstate->{'bytes'}); + return $sz->bsub($loweroffset)->bsub($lowersize); +} + +sub inner_backup { + # XXX assert level==0 if no --record + my ( $self, $fdout ) = @_; + my $fn = $self->target(); + my $level = $self->{'options'}->{'level'}; + my $fdin = POSIX::open($fn, &POSIX::O_RDONLY); + + if (!defined $fdin) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, errno => $!); + } + + my $start; + if ( 0 == $level ) { + $start = Math::BigInt->bzero(); + } else { + my $lowerstate = $self->{'localstate'}->{$level - 1}; + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level) unless defined $lowerstate; + my $loweroffset = Math::BigInt->new($lowerstate->{'byteoffset'}); + my $lowersize = Math::BigInt->new($lowerstate->{'bytes'}); + $start = $loweroffset->copy()->badd($lowersize); + my $istart = $self->big2int($start); + + die Amanda::Application::RetryDumpError->transitionalError( + delay => 0, level => 0, problem => 'growing file shrank') + if $istart > (POSIX::fstat($fdin))[7]; + + # Currently science fiction: optionally include in the state a digest + # of the file's past contents, to force a retry at level 0 in case of + # mismatch even if size does not catch it. Of course that would turn + # every incremental dump into a level-0 amount of I/O (well, I, anyway). + + POSIX::lseek($fdin, $istart, &POSIX::SEEK_SET); + + # sendbackup: HEADER, documented in the Application API/Operations wiki + # page, wasn't ever implemented, according to Jean-Louis. An option that + # becomes available in 3.3.8 is 'sendbackup: state' and retrieved with + # --recover-dump-state-file. The state file is kept on the server, not + # clear how /it/ is backed up. May not be available if recovering only + # from the tape. + # + # print {$self->{mesgout}} "sendbackup: HEADER startoffset=$start\n"; + # + # Without doing this, can just /assume/ that the file is currently as + # the lower-level restore left it, and append to the end. Not ideal, + # but other incremental strategies also perform restores without + # rigorous verification of the state they are restoring onto, so when + # in Rome.... + } + my $size = $self->shovel($fdin, $fdout); + + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'close', errno => $!) + unless defined POSIX::close($fdin); + + $self->emit_index_entry('/'); + + if ( $self->{'options'}->{'record'} ) { + $self->update_local_state($self->{'localstate'}, $level, { + 'byteoffset' => $start->bstr(), 'bytes' => $size->bstr() }); + } + + return $size; +} + +sub inner_restore { + my $self = shift; + my $fdin = shift; + my $dsf = shift; + my $level = $self->{'options'}->{'level'}; + + if ( 1 != scalar(@_) or $_[0] ne '.' ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'restore targets', + problem => 'Only one (.) supported'); + } + + my $fn = $self->target('amgrowingfile-restored'); + + my ( $volume, $directories, $file ) = File::Spec->splitpath($fn); + make_path(File::Spec->catpath($volume, $directories, '')); + + # Where to begin writing if applying a level > 0 (incremental) dump? + # In a cautious world, save the starting offset at dump time, and verify + # at restore time that that's where the file ends. That could require either + # saving the offset in the dump stream somehow, or using the server-side + # state file that appears in 3.3.8 (which still might not be available in a + # restoration from only the tape). Short of that, just blindly open in + # append mode and write the increment. After all, does an incremental tar + # restore actually verify the current directory tree is exactly as the prior + # dump level left it? No, it just relies on restoration being done carefully + # and in sequence. + my $oflags = &POSIX::O_RDWR; + if ( $level > 0 ) { + $oflags |= &POSIX::O_APPEND; + } else { + $oflags |= ( &POSIX::O_CREAT | &POSIX::O_TRUNC ); + } + + my $fdout = POSIX::open($fn, $oflags, 0600); + if (!defined $fdout) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, errno => $!); + } + + $self->shovel($fdin, $fdout); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'close', errno => $!) + unless defined POSIX::close($fdout); + POSIX::close($fdin); +} + +package main; + +Amanda::Application::AmGrowingFile->run(); diff --git a/application-src/amgrowingzip.pl b/application-src/amgrowingzip.pl new file mode 100644 index 0000000000..913a3a046d --- /dev/null +++ b/application-src/amgrowingzip.pl @@ -0,0 +1,280 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Application::AmGrowingZip; + +use base 'Amanda::Application::Abstract'; + +use Data::Dumper; +use Fcntl qw(:flock); +use File::Spec; +use File::Path qw(make_path); +use IO::File; + +my $usable; +eval { + require Archive::Zip; + $usable = 1; +} or do { + $usable = 0; +}; + +sub supports_host { my ( $class ) = @_; return 1; } +sub supports_disk { my ( $class ) = @_; return 1; } +sub supports_index_line { my ( $class ) = @_; return 1; } +sub supports_message_line { my ( $class ) = @_; return 1; } +sub supports_record { my ( $class ) = @_; return 1; } +sub supports_client_estimate { my ( $class ) = @_; return 1; } +sub supports_multi_estimate { my ( $class ) = @_; return 1; } + +sub max_level { my ( $class ) = @_; return 'DEFAULT'; } + +sub new { + my ( $class, $refopthash ) = @_; + my $self = $class->SUPER::new($refopthash); + $self->{'localstate'} = + $self->read_local_state( + ['level=i', 'length=s', 'centraldiroffset=s']); + return $self; +} + +sub declare_common_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'flock=s' ); + $refopthash->{'flock'} = $class->boolean_property_setter($refopthash); +} + +sub declare_restore_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_restore_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'filename=s' ); +} + +sub command_selfcheck { + my ( $self ) = @_; + $self->check($usable, 'Archive::Zip is not installed'); + $self->SUPER::command_selfcheck(); +} + +sub inner_estimate { + my ( $self, $level ) = @_; + my $fn = $self->target(); + my $sz = $self->int2big(-s $fn); + return $sz if 0 == $level; + + my $mxl = $self->{'localstate'}->{'maxlevel'}; + + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level) if $level > $mxl; + + my $lowerstate = $self->{'localstate'}->{$level - 1}; + my $lowerlength = Math::BigInt->new($lowerstate->{'length'}); + my $lowercdo = Math::BigInt->new($lowerstate->{'centraldiroffset'}); + return Math::BigInt->bzero() if 0 == $sz->bcmp($lowerlength); + return $sz->bsub($lowercdo); +} + +sub inner_backup { + # XXX assert level==0 if no --record + my ( $self, $fdout ) = @_; + my $fn = $self->target(); + my $level = $self->{'options'}->{'level'}; + my $flock = $self->{'options'}->{'flock'}; + my $fdin = POSIX::open($fn, &POSIX::O_RDONLY); + + if (!defined $fdin) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, errno => $!); + } + + my $ioh = IO::File->new(); + $ioh->fdopen($fdin, 'r'); + my $az = Archive::Zip->new(); + + if ( $flock and not flock($ioh, LOCK_SH) ) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'lock', errno => $!); + } + $az->readFromFileHandle($ioh); + my $cdo =$self->int2big($az->centralDirectoryOffsetWRTStartingDiskNumber()); + + my $start; + my $currentlength = $self->int2big(-s $ioh); + if ( 0 == $level ) { + $start = Math::BigInt->bzero(); + } else { + # ENHANCEMENT? could save a prior digest, and verify here + my $lowerstate = $self->{'localstate'}->{$level - 1}; + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level) unless defined $lowerstate; + my $lowerlength = Math::BigInt->new($lowerstate->{'length'}); + my $lowercdo = Math::BigInt->new($lowerstate->{'centraldiroffset'}); + if ( 0 == $currentlength->bcmp($lowerlength) ) { + # Length is unchanged -> nothing has changed (the zip file is + # assumed never to change except by appending). + $start = $currentlength; # In other words, dump nothing. + } + elsif ( 0 > $currentlength->bcmp($lowerlength) ) { + die Amanda::Application::RetryDumpError->transitionalError( + delay => 0, level => 0, problem => 'growing file shrank'); + } + else { + $start = $lowercdo; # Dump from lowercdo to current length. + } + + # sendbackup: HEADER, documented in the Application API/Operations wiki + # page, wasn't ever implemented, according to Jean-Louis. An option that + # becomes available in 3.3.8 is 'sendbackup: state' and retrieved with + # --recover-dump-state-file. The state file is kept on the server, not + # clear how /it/ is backed up. May not be available if recovering only + # from the tape. + # + # print {$self->{mesgout}} "sendbackup: HEADER startoffset=$start\n"; + # + # Without doing this, can just /assume/ that the file is currently as + # the lower-level restore left it, and append to the end. Not ideal, + # but other incremental strategies also perform restores without + # rigorous verification of the state they are restoring onto, so when + # in Rome.... + } + + my $istart = $self->big2int($start); + + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'seek', errno => $!) + unless defined POSIX::lseek($fdin, $istart, &POSIX::SEEK_SET); + + my $size = $self->shovel($fdin, $fdout); + if ( $flock and not flock($ioh, LOCK_UN) ) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'unlock', errno => $!); + } + + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'close', errno => $!) + unless defined POSIX::close($fdin); + + $self->emit_index_entry('/'); + + if ( $self->{'options'}->{'record'} ) { + $self->update_local_state($self->{'localstate'}, $level, { + 'length' => $currentlength->bstr(), + 'centraldiroffset' => $cdo->bstr() }); + } + + return $size; +} + +sub inner_restore { + my $self = shift; + my $fdin = shift; + my $dsf = shift; + my $level = $self->{'options'}->{'level'}; + # + # There is no point honoring flock during restore. It would be madness + # to try to restore a sequence of increments while writes from other + # sources could intervene. Restoration must always be done into a secluded + # directory/path, only moving the fully restored file back to where other + # processes may write it. + + if ( 1 != scalar(@_) or $_[0] ne '.' ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'restore targets', + problem => 'Only one (.) supported'); + } + + my $fn = $self->target('amgrowingzip-restored'); + + my ( $volume, $directories, $file ) = File::Spec->splitpath($fn); + make_path(File::Spec->catpath($volume, $directories, '')); + + # Where to begin writing if applying a level > 0 (incremental) dump? In a + # cautious world, save the starting offset at dump time, and verify at + # restore time that that's where the central directory starts. That could + # require either saving the offset in the dump stream somehow, or using the + # server-side state file that appears in 3.3.8 (which still might not be + # available in a restoration from only the tape). Short of that, just + # blindly open in rdwr mode, seek to its central directory offset, and write + # the increment. After all, does an incremental tar restore actually verify + # the current directory tree is exactly as the prior dump level left it? No, + # it just relies on restoration being done carefully and in sequence. + my $oflags = &POSIX::O_RDWR; + if ( $level == 0 ) { + $oflags |= ( &POSIX::O_CREAT | &POSIX::O_TRUNC ); + } + + my $fdout = POSIX::open($fn, $oflags, 0600); + if (!defined $fdout) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, errno => $!); + } + + my $ioh = IO::File->new(); # don't let out of scope before shovel() + if ( $level > 0 ) { + $ioh->fdopen($fdout, 'r'); + my $az = Archive::Zip->new(); + $az->readFromFileHandle($ioh); + my $cdo = + $self->int2big($az->centralDirectoryOffsetWRTStartingDiskNumber()); + my $ioff = $self->big2int($cdo); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'seek', errno => $!) + unless defined POSIX::lseek($fdout, $ioff, &POSIX::SEEK_SET); + # We are now positioned at the beginning of the "central" directory + # found at the end of the zip file, and the file is open for RDWR + # without TRUNC. If the increment was dumped when more content had been + # appended to the zip, there will be a stream to write here that is + # strictly longer than the old directory, so no need to truncate. If + # the increment was dumped when nothing had changed, there is a zero + # length stream to shovel here, leaving the file untruncated and intact. + # (Amanda seems to discard increments that dump zero bytes, anyway, so + # this case normally should not even arise.) + } + + $self->shovel($fdin, $fdout); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'close', errno => $!) + unless defined POSIX::close($fdout); + POSIX::close($fdin); +} + +package main; + +Amanda::Application::AmGrowingZip->run(); diff --git a/application-src/amlibvirtfsfreeze.pl b/application-src/amlibvirtfsfreeze.pl new file mode 100644 index 0000000000..b637f5ef18 --- /dev/null +++ b/application-src/amlibvirtfsfreeze.pl @@ -0,0 +1,184 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017-2020 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Script::AmLibvirtFSFreeze; + +use base 'Amanda::Script::Abstract'; + +use File::Spec; +use IO::File; + +sub new { + my ( $class, $execute_where, $refopthash ) = @_; + my $self = $class->SUPER::new($execute_where, $refopthash); + + return $self; +} + +sub check_properties { + my ( $self ) = @_; + + $self->{'virshexecutable'} = $self->{'options'}->{'virshexecutable'}; + + $self->{'freezeorthaw'} = $self->{'options'}->{'freezeorthaw'}; + if ( !defined $self->{'freezeorthaw'} + or $self->{'freezeorthaw'} !~ /^(?:freeze|thaw|trim)$/ ) { + die Amanda::Script::InvocationError->transitionalError( + item => 'property', value => 'freezeorthaw', + problem => 'must be freeze or thaw (or trim)'); + } + + $self->{'domain'} = $self->{'options'}->{'domain'}; + if ( !defined $self->{'domain'} ) { + die Amanda::Script::InvocationError->transitionalError( + item => 'property', value => 'domain', problem => 'missing'); + } + + $self->{'mountpoint'} = $self->{'options'}->{'mountpoint'}; + + if ( 'trim' eq $self->{'freezeorthaw'} ) { + my @mountpoints = @{$self->{'mountpoint'}}; + die Amanda::Script::InvocationError->transitionalError( + item => 'property', value => 'mountpoint', + problem => 'trim cannot accept more than one') + if 1 < scalar(@mountpoints); + } +} + +sub declare_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_options($refopthash, $refoptspecs); + push @$refoptspecs, ( + 'virshexecutable=s', + 'freezeorthaw=s', + 'domain=s', + 'mountpoint=s@' + ); + # properties that have defaults and are not mandatory to receive with the + # request can be initialized here as an alternative to checking for !defined + # and applying the defaults in check_properties(). + $class->store_option($refopthash, 'virshexecutable', 'virsh'); + $class->store_option($refopthash, 'mountpoint', []); +} + +sub command_pre_dle_estimate { + my ( $self ) = @_; + + my $domain = $self->{'domain'}; + my @args; + + if ( 'freeze' eq $self->{'freezeorthaw'} ) { + my @mountpoints = @{$self->{'mountpoint'}}; + @args = ( 'virsh', 'domfsfreeze', $domain, @mountpoints ); + } elsif ( 'trim' eq $self->{'freezeorthaw'} ) { + my @mountpoints = @{$self->{'mountpoint'}}; + unshift @mountpoints, '--mountpoint' if 0 < scalar(@mountpoints); + @args = ( 'virsh', 'domfstrim', $domain, @mountpoints ); + } else { + @args = ( 'virsh', 'domfsthaw', $domain ); + } + + unless ( $self->domain_is_running() ) { + warn Amanda::Script::Message->transitionalGood( + message=>"$domain not running, $self->{'freezeorthaw'} skipped\n"); + return; + } + + my $rslt = system {$self->{'virshexecutable'}} (@args); + + unless ( 0 == $rslt ) { + die Amanda::Script::CalledProcessError->transitionalError( + cmd => \@args, returncode => $rslt); + } +} + +# In an ideal world, just run at PRE-DLE-ESTIMATE to make one snapshot, +# estimate from it, dump from it, then free it in POST-DLE-BACKUP. But on +# a host with little free space for a snapshot and possibly a long planning wait +# between the estimate and the dump, it may be better to support being called +# at PRE-DLE-BACKUP also, in case a second snapshot is to be made then. + +sub command_pre_dle_backup { + my ( $self ) = @_; + $self->command_pre_dle_estimate(); +} + +sub domain_is_running { + # Perl's 'system' is adequate for most of the work of this script, where + # only a command's exit status is needed, but domain_is_running needs to + # execute a command, capture its output, and compare to what's expected, and + # has to do that while possibly invoking a wrapper program with an argv[0] + # reflecting the real target, which is something Perl's exec and system can + # do but other 'simple' approaches like open/open2/open3 can't. Hence this + # simple task quickly gets grotty. Cribbed from the similar get_data_percent + # in amlvmsnapshot. + # + # Having to write subprocess manipulations at this low level is kind of + # a drag on easy script development, and think it would be preferable for + # Amanda to provide common facilities for them, or bless a widely-known + # existing module like IPC::Run and consider it an expected dependency + # so it could be used without hesitation. + my ( $self ) = @_; + my @args = ( 'virsh', 'domstate', $self->{'domain'} ); + + unless ( defined (my $pid = open my $pipefh, '-|') ) { + die Amanda::Script::EnvironmentError->transitionalError( + item => 'spawning', value => 'virsh domstate', errno => $!); + } else { + unless ( $pid ) { + unless ( exec {$self->{'virshexecutable'}} @args ) { + print STDERR $!."\n"; + exit 1; + } + } + my $got = <$pipefh>; + unless ( close $pipefh ) { + die Amanda::Script::CalledProcessError->transitionalError( + cmd => @args, returncode => $?) + if 0 == $!; + die Amanda::Script::EnvironmentError->transitionalError( + item => 'spawning', value => 'virsh domstate', errno => $!); + } + return "running\n" eq $got; + } +} + +package main; + +Amanda::Script::AmLibvirtFSFreeze->run(); diff --git a/application-src/amlvmsnapshot.pl b/application-src/amlvmsnapshot.pl new file mode 100644 index 0000000000..c465c809a2 --- /dev/null +++ b/application-src/amlvmsnapshot.pl @@ -0,0 +1,241 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Script::AmLvmSnapshot; + +use base 'Amanda::Script::Abstract'; + +use File::Spec; +use File::Path qw(make_path remove_tree); +use IO::File; + +sub new { + my ( $class, $execute_where, $refopthash ) = @_; + my $self = $class->SUPER::new($execute_where, $refopthash); + + return $self; +} + +sub check_properties { + my ( $self ) = @_; + + $self->{'lvmexecutable'} = $self->{'options'}->{'lvmexecutable'}; + $self->{'mountexecutable'} = $self->{'options'}->{'mountexecutable'}; + $self->{'umountexecutable'} = $self->{'options'}->{'umountexecutable'}; + + for my $prop ( 'volumegroup', 'logicalvolume', 'snapshotname', 'extents' ) { + $self->{$prop} = $self->{'options'}->{$prop}; + die Amanda::Script::InvocationError->transitionalError( + item => 'property', value => $prop, problem => 'missing') + unless defined $self->{$prop}; + } + + $self->{'mountopts'} = $self->{'options'}->{'mountopts'}; +} + +sub declare_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_options($refopthash, $refoptspecs); + push @$refoptspecs, ( + 'lvmexecutable=s', + 'mountexecutable=s', + 'umountexecutable=s', + 'volumegroup=s', + 'logicalvolume=s', + 'snapshotname=s', + 'extents=s', + 'mountopts=s@', + 'unmountswhenfilled=s' + ); + $class->store_option($refopthash, + 'unmountswhenfilled', $class->boolean_property_setter($refopthash)); + # properties that have defaults and are not mandatory to receive with the + # request can be initialized here as an alternative to checking for !defined + # and applying the defaults in check_properties(). + $class->store_option($refopthash, 'lvmexecutable', 'lvm'); + $class->store_option($refopthash, 'mountexecutable', 'mount'); + $class->store_option($refopthash, 'umountexecutable', 'umount'); + $class->store_option($refopthash, 'mountopts', []); +} + +sub command_pre_dle_estimate { + my ( $self ) = @_; + + my $vg = $self->{'volumegroup'}; + my $lv = $self->{'logicalvolume'}; + my $sn = $self->{'snapshotname'}; + my $extents = $self->{'extents'}; + + my $dst = $self->{'options'}->{'device'}; + + make_path($dst); + + # NOTE: lvm mumbles a couple lines on stdout, where they'll be mistaken + # for script output, eventually confusing the Amanda reader of the stream, + # which will think the sendsize timed out. Simply redirecting stdout to + # stderr for the lvm invocation fixes that. For now, instead of being done + # here in Perl, it's being done in the setuid wrapper executable supplied + # as 'lvmexecutable'. For clarity and generality it would be nicer here. + # Even nicer would be to handle it all transparently in the superclass. + + my $rslt = system {$self->{'lvmexecutable'}} ( + 'lvm', 'lvcreate', + '--snapshot', $vg.'/'.$lv, + '--extents', $extents, + '--name', $sn, + '--permission', 'r' + ); + + die Amanda::Script::CalledProcessError->transitionalError( + cmd => 'lvcreate', returncode => $rslt) + unless 0 == $rslt; + + $rslt = system {$self->{'mountexecutable'}} ( + 'mount', + '-o', join(',', ('ro', @{$self->{'mountopts'}})), + '/dev/disk/by-id/dm-name-'.$vg.'-'.$sn, + $dst + ); + + die Amanda::Script::CalledProcessError->transitionalError( + cmd => 'mount', returncode => $rslt) + unless 0 == $rslt; +} + +sub command_post_dle_backup { + my ( $self ) = @_; + + my $vg = $self->{'volumegroup'}; + my $sn = $self->{'snapshotname'}; + + my $dst = $self->{'options'}->{'device'}; + + my $rslt = system {$self->{'umountexecutable'}} ( + 'umount', $dst + ); + + my $pctused = $self->get_data_percent(); + + if ( 0 == $rslt ) { # The umount succeeded. + if ( 100 == $pctused and $self->{'options'}->{'unmountswhenfilled'} ) { + # If we know the OS auto-unmounts snapshots that fill, and our + # explicit unmount did not fail, then we know the snapshot had not + # filled by that point, even if get_data_percent() returned 100 + # immediately after. Make $pctused just slightly < 100 then, to + # produce a warning later rather than a hard error. + $pctused -= ldexp(POSIX::DBL_EPSILON, 6); # 2^6 < 100 < 2^7 + } + } + else { # The umount failed for some reason. + die Amanda::Script::CalledProcessError->transitionalError( + cmd => 'umount', returncode => $rslt) + unless 100 == $pctused; + # If the snapshot filled, do not complain here because of the + # umount failing. Just carry on, and failure because $pctused == 100 + # will be reported explicitly later. + } + + $rslt = system {$self->{'lvmexecutable'}} ( + 'lvm', + 'lvremove', '--force', $vg.'/'.$sn + ); + + die Amanda::Script::CalledProcessError->transitionalError( + cmd => 'lvremove', returncode => $rslt) + unless 0 == $rslt; + + die Amanda::Script::EnvironmentError->transitionalError( + item => 'snapshot', value => $sn, + problem => 'reached max allocation during backup') + unless $pctused < 100; + print STDERR "warning: snapshot $sn reached $pctused % allocation " . + "during backup\n" + unless $pctused < 90; +} + +# In an ideal world, just run at PRE-DLE-ESTIMATE to make one snapshot, +# estimate from it, dump from it, then free it in POST-DLE-BACKUP. But on +# a host with little free space for a snapshot and possibly a long planning wait +# between the estimate and the dump, it may be better to support being called +# at POST-DLE-ESTIMATE (to free one snapshot, same as POST-DLE-BACKUP) and +# at PRE-DLE-BACKUP (to make another, same as PRE-DLE-ESTIMATE). + +sub command_post_dle_estimate { + my ( $self ) = @_; + $self->command_post_dle_backup(); +} + +sub command_pre_dle_backup { + my ( $self ) = @_; + $self->command_pre_dle_estimate(); +} + +sub get_data_percent { + my ( $self ) = @_; + + unless ( defined (my $pid = open my $pipefh, '-|') ) { + die Amanda::Script::EnvironmentError->transitionalError( + item => 'spawning', value => 'lvm lvs', errno => $!); + } else { + unless ( $pid ) { + exec {$self->{'lvmexecutable'}} ( + 'lvm', + 'lvs', '--noheadings', '--options', 'data_percent', + $self->{'volumegroup'} . '/' . $self->{'snapshotname'} + ); + } + my $got = <$pipefh>; + unless ( close $pipefh ) { + die Amanda::Script::CalledProcessError->transitionalError( + cmd => 'lvm lvs', returncode => $?) + if 0 == $!; + die Amanda::Script::EnvironmentError->transitionalError( + item => 'spawning', value => 'lvm lvs', errno => $!); + } + die Amanda::Script::EnvironmentError->transitionalError( + item => 'data_percent', value => $got, + problem => 'expected numeric') + unless $got =~ /^\s*\d+(?:\.\d+)?\s*$/; + return 0 + $got; + } +} + +package main; + +Amanda::Script::AmLvmSnapshot->run(); diff --git a/application-src/amooraw.pl b/application-src/amooraw.pl new file mode 100644 index 0000000000..270b8a3c38 --- /dev/null +++ b/application-src/amooraw.pl @@ -0,0 +1,116 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Application::Amooraw; + +use base 'Amanda::Application::Abstract'; + +use Data::Dumper; +use File::Spec; +use File::Path qw(make_path); + +sub supports_message_line { my ( $class ) = @_; return 1; } +sub supports_index_line { my ( $class ) = @_; return 1; } +sub supports_client_estimate { my ( $class ) = @_; return 1; } + +sub declare_restore_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_restore_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'filename=s' ); +} + +sub inner_estimate { + my ( $self, $level ) = @_; + my $fn = $self->target(); + return $self->int2big(-s $fn); +} + +sub inner_backup { + my ( $self, $fdout ) = @_; + my $fn = $self->target(); + my $fdin = POSIX::open($fn, &POSIX::O_RDONLY); + + if (!defined $fdin) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, errno => $!); + } + + my $size = $self->shovel($fdin, $fdout); + + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'close', errno => $!) + unless defined POSIX::close($fdin); + + $self->emit_index_entry('/'); + + return $size; +} + +sub inner_restore { + my $self = shift; + my $fdin = shift; + my $dsf = shift; + + if ( 1 != scalar(@_) or $_[0] ne '.' ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'restore targets', + problem => 'Only one (.) supported'); + } + + my $fn = $self->target('amooraw-restored'); + + my ( $volume, $directories, $file ) = File::Spec->splitpath($fn); + make_path(File::Spec->catpath($volume, $directories, '')); + + my $fdout = POSIX::open($fn, &POSIX::O_CREAT | &POSIX::O_RDWR, 0600); + if (!defined $fdout) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, errno => $!); + } + + $self->shovel($fdin, $fdout); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $fn, problem => 'close', errno => $!) + unless defined POSIX::close($fdout); + POSIX::close($fdin); +} + +package main; + +Amanda::Application::Amooraw->run(); diff --git a/application-src/amopaquetree.pl b/application-src/amopaquetree.pl new file mode 100644 index 0000000000..a802c39cc1 --- /dev/null +++ b/application-src/amopaquetree.pl @@ -0,0 +1,347 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Application::AmOpaqueTree::DirWrap; +# +# A tiny class that wraps a directory name as an object with a dirname() +# method that returns it ... so it can be treated the same way as a result +# from File::Temp->newdir(), which behaves that way. +# + +sub new { + my ( $class, $dirname ) = @_; + my $self = { 'dn' => $dirname }; + return bless($self, $class); +} + +sub dirname { + my ( $self ) = @_; + return $self->{'dn'}; +} + +package Amanda::Application::AmOpaqueTree; +# +# The main attraction. +# + +use base 'Amanda::Application::Abstract'; + +use Data::Dumper; +use Fcntl; +use File::Spec; +use File::Temp; +use File::Path qw(make_path remove_tree); +use IO::File; +use IPC::Open3; + +sub supports_host { my ( $class ) = @_; return 1; } +sub supports_disk { my ( $class ) = @_; return 1; } +sub supports_index_line { my ( $class ) = @_; return 1; } +sub supports_message_line { my ( $class ) = @_; return 1; } +sub supports_record { my ( $class ) = @_; return 1; } +sub supports_client_estimate { my ( $class ) = @_; return 1; } +sub supports_multi_estimate { my ( $class ) = @_; return 1; } + +sub max_level { my ( $class ) = @_; return 'DEFAULT'; } + +sub rsync_is_unusable { + my ( $self ) = @_; + my ( $wtr, $rdr ); + my $pid = eval { + open3($wtr, $rdr, undef, $self->{'rsyncexecutable'}, '--version'); + }; + return $@ if $@; + close $wtr; + my $output = do { local $/; <$rdr> }; + close $rdr; + waitpid $pid, 0; + return $output if $?; + unless ( $output =~ qr/(?:^\s|,\s)hardlinks(?:,\s|$)/m ) { + return $self->{'rsyncexecutable'} . ' lacks hardlink support.'; + } + unless ( $output =~ qr/(?:^\s|,\s)batchfiles(?:,\s|$)/m ) { + return $self->{'rsyncexecutable'} . ' lacks batchfile support.'; + } + return 0; # hooray, it isn't unusable. +} + +sub new { + my ( $class, $refopthash ) = @_; + my $self = $class->SUPER::new($refopthash); + + $self->{'rsyncexecutable'} = $self->{'options'}->{'rsyncexecutable'}; + + $self->{'localstatedir'} = $self->{'options'}->{'localstatedir'}; + + $self->{'rsyncstatesdir'} = $self->{'options'}->{'rsyncstatesdir'}; + # This default is computed here, based on the final value of localstatedir, + # so it can't just be stored in declare_common_options as a fixed default. + if ( !defined $self->{'rsyncstatesdir'} ) { + my ( $dirpart, $filepart ) = $self->local_state_path(); + $self->{'rsyncstatesdir'} = + File::Spec->catdir($dirpart, 'rsyncstates'); + } + + $self->{'rsynctempbatchdir'} = $self->{'options'}->{'rsynctempbatchdir'}; + + $self->{'localstate'} = + $self->read_local_state(['level=i', 'rsyncstate=s']); + + return $self; +} + +sub declare_common_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, + ( 'rsyncexecutable=s', 'rsyncstatesdir=s', 'rsynctempbatchdir=s', + 'localstatedir=s' ); + $class->store_option($refopthash, 'rsyncexecutable', 'rsync'); +} + +sub local_state_path { + my ( $self ) = @_; + if ( defined $self->{'localstatedir'} ) { + return $self->build_state_path($self->{'localstatedir'}); + } + return $self->SUPER::local_state_path(); +} + +sub generate_rsync_batch { + my ( $self, $basedOn, $yielding ) = @_; + my $batch = File::Temp->new( + DIR => $self->{'rsynctempbatchdir'}, + EXLOCK => 0 + ); + + my $rslt = system {$self->{'rsyncexecutable'}} ( + 'rsync', + '-rlpt', '--checksum', '--delete-during', '--compress', '--sparse', + '--only-write-batch', $batch->filename, + File::Spec->catfile($yielding, ''), + File::Spec->catfile($basedOn, '') + ); + die Amanda::Application::CalledProcessError->transitionalError( + cmd => 'rsync', returncode => $rslt) + unless 0 == $rslt; + + unlink($batch->filename . '.sh'); + + return $batch; +} + +sub empty_rsync_state_dir { + my ( $self, $transient ) = @_; + if ( ! -d $self->{'rsyncstatesdir'} ) { + make_path($self->{'rsyncstatesdir'}); + } + return File::Temp->newdir( + DIR => $self->{'rsyncstatesdir'}, CLEANUP => $transient); +} + +sub best_link_dest { + my ( $self, $level ) = @_; + + my $prior = ( $level > 0 ) ? $level : $self->{'localstate'}->{'maxlevel'}; + $prior -= 1; + my $linkdest; + if ( exists $self->{'localstate'}->{$prior} ) { + $linkdest = $self->{'localstate'}->{$prior}->{'rsyncstate'}; + } + if ( defined $linkdest ) { + $linkdest = Amanda::Application::AmOpaqueTree::DirWrap->new($linkdest); + } else { + $linkdest = $self->empty_rsync_state_dir(1); + } + return $linkdest; +} + +sub capture_rsync_state { + my ( $self, $srcdir, $dstdir, $linkdestdir ) = @_; + + my $rslt = system {$self->{'rsyncexecutable'}} ( + 'rsync', + '-rlp', '--whole-file', '--checksum', '--copy-dirlinks', '--sparse', + '--link-dest', $linkdestdir, + File::Spec->catfile($srcdir, ''), + File::Spec->catfile($dstdir, '') + ); + die Amanda::Application::CalledProcessError->transitionalError( + cmd => 'rsync', returncode => $rslt) + unless 0 == $rslt; +} + +sub rsync_ref_for_level { + my ( $self, $level ) = @_; + my $ref; + if ( 0 == $level ) { + $ref = $self->empty_rsync_state_dir(1); + } else { + my $lowerstate = $self->{'localstate'}->{$level - 1}; + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level) unless defined $lowerstate; + $ref = $lowerstate->{'rsyncstate'}; + $ref = Amanda::Application::AmOpaqueTree::DirWrap->new($ref); + } + + return $ref; +} + +sub inner_estimate { + my ( $self, $level ) = @_; + my $mxl = $self->{'localstate'}->{'maxlevel'}; + + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level) if $level > $mxl; + + my $dn = $self->target(); + my $ref = $self->rsync_ref_for_level($level); + my $batch = $self->generate_rsync_batch($ref->dirname(), $dn); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'rsync batch', problem => 'seek', errno => $!) + unless $batch->seek(0, &Fcntl::SEEK_END); + my $sz = $self->int2big($batch->tell()); + return $sz; + # $batch is removed once out of scope +} + +sub inner_backup { + # XXX assert level==0 if no --record + my ( $self, $fdout ) = @_; + my $dn = $self->target(); + my $level = $self->{'options'}->{'level'}; + + my $dst; # only used in --record case, but needed again further below + if ( $self->{'options'}->{'record'} ) { + my $bld = $self->best_link_dest($level); + $dst = $self->empty_rsync_state_dir(0)->dirname(); + $self->{'newrsyncstate'} = $dst; # save in case repair needed + $self->capture_rsync_state($dn, $dst, $bld->dirname()); + $dn = $dst; + } + + my $ref = $self->rsync_ref_for_level($level); + my $batch = $self->generate_rsync_batch($ref->dirname(), $dn); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'rsync batch', problem => 'seek', errno => $!) + unless $batch->seek(0, &Fcntl::SEEK_SET); + my $fdin = fileno($batch); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'rsync batch', problem => 'seek', errno => $!) + unless defined POSIX::lseek($fdin, 0, &POSIX::SEEK_SET); # paranoid? + my $size = $self->shovel($fdin, $fdout); + + $self->emit_index_entry('/'); + + if ( $self->{'options'}->{'record'} ) { + $self->update_local_state($self->{'localstate'}, $level, { + 'rsyncstate' => $dst }); + } + + return $size; +} + +sub inner_restore { + my $self = shift; + my $fdin = shift; + my $dsf = shift; + my $level = $self->{'options'}->{'level'}; + + if ( 1 != scalar(@_) or $_[0] ne '.' ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'restore targets', + problem => 'Only one (.) supported'); + } + + $self->chdir_to_target(); + my $dn = File::Spec->curdir(); + + if ( 0 == $level ) { + remove_tree($dn, {keep_root => 1}); + } + + my $rslt = system {$self->{'rsyncexecutable'}} ( + 'rsync', + '-rlpt', '--checksum', '--delete-during', '--compress', '--sparse', + '--read-batch', '-', File::Spec->catfile($dn, '') + ); + die Amanda::Application::CalledProcessError->transitionalError( + cmd => 'rsync', returncode => $rslt) + unless 0 == $rslt; + + POSIX::close($fdin); +} + +sub update_local_state { + my ( $self, $state, $level, $opthash ) = @_; + $self->{'orphanedrsyncstates'} = []; + for ( my ($l, $oh); ($l, $oh) = each %$state; ) { + next if 'maxlevel' eq $l or (0 + $l) lt $level; + push @{$self->{'orphanedrsyncstates'}}, $oh->{'rsyncstate'}; + } + $self->SUPER::update_local_state($state, $level, $opthash); +} + +sub write_local_state { + my ( $self, $levhash ) = @_; + $self->SUPER::write_local_state($levhash); + for my $ors ( @{$self->{'orphanedrsyncstates'}} ) { + remove_tree($ors); + } +} + +# Should write_local_state not be called, the (possibly large) newrsyncstate +# directory would be leaked: not referred to by any saved state, so never +# reclaimed in normal operation. So, remove it here. +sub repair_local_state { + my ( $self ) = @_; + remove_tree($self->{'newrsyncstate'}); + $self->SUPER::repair_local_state(); +} + +sub command_selfcheck { + my ( $self ) = @_; + my $why = $self->rsync_is_unusable(); + $self->check( ! $why, $why); + $self->SUPER::command_selfcheck(); +} + +package main; + +Amanda::Application::AmOpaqueTree->run(); diff --git a/application-src/amsvnmakehotcopy.pl b/application-src/amsvnmakehotcopy.pl new file mode 100644 index 0000000000..897ab08c7f --- /dev/null +++ b/application-src/amsvnmakehotcopy.pl @@ -0,0 +1,105 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Script::AmSvnMakeHotcopy; + +use base 'Amanda::Script::Abstract'; + +use File::Spec; +use File::Path qw(make_path remove_tree); +use IO::File; + +sub new { + my ( $class, $execute_where, $refopthash ) = @_; + my $self = $class->SUPER::new($execute_where, $refopthash); + + return $self; +} + +sub check_properties { + my ( $self ) = @_; + + $self->{'svnadminexecutable'} = $self->{'options'}->{'svnadminexecutable'}; + + $self->{'svnrepository'} = $self->{'options'}->{'svnrepository'}; + if ( !defined $self->{'svnrepository'} ) { + die Amanda::Script::InvocationError->transitionalError( + item => 'property', value => 'svnrepository', problem => 'missing'); + } + + $self->{'incremental'} = $self->{'options'}->{'incremental'}; +} + +sub declare_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'svnadminexecutable=s', 'svnrepository=s', + 'clean-logs=s', 'incremental=s' ); + + $class->store_option($refopthash, 'svnadminexecutable', 'svnadmin'); + $class->store_option($refopthash, + 'clean-logs', $class->boolean_property_setter($refopthash)); + $class->store_option($refopthash, + 'incremental', $class->boolean_property_setter($refopthash)); +} + +sub command_pre_dle_estimate { + my ( $self ) = @_; + + my $repo = $self->{'svnrepository'}; + my $dst = $self->{'options'}->{'device'}; + my @opts; + + push @opts, '--clean-logs' if $self->{'options'}->{'clean-logs'}; + push @opts, '--incremental' if $self->{'incremental'}; + + make_path($dst); + remove_tree($dst, {keep_root => 1}) unless $self->{'incremental'}; + + my $rslt = system {$self->{'svnadminexecutable'}} ( + 'svnadmin', 'hotcopy', @opts, '--', $repo, $dst ); + + die Amanda::Script::CalledProcessError->transitionalError( + cmd => 'svnadmin hotcopy', returncode => $rslt) + unless 0 == $rslt; +} + +package main; + +Amanda::Script::AmSvnMakeHotcopy->run(); diff --git a/application-src/amzfs-holdsend.pl b/application-src/amzfs-holdsend.pl new file mode 100644 index 0000000000..57db04b06d --- /dev/null +++ b/application-src/amzfs-holdsend.pl @@ -0,0 +1,760 @@ +#!@PERL@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use lib '@amperldir@'; +use strict; +use warnings; + +package Amanda::Application::AmZfsHoldSend::Dataset; +# A class representing a single ZFS dataset (filesystem, volume, or snapshot). +# name -- name (last component, relative to name of parent) +# fqname -- fully-qualified name +# dstype -- filesystem, volume, or snapshot +# parent -- another Dataset instance (unless this is the exploration root) +# snapshots -- array of instances representing snapshots (where applicable) +# children -- array of instances representing children (where applicable) +# nholds -- for a snapshot, the number of holds recorded on it +# holds() -- for a snapshot, array of tags placed on it with 'zfs hold' + +my $rx_list = qr/ + ^ + (?P[^\t]++) + \t + (?Pfilesystem|snapshot|volume) + \t + (?P-|\d++) + $ +/ox; + +my $holdtagprefix; +my $rx_holdtag; +my $zfsexecutable; + +# The caller should supply the prefix for the hold tags of interest (there may +# be other hold tags placed on snapshots for purposes nothing to do with +# Amanda). The regex will match a string beginning with the HOLDTAGPREFIX +# property and containing a matching group of digits -- the digit string +# represents an Amanda backup level made from the tagged snapshot. It +# also allows (and does not capture) arbitrary trailing whitespace. +sub set_holdtagprefix { + my ( $class, $pfx ) = @_; + my $qpfx = quotemeta($pfx); + my $rx = qr/^$qpfx (\d++)\s*+$/o; + $holdtagprefix = $pfx; + $rx_holdtag = $rx; +} + +sub set_zfsexecutable { + my ( $class, $zx ) = @_; + $zfsexecutable = $zx; +} + +# Construct a Dataset instance from a single line of output from the appropriate +# zfs list command. The caller should first allocate an empty map and pass it +# as $ns when constructing the first dataset encountered (the DEVICE specified +# for the DLE, the root of recursive exploration). The caller should continue +# passing the same $ns on subsequent constructor calls, and it will be used to +# resolve parent links. Once all of the zfs list output has been processed, $ns +# is no longer needed. +sub new { + my ( $class, $listline, $ns ) = @_; + + my $self; + if ( not my ( $fqname, $dstype, $nholds ) = $listline =~ m/$rx_list/ ) { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'zfs list line', value => $listline, problem => 'strange'); + } else { + $self = { fqname => $fqname, dstype => $dstype }; + + my ( $parent, $name ); + + if ( 'snapshot' eq $dstype ) { + $self->{'nholds'} = $nholds; + $self->{'holds'} = [] if 0 == $nholds; + ( $parent, $name ) = split('@', $fqname, 2); + } else { + my @parts = $fqname =~ m,^(?:(.+)/)?([^/]+)$,o; + if ( 1 < scalar(@parts) ) { + ( $parent, $name ) = @parts; + } else { + $name = $fqname; + } + } + + $self->{'name'} = $name; + $self->{'children'} = [] if 'filesystem' eq $dstype; + $self->{'snapshots'} = [] if 'snapshot' ne $dstype; + $self->{'parent'} = $ns->{$parent} if $parent; + + bless $self, $class; + + if ( $self->{'parent'} ) { + if ( 'snapshot' eq $dstype ) { + push @{$self->{'parent'}->{'snapshots'}}, $self; + } else { + push @{$self->{'parent'}->{'children'}}, $self; + } + } + $ns->{$fqname} = $self; + } + return $self; +} + +# Called on a snapshot, return array of holds on that snapshot. +# The only holds included are those that match the regular expression +# passed to set_rx_holdtag, which the caller must have done before first +# calling this. Only the tagarg (matching group) portion of each hold +# tag is stored in the array. +# This sub is lazy, and spawns the 'zfs holds' command to gather the +# information the first time it is called. +# The tagargs are assumed to be digit strings (representing Amanda backup +# levels), will have 0 added to numify them, and the constructed array will +# have them in ascending order (for simplicity later). +sub holds { + my ( $self ) = @_; + + my $hs = $self->{'holds'}; + return $hs if defined $hs; + + die Amanda::Application::ImplementationError->transitionalError( + item => 'holds()', problem => 'called on non-snapshot') + if 'snapshot' ne $self->{'dstype'}; + + $hs = []; + + open my $holdfh, '-|', $zfsexecutable, 'holds', '--', $self->{'fqname'}; + + my $header = <$holdfh>; + my $tagstart = index $header, 'TAG'; + my $timstart = index $header, 'TIMESTAMP', $tagstart; + + while ( my $line = <$holdfh> ) { + my $tag = substr($line, $tagstart, $timstart - $tagstart); + if ( my ($tagarg) = $tag =~ m/$rx_holdtag/o ) { + push @$hs, 0 + $tagarg; + } + } + $holdfh->close(); + + my @sorted = sort {$a <=> $b} @$hs; + $hs = \@sorted; + $self->{'holds'} = $hs; + return $hs; +} + +# Called on a filesystem or volume, returns (map, h) where map is a reference +# mapping backup levels to snapshots, and h is the highest level seen. +# Verifies that the recorded levels are consecutive integers starting at zero +# and that no two snapshots have been tagged with a given level. +sub levels_to_snapshots { + my ( $self ) = @_; + + my $lowestlevel; + my $latestlevel; + my %levtosnap; + + for my $snap ( @{$self->{'snapshots'}} ) { # zfs lists these in chron. order + for my $level ( @{$snap->holds()} ) { + unless ( defined $lowestlevel ) { + $lowestlevel = $level; + } else { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'dataset', value => $self->{'fqname'}, + problem => 'level tags nonmonotonic') + if $level < $lowestlevel; + } + unless ( defined $latestlevel ) { + $latestlevel = $level; + } else { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'dataset', value => $self->{'fqname'}, + problem => 'level tags nonconsecutive') + if $level != 1 + $latestlevel; + $latestlevel = $level; + } + unless ( exists $levtosnap{$level} ) { + $levtosnap{$level} = $snap; + } else { + die Amanda::Application::EnvironmentError->transitionalError( + item => 'dataset', value => $self->{'fqname'}, + problem => 'level tags nonunique'); + } + } + } + + # It is ok to have no tagged snapshots (no prior backup has been done). + return ( \%levtosnap, $latestlevel ) if not %levtosnap; + + # It is not ok to have some, but no level 0. + die Amanda::Application::EnvironmentError->transitionalError( + item => 'dataset', value => $self->{'fqname'}, + problem => 'level 0 hold not found') unless 0 == $lowestlevel; + + return ( \%levtosnap, $latestlevel ); +} + +# If this is a filesystem (can have children), generates the maps for any +# children and verifies they map the wanted levels to the same snapshot names. +sub confirm_matching_levels_children { + my ( $self, @levels ) = @_; + my ( $levtosnap, $highest ) = $self->levels_to_snapshots(); + + if ( 'filesystem' eq $self->{'dstype'} ) { + for my $kid ( @{$self->{'children'}} ) { + my ( $kidmap, $kidh ) = $kid->levels_to_snapshots(); + for my $lev ( @levels ) { + my $snap = $levtosnap->{$lev}; + my $ksnap = $kidmap->{$lev} or die + Amanda::Application::EnvironmentError->transitionalError( + problem => 'missing level ' . $lev, + item => 'dataset', value => $kid->{'fqname'}); + die Amanda::Application::EnvironmentError->transitionalError( + problem => 'level ' . $lev . + ' maps to different snapshot name', + item => 'dataset', value => $kid->{'fqname'}) + unless $ksnap->{'name'} eq $snap->{'name'}; + } + } + } +} + +# For a volume, or a filesystem with no children, just its list of snapshots. +# For a filesystem with children, that list, less any snapshots missing from +# common_snapshots of any child. The order of my original snapshot list is +# preserved, so that the concept of "oldest" or "latest" common snapshot can +# mean something (though it is not guaranteed to be so in all children; a +# remaining test that will need to be made, for an incremental backup, is that +# at each node in the tree to back up, the chosen base snapshot precedes the +# chosen goal snapshot. No stronger ordering assumptions necessarily hold.) +sub common_snapshots { + my ( $self ) = @_; + my @snapnames = map { $_->{'name'} } @{$self->{'snapshots'}}; + my $kids = $self->{'children'}; + return \@snapnames unless defined($kids) and @$kids; + + my %counts; + for my $kid ( @$kids ) { + for my $snap ( @{$kid->common_snapshots()} ) { + $counts{$snap}++; + } + } + + my $fullcount = scalar(@$kids); + my @common; + for my $snap ( @snapnames ) { + push @common, $snap if $fullcount == ( $counts{$snap} // 0 ); + } + + return \@common; +} + +# The trivial case snap1 eq snap2 is allowed (Amanda could run before another +# snapshot has been created); else snap1 must be everywhere older than snap2. +# (The case of multiple snapshots with the same name on a single dataset does +# not arise; zfs itself sees to that.) +sub consistently_ordered { + my ( $self, $snap1, $snap2 ) = @_; + + my @snapnames = map { $_->{'name'} } @{$self->{'snapshots'}}; + my @pair = grep { $_ eq $snap1 or $_ eq $snap2 } @snapnames; + return 0 unless @pair and $pair[0] eq $snap1; + + my $kids = $self->{'children'} or return 1; + + for my $kid ( @$kids ) { + return 0 unless $kid->consistently_ordered($snap1, $snap2); + } + + return 1; +} + +sub apply_hold { + my ( $self, $holdtag ) = @_; + + my $rslt = system {$zfsexecutable} ( + 'zfs', 'hold', '-r', '--', $holdtag, $self->{'fqname'} + ); + die Amanda::Application::CalledProcessError->transitionalError( + cmd => 'zfs hold', returncode => $rslt) + unless 0 == $rslt; +} + +sub release_hold { + my ( $self, $level ) = @_; + + # This may need to work even when levels_to_snapshots() wouldn't succeed, + # meaning the same hold tag may not map to the same snapshot throughout the + # tree. So rather than a simple release -r, descend the tree and do a + # single zfs release at each child on the snapshot there mapped to the + # given level. + my $holdtag = $holdtagprefix . ' ' . $level; + my $rslt = system {$zfsexecutable} ( + 'zfs', 'release', '--', $holdtag, $self->{'fqname'} + ); + die Amanda::Application::CalledProcessError->transitionalError( + cmd => 'zfs release', returncode => $rslt) + unless 0 == $rslt; + + for my $pkid ( @{$self->{'parent'}->{'children'}} ) { + for my $snap ( @{$pkid->{'snapshots'}} ) { + if ( grep { $_ == $level } @{$snap->holds()} ) { + $snap->release_hold($level); + } + } + } +} + + +# +# The main attraction. +# +package Amanda::Application::AmZfsHoldSend; + +use base 'Amanda::Application::Abstract'; + +use IPC::Open3; + +sub supports_host { my ( $class ) = @_; return 1; } +sub supports_disk { my ( $class ) = @_; return 1; } +sub supports_index_line { my ( $class ) = @_; return 1; } +sub supports_message_line { my ( $class ) = @_; return 1; } +sub supports_record { my ( $class ) = @_; return 1; } +sub supports_calcsize { my ( $class ) = @_; return 1; } +sub supports_client_estimate { my ( $class ) = @_; return 1; } +sub supports_multi_estimate { my ( $class ) = @_; return 1; } + +sub max_level { my ( $class ) = @_; return 'DEFAULT'; } + +sub new { + my ( $class, $refopthash ) = @_; + my $self = $class->SUPER::new($refopthash); + + $self->{'zfsexecutable'} = $self->{'options'}->{'zfsexecutable'}; + $self->{'holdtagprefix'} = $self->{'options'}->{'holdtagprefix'}; + + $self->{'localstate'} = $self->read_local_state(); # custom overridden here + return $self; +} + +sub declare_common_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'zfsexecutable=s', 'holdtagprefix=s', + 'uncompressed=s', 'dedup=s', 'large-block=s', + 'embed=s', 'raw=s' ); + + $class->store_option($refopthash, 'zfsexecutable', 'zfs'); + $class->store_option($refopthash, + 'holdtagprefix', 'org.amanda holdsend'); + $class->store_option($refopthash, + 'uncompressed', $class->boolean_property_setter($refopthash)); + $class->store_option($refopthash, + 'uncompressed', 1); # why 1? -c not always supported + $class->store_option($refopthash, + 'dedup', $class->boolean_property_setter($refopthash)); + $class->store_option($refopthash, + 'large-block', $class->boolean_property_setter($refopthash)); + $class->store_option($refopthash, + 'embed', $class->boolean_property_setter($refopthash)); + $class->store_option($refopthash, + 'raw', $class->boolean_property_setter($refopthash)); +} + +sub declare_restore_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->SUPER::declare_restore_options($refopthash, $refoptspecs); + push @$refoptspecs, ( 'destructive=s' ); + push @$refoptspecs, ( 'unmounted=s' ); + push @$refoptspecs, ( 'overrideproperty=s@' ); + push @$refoptspecs, ( 'excludeproperty=s@' ); + + $class->store_option($refopthash, + 'destructive', $class->boolean_property_setter($refopthash)); + $class->store_option($refopthash, 'destructive', 1); + $class->store_option($refopthash, + 'unmounted', $class->boolean_property_setter($refopthash)); + $class->store_option($refopthash, 'overrideproperty', []); + $class->store_option($refopthash, 'excludeproperty', []); +} + +# This read_local_state does not take the getopt arguments taken by the +# generic version it overrides ... this version is specific to this class +# and knows exactly what it's retrieving. +sub read_local_state { + my ( $self ) = @_; + + Amanda::Application::AmZfsHoldSend::Dataset->set_holdtagprefix( + $self->{'holdtagprefix'} + ); + Amanda::Application::AmZfsHoldSend::Dataset->set_zfsexecutable( + $self->{'zfsexecutable'} + ); + + my @cmd = ( + $self->{'zfsexecutable'}, 'list', '-Hr', + '-t', 'filesystem,volume,snapshot', + '-o', 'name,type,userrefs', '--', + $self->target() + ); + + my %ns; # initially empty dataset namespace + my $topds; + + open my $listfh, '-|', @cmd; + while ( my $line = <$listfh> ) { + my $ds = Amanda::Application::AmZfsHoldSend::Dataset->new($line, \%ns); + $topds //= $ds; + } + $listfh->close(); + + my ( $levtosnap, $latestlevel ) = $topds->levels_to_snapshots(); + + my %state; + my $maxlevel = ( $latestlevel // -1) + 1; + + while ( my ( $lev, $snap ) = each %$levtosnap ) { + $state{$lev} = { level => $lev, snapshot => $snap->{'name'} }; + } + + my $snaps = $topds->common_snapshots(); + + # Even though it must have been true as of the last successful backup, + # the condition (each recorded level refers to a snapshot common to all + # nodes in the tree) can have been invalidated since by the creation of + # new datasets (or even clones). Be sure to know the last level (if any) + # for which the condition does hold. + + my $firstbrokenlevel = 0; + for ( ; $firstbrokenlevel < $maxlevel ; $firstbrokenlevel += 1 ) { + my $sname = $state{$firstbrokenlevel}->{'snapshot'}; + next if grep { $_ eq $sname } @$snaps; + $state{'oldlist'} //= []; + last; + } + while ( $firstbrokenlevel < $maxlevel ) { + $maxlevel -= 1; + push @{$state{'oldlist'}}, $state{$maxlevel}; + delete $state{$maxlevel}; + } + + $state{'topds'} = $topds; + $state{'maxlevel'} = $maxlevel; + if ( @$snaps ) { + $state{'oldestsnapshot'} = $snaps->[0]; + $state{'newestsnapshot'} = $snaps->[scalar(@$snaps)-1]; + } + return \%state; +} + +sub update_local_state { + my ( $self, $state, $level, $opthash ) = @_; + + $state->{'oldlist'} //= []; + $state->{'newlist'} //= []; + + push @{$state->{'newlist'}}, $opthash; + my $oldmax = $state->{'maxlevel'}; + for ( my $lv = $level; $lv < $oldmax; $lv += 1 ) { + push @{$state->{'oldlist'}}, $state->{$lv}; + delete $state->{$lv}; + } + $state->{'maxlevel'} = 1 + $level; + $state->{$level} = $opthash; +} + +sub write_local_state { + my ( $self, $state ) = @_; + return unless $state->{'oldlist'} and $state->{'newlist'}; # nothing to do + + my ( $levtosnap, $highest ) = $state->{'topds'}->levels_to_snapshots(); + + # Remove old holds first. This may be unsatisfying on a rigorous theoretical + # level, leaving a brief moment when no needed snapshot has a hold--and the + # concern could be more than theoretical, if an environment has done a lot + # of zfs destroy -d operations and the released snapshots all vanish before + # the new hold can be placed. But balancing that is another practical issue: + # if there has been no new snapshot created, a backup may try to reapply the + # same hold tag on the same snapshot, and that fails. + # An alternative would be to complicate the logic and avoid releasing the + # hold for $level if it will be going right back on the same snapshot. That + # would work, but sacrifice the accidental feature that the hold timestamp + # records when the latest corresponding backup was taken. + # Another alternative, not sacrificing the timestamp, would be to generate + # a random hold tag to apply on the snapshot, then release and reapply the + # original tag, then release the generated one. But that's still more + # complexity aimed at a low-likelihood event, so, some other day.... + for my $opthash ( @{$state->{'oldlist'}} ) { + my $level = $opthash->{'level'}; + my $snapobj = $levtosnap->{$level}; + $snapobj->release_hold($level); + } + + for my $opthash ( @{$state->{'newlist'}} ) { + my $level = $opthash->{'level'}; + my $holdtag = $self->{'holdtagprefix'} . ' ' . $level; + my $snapname = $opthash->{'snapshot'}; + my $snapobj = ( + grep { $_->{'name'} eq $snapname } + @{$state->{'topds'}->{'snapshots'}} + )[0]; + $snapobj->apply_hold($holdtag); + } +} + +my $rx_compratio = qr/^(\d++)\.(\d\d)x$/o; + +my $rx_nvPsize = qr/^size\t(\d++)$/o; + +sub inner_estimate { + my ( $self, $level ) = @_; + my $latestsnapshot = $self->{'localstate'}->{'newestsnapshot'}; + + if ( $self->{'options'}->{'calcsize'} and defined $latestsnapshot ) { + return $self->inner_estimate_nvP($level, $latestsnapshot); + } else { + return $self->inner_estimate_brute($level, $latestsnapshot); + } +} + +# The complete output of send -nvP -R includes sizes for all descendant +# datasets and intermediate snapshots, followed by one final line matching +# /^size\t/ with the total. This inner_estimate (which is called in a loop +# by command_estimate, once for each level) ignores all but the last line +# and returns that total as the size for the level. Clearly, it would be +# possible to override command_estimate itself, skip the loop, and calculate +# the estimates for all levels from the output of a single send -nvP. Left for +# future, as this is simple and not dreadfully slow. +# The output of send -nvP goes to stderr, not stdout. +sub inner_estimate_nvP { + my ( $self, $level, $latestsnapshot ) = @_; + + my @cmd = $self->construct_send_cmd($level, $latestsnapshot); + + splice @cmd, 2, 0, '-nvP'; + + my $sizestr; + my ( $wtr, $rdr ); + my $sendpid = open3($wtr, $rdr, $rdr, @cmd); + $wtr->close(); + while ( <$rdr> ) { + ( $sizestr ) = m/$rx_nvPsize/o; + } + $rdr->close(); + waitpid($sendpid, 0); + + die Amanda::Application::CalledProcessError->transitionalError( + cmd => 'zfs send -nvP', returncode => $?) + unless 0 == $?; + + die Amanda::Application::EnvironmentError->transitionalError( + item => 'reading size from zfs send -nvP', problem => 'failed') + unless defined($sizestr); + + return Math::BigInt->new($sizestr); +} + +sub inner_estimate_brute { + my ( $self, $level, $latestsnapshot ) = @_; + + if ( 0 == $level ) { + open my $getfh, '-|', $self->{'zfsexecutable'}, + 'get', '-Hp', '-o', 'value', '--', 'used,compressratio', + $self->target(); + my $used = <$getfh>; + my $compratio = <$getfh>; + $getfh->close(); + $used = Math::BigInt->new($used); + if ( $self->{'options'}->{'uncompressed'} ) { + my ( $wholes, $cents ) = $compratio =~ m/$rx_compratio/o; + $used->bmul($wholes . $cents)->badd(99)->bdiv(100) + } + return $used; + } + + my $mxl = $self->{'localstate'}->{'maxlevel'}; + + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level) if $level > $mxl; + + my $priorsnapshot = $self->{'localstate'}->{$level - 1}->{'snapshot'}; + + # In this easy case, zero isn't quite accurate; if the planner actually + # does, for some reason, choose this level, inner_backup will in fact do + # an incremental zfs send from the snapshot to itself. That produces + # warnings from zfs send, but also a valid, short but non-zero-length, + # send stream from which zfs receive does nothing, successfully. (Just + # generating a zero-length stream will make zfs receive do nothing, + # unsuccessfully.) Almost nothing, anyway: the stream does contain the + # latest values of properties. + # So returning zero here is a lie, but an easy and not-far-from-the-truth + # one, and this is, after all, an estimate. Perhaps the planner will see + # this zero estimate and choose a different level, which would be ok. + + return Math::BigInt->bzero() if $latestsnapshot eq $priorsnapshot; + + # Ok, it wasn't any of the easy cases. + + open my $sendfh, '-|', $self->construct_send_cmd($level, $latestsnapshot); + my $buffer; + my $size = Math::BigInt->bzero(); + my $s; + + while (($s = sysread($sendfh, $buffer, 32768)) > 0) { + $size->badd($s); + } + $sendfh->close(); + return $size; +} + +sub construct_send_cmd { + my ( $self, $level, $latestsnapshot ) = @_; + my $dn = $self->target(); + + my $mxl = $self->{'localstate'}->{'maxlevel'}; + + my @compressed = $self->{'options'}->{'uncompressed'} ? () : ( '-c' ); + my @misc_opts = $self->ozfs_send_options(); + + if ( 0 == $level ) { + return $self->{'zfsexecutable'}, 'send', @compressed, @misc_opts, '-R', + '--', $dn . '@' . $latestsnapshot; + } elsif ( $level > $mxl ) { + die Amanda::Application::DiscontiguousLevelError->transitionalError( + value => $level); + } else { + my $priorsnapshot = $self->{'localstate'}->{$level - 1}->{'snapshot'}; + $self->{'localstate'}->{'topds'}-> + confirm_matching_levels_children($level - 1); + unless ( $self->{'localstate'}->{'topds'}-> + consistently_ordered($priorsnapshot, $latestsnapshot) ) { + die Amanda::Application::EnvironmentError->transitionalError( + item => "Snapshots '$priorsnapshot' and '$latestsnapshot'", + problem => "not consistently ordered"); + } + + return $self->{'zfsexecutable'}, 'send', @compressed, @misc_opts, '-R', + '-I', $priorsnapshot, + '--', $dn . '@' . $latestsnapshot; + } +} + +sub ozfs_send_options { + my ( $self ) = @_; + + my @opts; + push @opts, '-D' if $self->{'options'}->{'dedup'}; + push @opts, '-L' if $self->{'options'}->{'large-block'}; + push @opts, '-e' if $self->{'options'}->{'embed'}; + push @opts, '-w' if $self->{'options'}->{'raw'}; + return @opts; +} + +sub inner_backup { + # XXX assert level==0 if no --record + my ( $self, $fdout ) = @_; + my $dn = $self->target(); + my $level = $self->{'options'}->{'level'}; + + my $latestsnapshot = $self->{'localstate'}->{'newestsnapshot'}; + unless ( defined $latestsnapshot ) { + die Amanda::Application::EnvironmentError->transitionalError( + item => $dn, problem => 'At least one snapshot must exist'); + } + + open my $sendfh, '-|', $self->construct_send_cmd($level, $latestsnapshot); + + my $size = Math::BigInt->bzero(); + my $buffer; + my $s; + + while (($s = sysread($sendfh, $buffer, 32768)) > 0) { + Amanda::Util::full_write($fdout, $buffer, $s); + $size->badd($s); + } + $sendfh->close(); + + $self->emit_index_entry('/'); + + if ( $self->{'options'}->{'record'} ) { + $self->update_local_state($self->{'localstate'}, $level, { + level => $level, snapshot => $latestsnapshot }); + } + + return $size; +} + +sub check_restore_options { + my ( $self ) = @_; + + $self->SUPER::check_restore_options(); + + $self->{'destructive'} = $self->{'options'}->{'destructive'}; +} + +sub inner_restore { + my $self = shift; + my $fdin = shift; + my $dsf = shift; + my $level = $self->{'options'}->{'level'}; + + if ( 1 != scalar(@_) or $_[0] ne '.' ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'restore targets', + problem => 'Only one (.) supported'); + } + + my $dn = $self->target(); + my @force = ( $self->{'destructive'} or 0 == $level ) ? ( '-F' ) : (); + my @unmounted = $self->{'options'}->{'unmounted'} ? ( '-u' ) : (); + my @propoverrides = + map { ('-o', $_) } @{$self->{'options'}->{'overrideproperty'}}; + my @propexcludes = + map { ('-o', $_) } @{$self->{'options'}->{'excludeproperty'}}; + + # $fdin happens to be fileno(STDIN), so may as well just spawn zfs receive + # to read directly, rather than spoonfeeding it through a pipe. + my $rslt = system {$self->{'zfsexecutable'}} ( + 'zfs', 'receive', @force, @unmounted, @propoverrides, @propexcludes, + '--', $dn + ); + if ( 0 != $rslt ) { + die Amanda::Application::CalledProcessError->transitionalError( + cmd => 'zfs receive', returncode => $?); + }; +} + +package main; + +Amanda::Application::AmZfsHoldSend->run(); diff --git a/installcheck/Makefile.am b/installcheck/Makefile.am index c025a88c81..1e6a9226ba 100644 --- a/installcheck/Makefile.am +++ b/installcheck/Makefile.am @@ -21,7 +21,11 @@ all_tests += $(common_tests) client_tests = \ noop \ ambsdtar \ + amgrowingfile \ + amgrowingzip \ amgtar \ + amooraw \ + amopaquetree \ ampgsql \ amraw \ amstar \ diff --git a/installcheck/amgrowingfile.pl b/installcheck/amgrowingfile.pl new file mode 100644 index 0000000000..7b0118d7e4 --- /dev/null +++ b/installcheck/amgrowingfile.pl @@ -0,0 +1,119 @@ +# Copyright (c) 2009-2012 Zmanda, Inc. All Rights Reserved. +# Copyright (c) 2013-2016 Carbonite, Inc. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Contact information: Carbonite Inc., 756 N Pastoria Ave +# Sunnyvale, CA 94086, USA, or: http://www.zmanda.com + +use Test::More tests => 30; + +use lib '@amperldir@'; +use strict; +use warnings; +use Installcheck; +use Amanda::Constants; +use Amanda::Debug; +use Amanda::Paths; +use Amanda::Tests; +use File::Path; +use Installcheck::Application; +use IO::File; + +Amanda::Debug::dbopen("installcheck"); +Installcheck::log_test_output(); + +my $app = Installcheck::Application->new('amgrowingfile'); + +my $support = $app->support(); +is($support->{'INDEX-LINE'}, 'YES', "supports indexing"); +is($support->{'MESSAGE-LINE'}, 'YES', "supports messages"); +is($support->{'CLIENT-ESTIMATE'}, 'YES', "supports estimates"); +is($support->{'RECORD'}, 'YES', "supports record"); +is($support->{'MULTI-ESTIMATE'}, 'YES', "supports multi-estimates"); +is($support->{'CMD-STREAM'}, 'YES', + "supports command stream to/from sendbackup"); +is($support->{'WANT-SERVER-BACKUP-RESULT'}, 'YES', + "supports server backup results"); + +my $root_dir = "$Installcheck::TMP/installcheck-amgrowingfile"; +my $back_file = "$root_dir/to_backup"; +my $rest_dir = "$root_dir/restore"; + +File::Path::mkpath($root_dir); +File::Path::mkpath($rest_dir); +Amanda::Tests::write_random_file(0xabcde, 1024*256, $back_file); + +my $selfcheck = $app->selfcheck('device' => $back_file, 'level' => 0, 'index' => 'line'); +is($selfcheck->{'exit_status'}, 0, "error status ok"); +ok(!@{$selfcheck->{'errors'}}, "no errors during selfcheck"); + +my $backup = $app->backup('device' => $back_file, 'level' => 0, + 'index' => 'line', 'record' => undef); +is($backup->{'exit_status'}, 0, "error status ok"); +ok(!@{$backup->{'errors'}}, "no errors during backup") + or diag(@{$backup->{'errors'}}); + +is(length($backup->{'data'}), $backup->{'size'}, "reported and actual size match"); + +ok(@{$backup->{'index'}}, "index is not empty"); +is_deeply($backup->{'index'}, ["/"], "index is '/'"); + +open my $bfh, '>>', $back_file; +print $bfh $backup->{'data'}; +close $bfh; + +my $backup1 = $app->backup('device' => $back_file, 'level' => 1, + 'index' => 'line'); +is($backup1->{'exit_status'}, 0, "error status ok"); +ok(!@{$backup1->{'errors'}}, "no errors during backup") + or diag(@{$backup1->{'errors'}}); + +is(length($backup1->{'data'}), $backup1->{'size'}, "reported and actual size match"); + +ok(@{$backup1->{'index'}}, "index is not empty"); +is_deeply($backup1->{'index'}, ["/"], "index is '/'"); + +my $orig_cur_dir = POSIX::getcwd(); +ok($orig_cur_dir, "got current directory"); + +ok(chdir($rest_dir), "changed working directory (for restore)"); + +my $restore = $app->restore('objects' => ['.'],'data' => $backup->{'data'}); +is($restore->{'exit_status'}, 0, "error status ok (default filename)"); +$restore = $app->restore('objects' => ['.'],'data' => $backup1->{'data'}, + 'level'=> 1); +is($restore->{'exit_status'}, 0, "error status ok (default filename; increment)"); + +$app->add_property('target', 'custom-filename'); +$restore = $app->restore('objects' => ['.'],'data' => $backup->{'data'}); +is($restore->{'exit_status'}, 0, "error status ok (custom filename)"); +$restore = $app->restore('objects' => ['.'],'data' => $backup1->{'data'}, + 'level'=> 1); +is($restore->{'exit_status'}, 0, "error status ok (custom filename; increment)"); + +ok(chdir($orig_cur_dir), "changed working directory (back to original)"); + +my $restore_file = "$rest_dir/amgrowingfile-restored"; +ok(-f "$restore_file", "amgrowingfile-restored restored"); +is(`cmp $back_file $restore_file`, "", "restore match"); + +$restore_file = "$rest_dir/custom-filename"; +ok(-f "$restore_file", "custom-filename restored"); +is(`cmp $back_file $restore_file`, "", "restore match"); + +# cleanup +#exit(1); +rmtree($root_dir); diff --git a/installcheck/amgrowingzip.pl b/installcheck/amgrowingzip.pl new file mode 100644 index 0000000000..6dd09bdf66 --- /dev/null +++ b/installcheck/amgrowingzip.pl @@ -0,0 +1,198 @@ +# Copyright (c) 2009-2012 Zmanda, Inc. All Rights Reserved. +# Copyright (c) 2013-2016 Carbonite, Inc. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Contact information: Carbonite Inc., 756 N Pastoria Ave +# Sunnyvale, CA 94086, USA, or: http://www.zmanda.com + +use Test::More; + +use lib '@amperldir@'; +use strict; +use warnings; +use Installcheck; +use Amanda::Constants; +use Amanda::Debug; +use Amanda::Paths; +use Amanda::Tests; +use Fcntl qw(SEEK_SET); +use File::Path; +use Installcheck::Application; +use IO::File; + +eval { + require Archive::Zip; + plan tests => 33; + 1; +} or do { + plan skip_all => 'tested only if Archive::Zip is installed'; +}; + +Amanda::Debug::dbopen("installcheck"); +Installcheck::log_test_output(); + +my $app = Installcheck::Application->new('amgrowingzip'); + +my $support = $app->support(); +is($support->{'INDEX-LINE'}, 'YES', "supports indexing"); +is($support->{'MESSAGE-LINE'}, 'YES', "supports messages"); +is($support->{'CLIENT-ESTIMATE'}, 'YES', "supports estimates"); +is($support->{'RECORD'}, 'YES', "supports record"); +is($support->{'MULTI-ESTIMATE'}, 'YES', "supports multi-estimates"); +is($support->{'CMD-STREAM'}, 'YES', + "supports command stream to/from sendbackup"); +is($support->{'WANT-SERVER-BACKUP-RESULT'}, 'YES', + "supports server backup results"); + +my $root_dir = "$Installcheck::TMP/installcheck-amgrowingzip"; +my $back_file = "$root_dir/to_backup"; +my $rest_dir = "$root_dir/restore"; + +File::Path::mkpath($root_dir); +File::Path::mkpath($rest_dir); + +# Create a test ZIP archive whose members will consist of random stuff; +# Amanda::Tests::write_random_file only writes an actual file. So (this +# is a test, it doesn't have to be pretty) just do that, then slurp in +# the file contents as $random_stuff; the file will get truncated and +# overwritten later as the actual ZIP archive. + +Amanda::Tests::write_random_file(0xabcde, 1024*256, $back_file); + +my $random_stuff; +{ + local $/; + open my $fh, '<', $back_file; + $random_stuff = <$fh>; + close $fh; +} + +# Ironically, while Perl's Archive::Zip provides all the functionality needed +# to do incremental backup of incrementally-growing ZIP files (a function to +# return the central directory offset is enough), it isn't adequate to actually +# CREATE incrementally-growing ZIP files (easy as that is in Python with the +# zipfile module). That makes creating the test case in Perl a bit tricky. +# Perl does, however, provide enough functionality to do the job backwards: +# write a two-entry ZIP file, then delete the second entry from the central +# directory, and rewrite the central directory at the original offset of the +# second entry in the ZIP file, leaving a one-entry ZIP. By first saving the +# raw bytes from the second entry offset to the end of the file before +# truncating, they can be written back later to emulate 'growing' the ZIP by +# adding the second entry. The things done in the name of testing.... + +my $zf = Archive::Zip->new(); +my $thing1 = $zf->addString($random_stuff, 'Thing1'); +my $thing2 = $zf->addString($random_stuff, 'Thing2'); + +open my $bfh, '+>', $back_file; +$zf->writeToFileHandle($bfh, 1); +ok($thing2->wasWritten(), "both entries in test zip were written"); + +# Backspace to the offset where thing2 begins, and save everything from there +# to the end of the file (including the directory records at the end). The +# writeLocalHeaderRelativeOffset method is just a getter, doesn't write stuff. + +my $thing2offset = $thing2->writeLocalHeaderRelativeOffset(); +seek $bfh, $thing2offset, SEEK_SET; +my $tail_data; +{ + local $/; + $tail_data = <$bfh>; +} + +# Remove thing2 from the in-memory directory of entries, then rewrite that +# directory at the offset where thing2 originally started in the file. + +$zf->removeMember($thing2); +$zf->writeCentralDirectory($bfh, $thing2offset); +truncate $bfh, (tell $bfh); + +# Wasn't that easy? Now it's a one-entry archive suitable for testing +# backup level 0. + +close $bfh; + +my $selfcheck = $app->selfcheck('device' => $back_file, 'level' => 0, 'index' => 'line'); +is($selfcheck->{'exit_status'}, 0, "error status ok"); +ok(!@{$selfcheck->{'errors'}}, "no errors during selfcheck"); + +my $backup = $app->backup('device' => $back_file, 'level' => 0, + 'index' => 'line', 'record' => undef); +is($backup->{'exit_status'}, 0, "error status ok"); +ok(!@{$backup->{'errors'}}, "no errors during backup") + or diag(@{$backup->{'errors'}}); + +my $size_rounded_up = 1024 * int(( length($backup->{'data'}) + 1023 ) / 1024); +is($size_rounded_up, $backup->{'size'}, "reported and actual size match"); + +ok(@{$backup->{'index'}}, "index is not empty"); +is_deeply($backup->{'index'}, ["/"], "index is '/'"); + +# Ok, here it goes, back to a two-entry archive.... + +open $bfh, '+<', $back_file; +seek $bfh, $thing2offset, SEEK_SET; +print $bfh $tail_data; +close $bfh; + +my $backup1 = $app->backup('device' => $back_file, 'level' => 1, + 'index' => 'line'); +is($backup1->{'exit_status'}, 0, "error status ok"); +ok(!@{$backup1->{'errors'}}, "no errors during backup") + or diag(@{$backup1->{'errors'}}); + +$size_rounded_up = 1024 * int(( length($backup1->{'data'}) + 1023 ) / 1024); +is($size_rounded_up, $backup1->{'size'}, "reported and actual size match"); + +ok(@{$backup1->{'index'}}, "index is not empty"); +is_deeply($backup1->{'index'}, ["/"], "index is '/'"); + +my $orig_cur_dir = POSIX::getcwd(); +ok($orig_cur_dir, "got current directory"); + +ok(chdir($rest_dir), "changed working directory (for restore)"); + +my $restore = $app->restore('objects' => ['.'],'data' => $backup->{'data'}); +is($restore->{'exit_status'}, 0, "error status ok (default filename)"); +$restore = $app->restore('objects' => ['.'],'data' => $backup1->{'data'}, + 'level'=> 1); +is($restore->{'exit_status'}, 0, "error status ok (default filename; increment)"); + +$app->add_property('target', 'custom-filename'); +$restore = $app->restore('objects' => ['.'],'data' => $backup->{'data'}); +is($restore->{'exit_status'}, 0, "error status ok (custom filename)"); +$restore = $app->restore('objects' => ['.'],'data' => $backup1->{'data'}, + 'level'=> 1); +is($restore->{'exit_status'}, 0, "error status ok (custom filename; increment)"); + +ok(chdir($orig_cur_dir), "changed working directory (back to original)"); + +my $restore_file = "$rest_dir/amgrowingzip-restored"; +ok(-f "$restore_file", "amgrowingzip-restored restored"); +is(`cmp $back_file $restore_file`, "", "restore match"); + +$restore_file = "$rest_dir/custom-filename"; +ok(-f "$restore_file", "custom-filename restored"); +is(`cmp $back_file $restore_file`, "", "restore match"); + +$zf = Archive::Zip->new($restore_file); +isnt($zf, undef, "restored archive is present and well-formed"); +is_deeply(\[$zf->memberNames()], \['Thing1', 'Thing2'], + "contains the right entries"); + +# cleanup +#exit(1); +rmtree($root_dir); diff --git a/installcheck/amooraw.pl b/installcheck/amooraw.pl new file mode 100644 index 0000000000..3d3d505a50 --- /dev/null +++ b/installcheck/amooraw.pl @@ -0,0 +1,95 @@ +# Copyright (c) 2009-2012 Zmanda, Inc. All Rights Reserved. +# Copyright (c) 2013-2016 Carbonite, Inc. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Contact information: Carbonite Inc., 756 N Pastoria Ave +# Sunnyvale, CA 94086, USA, or: http://www.zmanda.com + +use Test::More tests => 21; + +use lib '@amperldir@'; +use strict; +use warnings; +use Installcheck; +use Amanda::Constants; +use Amanda::Debug; +use Amanda::Paths; +use Amanda::Tests; +use File::Path; +use Installcheck::Application; +use IO::File; + +Amanda::Debug::dbopen("installcheck"); +Installcheck::log_test_output(); + +my $app = Installcheck::Application->new('amooraw'); + +my $support = $app->support(); +is($support->{'INDEX-LINE'}, 'YES', "supports indexing"); +is($support->{'MESSAGE-LINE'}, 'YES', "supports messages"); +is($support->{'CLIENT-ESTIMATE'}, 'YES', "supports estimates"); +is($support->{'CMD-STREAM'}, 'YES', + "supports command stream to/from sendbackup"); +is($support->{'WANT-SERVER-BACKUP-RESULT'}, 'YES', + "supports server backup results"); + +my $root_dir = "$Installcheck::TMP/installcheck-amooraw"; +my $back_file = "$root_dir/to_backup"; +my $rest_dir = "$root_dir/restore"; + +File::Path::mkpath($root_dir); +File::Path::mkpath($rest_dir); +Amanda::Tests::write_random_file(0xabcde, 1024*256, $back_file); + +my $selfcheck = $app->selfcheck('device' => $back_file, 'level' => 0, 'index' => 'line'); +is($selfcheck->{'exit_status'}, 0, "error status ok"); +ok(!@{$selfcheck->{'errors'}}, "no errors during selfcheck"); + +my $backup = $app->backup('device' => $back_file, 'level' => 0, 'index' => 'line'); +is($backup->{'exit_status'}, 0, "error status ok"); +ok(!@{$backup->{'errors'}}, "no errors during backup") + or diag(@{$backup->{'errors'}}); + +is(length($backup->{'data'}), $backup->{'size'}, "reported and actual size match"); + +ok(@{$backup->{'index'}}, "index is not empty"); +is_deeply($backup->{'index'}, ["/"], "index is '/'"); + +my $orig_cur_dir = POSIX::getcwd(); +ok($orig_cur_dir, "got current directory"); + +ok(chdir($rest_dir), "changed working directory (for restore)"); + +my $restore = $app->restore('objects' => ['.'],'data' => $backup->{'data'}); +is($restore->{'exit_status'}, 0, "error status ok (default filename)"); + +$app->add_property('target', 'custom-filename'); +$restore = $app->restore('objects' => ['.'],'data' => $backup->{'data'}); +is($restore->{'exit_status'}, 0, "error status ok (custom filename)"); + +ok(chdir($orig_cur_dir), "changed working directory (back to original)"); + +my $restore_file = "$rest_dir/amooraw-restored"; +ok(-f "$restore_file", "amooraw-restored restored"); +is(`cmp $back_file $restore_file`, "", "restore match"); + +$restore_file = "$rest_dir/custom-filename"; +ok(-f "$restore_file", "custom-filename restored"); +is(`cmp $back_file $restore_file`, "", "restore match"); + +# cleanup +#exit(1); +rmtree($root_dir); diff --git a/installcheck/amopaquetree.pl b/installcheck/amopaquetree.pl new file mode 100644 index 0000000000..95693e28e7 --- /dev/null +++ b/installcheck/amopaquetree.pl @@ -0,0 +1,154 @@ +# Copyright (c) 2009-2012 Zmanda, Inc. All Rights Reserved. +# Copyright (c) 2013-2016 Carbonite, Inc. All Rights Reserved. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# +# Contact information: Carbonite Inc., 756 N Pastoria Ave +# Sunnyvale, CA 94086, USA, or: http://www.zmanda.com + +use Test::More; + +use lib '@amperldir@'; +use strict; +use warnings; +use Installcheck; +use Amanda::Constants; +use Amanda::Debug; +use Amanda::Paths; +use Amanda::Tests; +use Amanda::Util; +use Fcntl qw(SEEK_SET); +use File::Path; +use Installcheck::Application; +use IO::File; +use IPC::Open3; + +Amanda::Debug::dbopen("installcheck"); +Installcheck::log_test_output(); + +sub rsync_is_unusable { + my ( $self ) = @_; + my ( $wtr, $rdr ); + my $pid = eval { open3($wtr, $rdr, undef, 'rsync', '--version') }; + return $@ if $@; + close $wtr; + my $output = do { local $/; <$rdr> }; + close $rdr; + waitpid $pid, 0; + return $output if $?; + unless ( $output =~ qr/(?:^\s|,\s)hardlinks(?:,\s|$)/m ) { + return 'rsync lacks hardlink support.'; + } + unless ( $output =~ qr/(?:^\s|,\s)batchfiles(?:,\s|$)/m ) { + return 'rsync lacks batchfile support.'; + } + return 0; # hooray, it isn't unusable. +} + +my $why = rsync_is_unusable(); +unless ( $why ) { + plan tests => 28; +} else { + plan skip_all => $why; +} + +my $app = Installcheck::Application->new('amopaquetree'); + +my $support = $app->support(); +is($support->{'INDEX-LINE'}, 'YES', "supports indexing"); +is($support->{'MESSAGE-LINE'}, 'YES', "supports messages"); +is($support->{'CLIENT-ESTIMATE'}, 'YES', "supports estimates"); +is($support->{'RECORD'}, 'YES', "supports record"); +is($support->{'MULTI-ESTIMATE'}, 'YES', "supports multi-estimates"); +is($support->{'CMD-STREAM'}, 'YES', + "supports command stream to/from sendbackup"); +is($support->{'WANT-SERVER-BACKUP-RESULT'}, 'YES', + "supports server backup results"); + +my $root_dir = "$Installcheck::TMP/installcheck-amopaquetree"; +my $back_dir = "$root_dir/to_backup"; +my $rest_dir = "$root_dir/restore"; +my $file1 = "file1"; +my $file2 = "file2"; + +File::Path::mkpath($back_dir); +File::Path::mkpath($rest_dir); + +Amanda::Tests::write_random_file(0xabcde, 1024*256, "$back_dir/$file1"); +Amanda::Tests::write_random_file(0xfedcb, 1024*256, "$back_dir/$file2"); + +my $selfcheck = $app->selfcheck('device' => $back_dir, 'level' => 0, 'index' => 'line'); +is($selfcheck->{'exit_status'}, 0, "error status ok"); +ok(!@{$selfcheck->{'errors'}}, "no errors during selfcheck") + or diag(@{$selfcheck->{'errors'}}); + +my $backup = $app->backup('device' => $back_dir, 'level' => 0, + 'index' => 'line', 'record' => undef); +is($backup->{'exit_status'}, 0, "error status ok"); +ok(!@{$backup->{'errors'}}, "no errors during backup") + or diag(@{$backup->{'errors'}}); + +my $size_rounded_up = 1024 * int(( length($backup->{'data'}) + 1023 ) / 1024); +is($size_rounded_up, $backup->{'size'}, "reported and actual size match"); + +ok(@{$backup->{'index'}}, "index is not empty"); +is_deeply($backup->{'index'}, ["/"], "index is '/'"); + +# make a small modification somewhere inside file2 (just grab some of the +# random data from some spot in file1, to some other spot in file2).... +open my $fh1, '<', $back_dir.'/'.$file1; +open my $fh2, '+<', $back_dir.'/'.$file2; +seek $fh1, 1024*23, SEEK_SET; +seek $fh2, 1024*57, SEEK_SET; +my $buf = Amanda::Util::full_read(fileno($fh1), 1024*32); +close $fh1; +Amanda::Util::full_write(fileno($fh2), $buf, 1024*32); +close $fh2; + +my $backup1 = $app->backup('device' => $back_dir, 'level' => 1, + 'index' => 'line'); +is($backup1->{'exit_status'}, 0, "error status ok"); +ok(!@{$backup1->{'errors'}}, "no errors during backup") + or diag(@{$backup1->{'errors'}}); + +$size_rounded_up = 1024 * int(( length($backup1->{'data'}) + 1023 ) / 1024); +is($size_rounded_up, $backup1->{'size'}, "reported and actual size match"); + +ok(@{$backup1->{'index'}}, "index is not empty"); +is_deeply($backup1->{'index'}, ["/"], "index is '/'"); + +my $orig_cur_dir = POSIX::getcwd(); +ok($orig_cur_dir, "got current directory"); + +ok(chdir($rest_dir), "changed working directory (for restore)"); + +my $restore = $app->restore('objects' => ['.'],'data' => $backup->{'data'}); +is($restore->{'exit_status'}, 0, "error status ok"); +$restore = $app->restore('objects' => ['.'],'data' => $backup1->{'data'}, + 'level'=> 1); +is($restore->{'exit_status'}, 0, "error status ok (increment)"); +ok(chdir($orig_cur_dir), "changed working directory (back to original)"); + +my $restore_file = "$rest_dir/$file1"; +ok(-f "$restore_file", "file1 restored"); +is(`cmp "$back_dir/$file1" $restore_file`, "", "file1 match"); + +$restore_file = "$rest_dir/$file2"; +ok(-f "$restore_file", "file2 restored"); +is(`cmp "$back_dir/$file2" $restore_file`, "", "file2 match"); + +# cleanup +#exit(1); +rmtree($root_dir); diff --git a/man/Makefile.am b/man/Makefile.am index f6ce7f1e01..bf64f2f80b 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -23,15 +23,24 @@ COMMON_MAN_PAGES = amanda.8 \ CLIENT_MAN_PAGES = \ amanda-applications.7 \ + am389bak.8 \ ambackup.8 \ ambsdtar.8 \ amdump_client.8 \ + amgrowingfile.8 \ + amgrowingzip.8 \ amgtar.8 \ + amlibvirtfsfreeze.8 \ + amlvmsnapshot.8 \ + amooraw.8 \ + amopaquetree.8 \ ampgsql.8 \ amraw.8 \ amsamba.8 \ amstar.8 \ amsuntar.8 \ + amsvnmakehotcopy.8 \ + amzfs-holdsend.8 \ amzfs-snapshot.8 \ amzfs-sendrecv.8 diff --git a/man/xml-source/am389bak.8.xml b/man/xml-source/am389bak.8.xml new file mode 100644 index 0000000000..05fb6a9a7b --- /dev/null +++ b/man/xml-source/am389bak.8.xml @@ -0,0 +1,177 @@ + + + + %global_entities; +]> + + + + +am389bak +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +am389bak +Amanda script for online copy of a 389 Directory Server + + +Chapman Flack + + + +DESCRIPTION + +am389bak is an Amanda script implementing the Script API. +It should not be run by users directly. It uses the +db2bak command to make a consistent copy +of a 389 Directory Server instance. + +The name of the directory server instance (the foo +part, if slapd-foo is the directory where its files +are found) is given in the INSTANCE property, and the consistent copy will be +made into the directory named by diskdevice in +the disklist (DLE). This is the directory where the estimate and backup +application ( could be appropriate) will +expect to find the data for backup. + +The copy is made after removing all existing +content under the directory named as the +diskdevice, all of +which happens during PRE-DLE-ESTIMATE, which must be set to be executed +on the client: + + execute-on pre-dle-estimate + execute-where client + + +The script is run as the amanda user, and must be able to run the +db2bak command as the user running 389. This can be done +by setting the DB2BAKEXECUTABLE property not to the path of +db2bak itself, but to a set-uid +and set-gid wrapper that obtains the effective user +and group IDs of the 389 user, copies those to the +real IDs (db2bak is quite finicky), and finally spawns +db2bak with the same arguments. The wrapper should +take all appropriate precautions to avoid being misused, such as checking +the arguments to verify that they match a backup request of an expected +389 instance. + +If the wrapper is made both set-uid and set-gid to the user +that runs 389, there is no way left (in traditional Unix permissions) to +leave the wrapper executable by the Amanda user (except by making it +executable to everyone, which misses the point). However, modern filesystems +typically support access control lists, so the wrapper can be given an ACL +with execute permission granted to the Amanda user specifically. + + +PROPERTIES + +This section lists the properties that control am389bak's +functionality. +See +for information on the Script API, script configuration. + + + + + DB2BAKEXECUTABLE +Path to the db2bak executable, search in $PATH by default. +In realistic settings, this will point instead to a set-uid and set-gid wrapper +that (after appropriate checks) will execute db2bak with +the same arguments. + + + INSTANCE +Name of the 389 Directory Server instance (the foo +part, if slapd-foo is the directory where its files +are found) to be backed up. + + + + + + +EXAMPLE + +This example defines a script make389bak, +using it with an (assumed defined) app_amopaquetree +application (see ) to do +incremental backup of the directory data for instance +myinst. + +In : + + define script "make389bak" { + plugin "am389bak" + execute-where client + execute-on pre-dle-estimate + property "db2bakexecutable" "/path/to/db2bak-wrapper" + } + + +In : + + ldaphost myinst /tmp/389bak { + global + estimate client + program "APPLICATION" + application "app_amopaquetree" + script { + "make389bak" + property "instance" "myinst" + } + } 1 enet100 + + + +BUGS +This script does not (yet) make any effort to detect failures. + +The db2bak utility does so many inconvenient things +that there is extra cleanup work needed when it is done. As that work also needs +to be done as the 389 user, it is most easily done in the set-uid wrapper, +which therefore cannot simply execute db2bak after some +setup, but must execute it in a subprocess, wait for it, and then do further +work. + + +If the directory into which the backup is to be made already exists, +db2bak first renames it to the same name with +.bak tacked on. There is no documented option to turn off +that behavior. Harmless enough, unless that directory +also already exists, in which case db2bak simply fails. +So the wrapper may need to remove any such directory so that situation is +avoided. + + +When db2bak creates files in the destination directory, +it sets their modes explicitly to disallow any group access, which complicates +making the files readable to the Amanda user after the script completes and +the backup application needs to read them. A POSIX default ACL on the +destination directory will not suffice, because even though the files inherit +it, the later explicit denial of group permission masks it off. So the wrapper +may need to change permissions or alter ACLs on the files after +db2bak completes. + + + +The number of tasks that have to be done under an ID other than Amanda's +makes a case for adding some general, secure, easily configured way for Amanda +applications and scripts to run specific operations with privilege, but that +remains future work. + + + +, +, + + + + + diff --git a/man/xml-source/amanda-applications.7.xml b/man/xml-source/amanda-applications.7.xml index 11f085c180..2cd8ec7aef 100644 --- a/man/xml-source/amanda-applications.7.xml +++ b/man/xml-source/amanda-applications.7.xml @@ -37,10 +37,29 @@ +, +- incremental backup of a single large file known to be only appended. + + +, +- incremental backup of a single large ZIP known to only have new members +appended. + + , - use GNU Tar to backup and restore data. +, +- use open and read to read the data. Like , +but in simpler, OO style. + + +, +- use rsync to make increments only from differing regions, not whole changed +files. + + , - use PostgreSQL's continuous WAL archiving. @@ -61,12 +80,13 @@ - use native tar on Solaris to backup and restore data. -, -- use zfs to create a snapshot and use 'zfs send' to generate the backup. +, +- use 'zfs send' to generate a backup as a replication stream of existing +snapshots. -, -- use zfs to create a snapshot and for use with other applications (e.g. amgtar) +, +- use zfs to create a snapshot and use 'zfs send' to generate the backup. diff --git a/man/xml-source/amanda-scripts.7.xml b/man/xml-source/amanda-scripts.7.xml index 071b3f382b..978b0e6172 100644 --- a/man/xml-source/amanda-scripts.7.xml +++ b/man/xml-source/amanda-scripts.7.xml @@ -41,6 +41,22 @@ see http://wiki.zmanda.com/index.php/Script_API. +, +- generate consistent point-in-time copy of a 389 Directory Server instance. + + +, +- freeze/thaw guest VM filesystems so image files are not dirty on host backup. + + +, +- create/destroy and mount/unmount LVM snapshot. + + +, +- generate consistent point-in-time copy of a Subversion repository. + + , - create/destroy zfs snapshot. diff --git a/man/xml-source/amgrowingfile.8.xml b/man/xml-source/amgrowingfile.8.xml new file mode 100644 index 0000000000..e04144caff --- /dev/null +++ b/man/xml-source/amgrowingfile.8.xml @@ -0,0 +1,111 @@ + + + + %global_entities; +]> + + + + +amgrowingfile +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amgrowingfile +Amanda Application for backup of single append-only file + + +Chapman Flack + + + +DESCRIPTION + +Amgrowingfile is an Amanda Application API script. It should not be run +by users directly. It can backup and restore a single file that is known +to change only by appending. + +Such a file may represent audit or logging data, a stream of data +being acquired from instrumentation, etc. Backup levels are supported, where +a backup level greater than zero simply consists of all data written to the +file beyond its length in the prior-level backup. + +For correctness, this strategy requires that the file be known +to grow only at the end. On some operating systems (see the +chattr command in Linux, for example), an +append-only flag can be set on the file to enforce this. +If the file is ever rewritten from the start (such as in log rotation), the +next amgrowingfile backup must be forced to level 0. + + +The diskdevice in the disklist (DLE) +is the filename amgrowingfile will read, unless overridden +by an explicit TARGET property. When restoring, +the TARGET property (which can be set with the +setproperty command in +) will be used, or, if it is not set, +amgrowingfile-restored in the current directory. +In other words, amgrowingfile will not default to +restoring data directly to the original location named by +diskdevice, but the TARGET property +can be set to that name, if that is what's desired. + + +When restoring, the file named with the TARGET property (or +amgrowingfile-restored) is truncated and rewritten +in place if it exists, or created with mode 0600 if it does not. + + +PROPERTIES + +This section lists the properties that control amgrowingfile's +functionality. +See +for information on application properties and how they are configured. + + + + + TARGET +For a restore command, the file name under which the data will +be restored, instead of amgrowingfile-restored. +Otherwise, names the file to be backed up, overriding DEVICE. + + + + + +EXAMPLE + + + define application-tool app_amgrowingfile { + plugin "amgrowingfile" + } + +A dumptype using this application might look like: + + define dumptype amgrowingfile { + global + program "APPLICATION" + application "app_amgrowingfile" + } + +Note that the program parameter must be set to +"APPLICATION" to use the application +parameter. + + + + +, + + + + + diff --git a/man/xml-source/amgrowingzip.8.xml b/man/xml-source/amgrowingzip.8.xml new file mode 100644 index 0000000000..5427dbdfab --- /dev/null +++ b/man/xml-source/amgrowingzip.8.xml @@ -0,0 +1,159 @@ + + + + %global_entities; +]> + + + + +amgrowingzip +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amgrowingzip +Amanda Application for backup of append-only ZIP archive + + +Chapman Flack + + + +DESCRIPTION + +Amgrowingzip is an Amanda Application API script. It should not be run +by users directly. It can backup and restore a single file that is +a ZIP archive known to change only by appending members. + +Such a file may be used to collect many small files that are created +over time, such as data acquired from instruments, write-ahead logs of some +databases, etc. Where many small files in a directory could use disk blocks +inefficiently, appending them, as they are generated, to a growing ZIP archive +can achieve efficient storage along with the ability to extract individual +files from the ZIP as needed. + +Amgrowingzip itself only deals with the ZIP archive as a unit. +When restoring, the entire archive will be restored. Any extraction of +individual members from the archive can then be done with ordinary ZIP +tools. The growing ZIP file can be large, and amgrowingzip supports backup +levels. + +For correctness, this application requires that the ZIP archive +be known to grow only by new members being appended to it. This is analogous to +an append-only operation at the file level, but not the same, so it cannot +be enforced by typical append-only flags offered by +the operating system. The structure of a ZIP archive consists of members of +the archive (typically compressed), followed by a directory structure +at the end. Appending a member involves seeking to the end of the file, +backspacing to the start of the directory structure, and overwriting with the +new member(s) followed by the new directory structure. For amgrowingzip, +a backup level greater than zero is simply everything that must be written +over the file starting at the directory structure offset in the prior-level +backup. + +Any time the ZIP archive is rebuilt rather than simply appended, +the next amgrowingzip backup must be forced to level 0. +Not all ZIP-related tools truly append a member as described here; +some perform any modification of the archive by rebuilding it from scratch. +To be usable with amgrowingzip, whatever process adds members to the archive +must be known to truly append them. The zipfile module in +Python offers a suitable append mode, for example. + + +If it is possible for the process that appends members to be +active while amgrowingzip is taking a backup, the FLOCK +property should be set, and the appending process should also be coded to +hold an advisory exclusive lock on the archive file during each modification. +Otherwise, a failure or unusable backup could result from trying to locate +the archive's directory while a write is in progress. + + +The diskdevice in the disklist (DLE) +names the ZIP archive amgrowingzip will read, unless +overridden by a TARGET property given explicitly. When restoring, +the TARGET property (which can be set with the +setproperty command in +) will be used, or, if it is not set, +amgrowingzip-restored in the current directory. +In other words, amgrowingzip will not default to +restoring data directly to the original location named by +diskdevice, but the TARGET property +can be set to that name, if that is what's desired. + + +When restoring, the file named with the TARGET property (or +amgrowingzip-restored) is truncated and rewritten +in place if it exists, or created with mode 0600 if it does not. + +No locking is used while restoring, regardless of the FLOCK property. +There is no sense in allowing any writes to the file being restored before +the last increment needed has been applied to it. The best approach is to +restore under a temporary name and, only after restoration, move the file +to the expected place. + + +PROPERTIES + +This section lists the properties that control amgrowingfile's +functionality. +See +for information on application properties and how they are configured. +For the recognized true/false +values for a boolean property, see . + + + + + + FLOCK +Set to a true value to require an advisory shared lock +before backing up the file, in case the process that appends members to it +could be simultaneously active. The appending process must also be coded to take +an advisory exclusive lock while writing. If this property is given a +false value, no locking is used. + + + + TARGET +For a restore command, the file name under which the data will +be restored, instead of amgrowingzip-restored. +Otherwise, names the file to be backed up, overriding DEVICE. + + + + + +EXAMPLE + + + define application-tool app_amgrowingzip { + plugin "amgrowingzip" + } + +A dumptype using this application might look like: + + define dumptype amgrowingzip { + global + program "APPLICATION" + application "app_amgrowingzip" + } + +Note that the program parameter must be set to +"APPLICATION" to use the application +parameter. + + + + +, + + + + + diff --git a/man/xml-source/amlibvirtfsfreeze.8.xml b/man/xml-source/amlibvirtfsfreeze.8.xml new file mode 100644 index 0000000000..a7c5907845 --- /dev/null +++ b/man/xml-source/amlibvirtfsfreeze.8.xml @@ -0,0 +1,290 @@ + + + + %global_entities; +]> + + + + +amlibvirtfsfreeze +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amlibvirtfsfreeze +Amanda script to freeze/thaw filesystems in VMs + + +Chapman Flack + + + +DESCRIPTION + +amlibvirtfsfreeze is an Amanda script implementing the Script API. +It should not be run by users directly. On a machine that hosts +virtual machines, it uses libvirt to instruct +a virtual machine to freeze one or more filesystems, and then to thaw +them after their image files on the host have been backed up. + +Freeze and thaw support in libvirt requires that a "guest agent" +(for example, for a Linux guest) be installed +in the guest OS. + +When a filesystem is frozen, any buffered changes are forced to storage +so the image can be backed up while consistent at the filesystem level. +Activity in the guest that modifies the filesystem may block until the +filesystem is thawed, so this technique is best combined with a script (such as +) to snapshot the host filesystem, +so the VM guest filesystems need only be frozen long enough to create +the snapshot on the host. + +While a frozen filesystem will not be 'dirty' at the filesystem level, +freezing alone does not ensure that any files in that filesystem represent +consistent states of running applications. A guest agent may provide the +option to run a freeze-hook script in the guest OS just before freezing and +after thawing, and such a script can take application-specific actions to +ensure important applications have reached a consistent state and forced it +to storage as well. More on freeze hooks will be found in the documentation +for the guest agent being used. + +The guest virtual machine ("domain" in libvirt parlance) is specified +by name in the DOMAIN property, and the MOUNTPOINT property (which can be +multivalued) specifies which of that domain's filesystems to act on. +The FREEZEORTHAW property specifies the action. + + +For a thaw action, no MOUNTPOINT is specified, and all filesystems +frozen in the guest domain are thawed. For a freeze action, MOUNTPOINT may +need to be omitted if the virtualization software or guest OS are too old +to support per-filesystem freezing, and in that case, every filesystem mounted +on the guest is frozen. + +This script can be run in PRE-DLE-ESTIMATE, in the case where one +snapshot will be made before estimating and removed after backup. If limited +free space in the volume group forces a small snapshot size, or changes to the +origin filesystem accumulate quickly, or the Amanda installation in total has +a long delay in planning/dumping between the estimate and backup phases, it is +possible to execute this script twice, on PRE-DLE-ESTIMATE and PRE-DLE-BACKUP, +in conjunction with a snapshot script that removes the snapshot used for +estimating and creates another one for backup. In that case, it would also +be possible to run this script only for PRE-DLE-BACKUP, as the estimate phase +can get the size of the image file accurately enough without needing it +frozen. + +The phase (or both phases) must be set to be executed on the client: + + execute-on pre-dle-estimate + #execute-on pre-dle-estimate, pre-dle-backup + execute-where client + + +The script is run as the amanda user, and must be able to run the +necessary virsh command. One approach is to set the +VIRSHEXECUTABLE property not to the actual path of the command, but to +a set-uid wrapper that takes the same arguments, checks that they represent +allowed operations, domains, and filesystems, and then executes the actual +command with the same arguments. + +A set-uid wrapper obtains its owner's effective +ID, but may need to set the real ID to match it before +the virsh command will permit the operations. + + +Trimming before freezing + +Another operation libvirt can request through the guest agent is to +trim filesystems (any specified with MOUNTPOINT, or all +filesystems if no mountpoint is given). The trim operation identifies all +blocks regarded as free by the filesystem, and reports them to the filesystem's +underlying block device as reclaimable. If the block device in the guest +presents an image file from the host, this may reduce the space taken by +the image on the host, by returning blocks to the host OS. Regular trimming +can counteract the steady growth of image files due to file creation and +deletion in the guest, especially if the guest filesystem is not mounted with +a discard option to report freed blocks to the device layer as they +are freed. A simple way to do regular trimming is to configure a DLE to trim +with this script, just before freezing the filesystem for regular backups, as +shown in the example below. + +Whether trimming is possible, or has any effect, depends on support +in the virtualization system, guest OS and filesystem, and the image file +format used on the host. For Linux guests and qemu-kvm, an image file must +be presented via the virtio-scsi driver, not by the earlier +virtio-blk. + + + +PROPERTIES + +This section lists the properties that control amlibvirtfsfreeze's +functionality. +See +for information on the Script API, script configuration. + + + + + DOMAIN +The name of the libvirt domain, or guest VM, whose filesystems should be +frozen or thawed. + + + FREEZEORTHAW +One of the values freeze or thaw. +A third value, trim, is also allowed, to request trimming +filesystems as described in . + + + MOUNTPOINT +Filesystems in the named domain to be frozen or trimmed, identified by their +mountpoints. This property can have multiple values. It is ignored in a +thaw operation; all frozen filesystems will be thawed. +For some old versions of virtualization infrastructure, the guest agent, or +guest OS, it must not be used for freeze or +trim either, in which case all mounted filesystems will be +acted on. + + + VIRSHEXECUTABLE +Path to the virsh executable, search in $PATH by default. +If necessary for authorization, this can be set +instead to a set-uid wrapper that (after appropriate checks) will +execute virsh with the same arguments. + + + + + + +EXAMPLE + +This example sets up three script definitions, +domfstrim, domfsfreeze, +and domfsthaw, all based on +this plugin, and a DLE that uses them, along with + and an assumed +remote-gtar dumptype, to first trim, and then freeze, +the root filesystems of several guest VMs running on a host, then snapshot +the host's filesystem (containing the consistent, frozen image files of the +guests), thaw the guest filesystems, and finally back up the host from the +snapshot. + +Note the use of the ORDER option in script definitions. The default +(applied to the lvmsnapshot script, which leaves it +unspecified) is 5000. The explicit 3000 in the domfstrim +definition, 4000 for domfsfreeze, and 6000 for +domfsthaw ensure that, +when all four scripts are run in a single phase such as PRE-DLE-ESTIMATE, +the trims occur first, then freezes, then the snapshot, then the thaws. + +In a realistic application where amandad does not +run as the superuser, a privileged wrapper may need to be written and named +in the VIRSHEXECUTABLE property as discussed above. + +In : + + define script domfstrim { + plugin "amlibvirtfsfreeze" + execute-where client + execute-on pre-dle-estimate + property "freezeorthaw" "trim" + property "virshexecutable" "/path/to/virsh-wrapper" + order 3000 # must execute before libvirtfsfreeze 'freeze' + } + + define script domfsfreeze { + plugin "amlibvirtfsfreeze" + execute-where client + execute-on pre-dle-estimate + property "freezeorthaw" "freeze" + property "virshexecutable" "/path/to/virsh-wrapper" + order 4000 # must execute before lvmsnapshot + } + + define script domfsthaw { + plugin "amlibvirtfsfreeze" + execute-where client + execute-on pre-dle-estimate + property "freezeorthaw" "thaw" + property "virshexecutable" "/path/to/virsh-wrapper" + order 6000 # must execute after lvmsnapshot + } + + define script "lvmsnapshot" { + plugin "amlvmsnapshot" + execute-where client + execute-on pre-dle-estimate, post-dle-backup + # ... see amlvmsnapshot.8 + } + + define dumptype remote-gtar # ... + + +And in : + + example / /tmp/rootsnap { + remote-gtar + script { + "domfstrim" + property "domain" "vm1" + property "mountpoint" "/" + } + script { + "domfsfreeze" + property "domain" "vm1" + property "mountpoint" "/" + } + script { + "domfsthaw" + property "domain" "vm1" + } + script { + "domfstrim" + property "domain" "vm2" + property "mountpoint" "/" + } + script { + "domfsfreeze" + property "domain" "vm2" + property "mountpoint" "/" + } + script { + "domfsthaw" + property "domain" "vm2" + } + script { + "lvmsnapshot" + property "volumegroup" "vg_example" + property "logicalvolume" "lv_root" + property "snapshotname" "snap_root" + property "extents" "10%ORIGIN" + } + } 1 enet100 + + + +BUGS +This script does not (yet) make any effort to detect failures, or to +capture and interpret standard output from the virsh, which +normally produces some. Uncaptured, such output mixes with Amanda's +client/server messaging, leading to backup failures. A cheap workaround is to +include dup2(2,1) in the wrapper that executes +virsh, so such output goes to standard error and ends up +in the client amandad log file. + + + +, +, + + + + + diff --git a/man/xml-source/amlvmsnapshot.8.xml b/man/xml-source/amlvmsnapshot.8.xml new file mode 100644 index 0000000000..bca966f738 --- /dev/null +++ b/man/xml-source/amlvmsnapshot.8.xml @@ -0,0 +1,256 @@ + + + + %global_entities; +]> + + + + +amlvmsnapshot +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amlvmsnapshot +Amanda script to make an LVM snapshot + + +Chapman Flack + + + +DESCRIPTION + +amlvmsnapshot is an Amanda script implementing the Script API. +It should not be run by users directly. It creates and mounts a +snapshot of an LVM filesystem in preparation for backup, then unmounts +and removes it. + +The logical volume to be snapshotted is specified with the VOLUMEGROUP +and LOGICALVOLUME properties, and the SNAPSHOTNAME property will be used to +give the created snapshot a name. The EXTENTS property must be set to indicate +the size of the snapshot. The mount point for the snapshot will be the +directory named by diskdevice in the +disklist (DLE). This is the directory where the estimate and backup application +will expect to find the data for backup. + +A snapshot begins with nearly zero space consumed, but grows (up to +the specified size) as the origin filesystem accumulates changes made since +the snapshot. The EXTENTS property must size the snapshot generously enough +for the volume of changes expected in the origin filesystem during the +snapshot's lifetime, while fitting in the volume group's available, unused +space. + +This script can be run in PRE-DLE-ESTIMATE and POST-DLE-BACKUP, in which +case one snapshot will be made and mounted before estimating, and unmounted and +removed after backup. If limited free space in the volume group forces a small +snapshot size, or changes to the origin filesystem accumulate quickly, or the +Amanda installation in total has a long delay in planning/dumping between the +estimate and backup phases, it may be safer to execute this script four times, +on PRE-DLE-ESTIMATE, POST-DLE-ESTIMATE, PRE-DLE-BACKUP, and POST-DLE-BACKUP. In +that case, the snapshot used for estimating is unmounted and removed after that +phase, and a new snapshot is created later for actual dumping. + +The snapshot is made and mounted after removing all existing +content under the directory named as the +diskdevice if it exists, or +creating it empty if it does not, which happens +during PRE-DLE-ESTIMATE (and PRE-DLE-BACKUP if also executed then). +Both (or all four) phases must be set to be executed on the client: + + execute-on pre-dle-estimate, post-dle-backup + #execute-on pre-dle-estimate, post-dle-estimate, pre-dle-backup, post-dle-backup + execute-where client + + +The script is run as the amanda user, and must be able to run the +necessary lvm, mount, and +umount commands. One approach is to set the +LVMEXECUTABLE, MOUNTEXECUTABLE, and UMOUNTEXECUTABLE not to the actual +paths of those commands, but to set-uid wrappers that take the same +arguments, check that they represent allowed operations and filesystems, +and then execute the actual commands with the same arguments. + +A set-uid wrapper obtains its owner's effective +ID, but may need to set the real ID to match it before +the lvm, mount, and +umount commands will permit the operations. + + +PROPERTIES + +This section lists the properties that control amlvmsnapshot's +functionality. +See +for information on the Script API, script configuration. + + + + + EXTENTS +The size to make the created snapshot. Syntaxes like +10%ORIGIN or 50%FREE +are possible; see . +There must be at least this much unused space available in +the same volume group as the origin filesystem, and this size +must be large enough for all the expected modification activity +for that filesystem during the snapshot's existence. + + + LOGICALVOLUME +The name of the logical volume that will be the origin of the snapshot. + + + LVMEXECUTABLE +Path to the lvm executable, search in $PATH by default. +If necessary for authorization, this can be set +instead to a set-uid wrapper that (after appropriate checks) will +execute lvm with the same arguments. + + + MOUNTEXECUTABLE +Path to the mount executable, search in $PATH by default. +If necessary for authorization, this can be set +instead to a set-uid wrapper that (after appropriate checks) will +execute mount with the same arguments. + + + MOUNTOPTS +Mount options to be used when mounting the snapshot. Multiple values +are accepted. If the filesystem is XFS, two mount options are necessary: + + property "mountopts" "nouuid" "norecovery" + + + + SNAPSHOTNAME +The name to be given to the newly-created snapshot. + + + UMOUNTEXECUTABLE +Path to the umount executable, search in $PATH by default. +If necessary for authorization, this can be set +instead to a set-uid wrapper that (after appropriate checks) will +execute mount with the same arguments. + + + UNMOUNTSWHENFILLED +Whether the OS kernel in use is known to unmount a mounted snapshot +automatically if its maximum allocation is reached. This affects the script's +treatment of the race condition between unmounting the snapshot post-backup +and checking whether its allocation reached 100%. A true setting allows the +backup to be considered successful if the unmount succeeded, even if the +allocation is then reported as 100% when checked. A false setting (the default) +forces a failure result any time the snapshot allocation is seen to reach +100%. This property should be set true only if the OS version in use has been +tested and definitely shown to automatically unmount snapshots that fill. + + + VOLUMEGROUP +Name of the volume group in which the logical volume to be snapshotted, +and in which the snapshot will be created. + + + + + + +EXAMPLE + +This example defines a script lvmsnapshot and a DLE +that backs up a logical volume lv_root in volume group +vg_example (assuming there is already a +remote-gtar dumptype defined. A single snapshot will be +used for both the estimate and backup phases. While in use, the snapshot +will be mounted at /tmp/rootsnap, which will be +created empty if absent, or emptied if present. + +A dumptype remote-gtar-root-tinysnap dumptype is also +created, setting the same VOLUMEGROUP, LOGICALVOLUME, SNAPSHOTNAME, and +EXTENTS properties to avoid repeating them in DLEs for several +identically-configured clients, and also overriding EXECUTE-ON to use +a shorter-lived, smaller snapshot for each phase, +in case a client may have insufficient free space in the volume group for +all of the modification activity during the longer life of a single +snapshot. + +In a realistic application where amandad does not +run as the superuser, privileged wrappers may need to be written and named +in the LVMEXECUTABLE, MOUNTEXECUTABLE, and UMOUNTEXECUTABLE properties as +discussed above. + +In : + + define script "lvmsnapshot" { + plugin "amlvmsnapshot" + execute-where client + execute-on pre-dle-estimate, post-dle-backup + property "lvmexecutable" "/path/to/lvm-wrapper" + property "mountexecutable" "/path/to/mount-wrapper" + property "umountexecutable" "/path/to/umount-wrapper" + } + + define dumptype remote-gtar-root-tinysnap { + remote-gtar + script { + "lvmsnapshot" + property "volumegroup" "vg_example" + property "logicalvolume" "lv_root" + property "snapshotname" "snap_root" + property "extents" "2%ORIGIN" + execute-on pre-dle-estimate, post-dle-estimate, pre-dle-backup, post-dle-backup + } + } + + +And in : + + example / /tmp/rootsnap { + remote-gtar + script { + "lvmsnapshot" + property "volumegroup" "vg_example" + property "logicalvolume" "lv_root" + property "snapshotname" "snap_root" + property "extents" "10%ORIGIN" + } + } 1 enet100 + + tiny / /tmp/rootsnap remote-gtar-root-tinysnap -1 enet100 + + + +BUGS +This script does not (yet) capture and interpret most standard output +from the lvm, which normally produces some. Uncaptured, such +output mixes with Amanda's client/server messaging, leading to backup failures. +A cheap workaround is to include dup2(2,1) in the wrapper that +executes lvm, so such output goes to standard error and +ends up in the client amandad log file. The wrapper should +avoid that dup2 if the lvm subcommand being +executed is lvs, because the output in that case is captured +and used. + +Also, this script does not (yet) do anything to prevent +lvm inheriting file descriptors other than standard +input/output/error that are used in Amanda's script API. That leads to +benign warnings from lvm about file descriptor leaks. +This also has a cheap workaround that can be added to the wrapper, namely +to define LVM_SUPPRESS_FD_WARNINGS in the environment +before executing lvm. + + + +, +, + + + + + diff --git a/man/xml-source/amooraw.8.xml b/man/xml-source/amooraw.8.xml new file mode 100644 index 0000000000..cc934de119 --- /dev/null +++ b/man/xml-source/amooraw.8.xml @@ -0,0 +1,104 @@ + + + + %global_entities; +]> + + + + +amooraw +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amooraw +Amanda Application open and read data + + +Chapman Flack + + + +DESCRIPTION + +Amooraw is an Amanda Application API script. It should not be run +by users directly. It directly reads and writes data in a single directory +entry. + +Amooraw can backup only one directory entry, it can be a single file, +a raw device, anything that amanda can open and read. + +It is a reimplementation of illustrating +the simpler construction of an Amanda application in object-oriented style +(the oo in amooraw) using +Amanda::Application::Abstract. + +When restoring, +if the TARGET property is not specified, restoration will create +a file amooraw-restored instead of blithely overwriting +the original diskdevice named in the DLE. +If direct overwriting is what's wanted, simply set the TARGET property to match +the diskdevice. + +The diskdevice in the disklist (DLE) +must be the filename amooraw is to open and read. + +Restore is done in place: an open is done and the data is written to it. +A file owned by root and permission 0600 is created if the directory entry +doesn't exist before the restore. + +Only full backup is allowed. + + +PROPERTIES + +This section lists the properties that control amooraw's functionality. +See +for information on application properties and how they are configured. + + + + + TARGET +For a restore command, can be a device name or file, the data will +be restored to it, instead of to amooraw-restored. +Otherwise, names the file to be backed up, overriding DEVICE. + + + + + +EXAMPLE + + + define application-tool app_amooraw { + plugin "amooraw" + } + +A dumptype using this application might look like: + + define dumptype amooraw { + global + program "APPLICATION" + application "app_amooraw" + } + +Note that the program parameter must be set to +"APPLICATION" to use the application +parameter. + + + + +, + + + + + diff --git a/man/xml-source/amopaquetree.8.xml b/man/xml-source/amopaquetree.8.xml new file mode 100644 index 0000000000..90c6c15159 --- /dev/null +++ b/man/xml-source/amopaquetree.8.xml @@ -0,0 +1,177 @@ + + + + %global_entities; +]> + + + + +amopaquetree +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amopaquetree +Amanda Application for backup of a directory tree opaquely + + +Chapman Flack + + + +DESCRIPTION + +Amopaquetree is an Amanda Application API script. It should not be run +by users directly. It can backup and restore a directory tree opaquely +(that is, as a whole, with no option to restore just some entries), +with efficient increments when large files have only small changes. + +Most Amanda applications that backup a directory tree allow selective +restoration of only some files or subtrees. Also, the typical applications +that support backup levels, such as amgtar, operate at +the file level: changes, however small, to any file cause the entire file +to be included in the incremental backup. These properties are suitable for +many backup needs. + +Some applications, such as databases and products that incorporate them, +may keep files in a directory structure with obscure naming or numbering, +where an administrator would rarely be expected to identify or restore selected +files, but simply to restore the tree at a consistent moment. For such +applications, the individual files can also be large, and modified only in +small regions, so that backing up entire changed files would be wasteful. + +Amopaquetree is intended for those cases. +A directory-list entry (DLE) backed up with amopaquetree can only be +restored in full, without selective access to subcomponents. Incremental +backups will identify only changed regions within files, rather than +backing up entire files when changed, using more storage and computation +on the client, but saving network bandwidth, holding disk, and tape space. + + +The extra storage required on the client is similar to the size of +the tree to be backed up. Other Amanda applications supporting incremental +backup store only a small client-local state, such as a date or a file list, +for each previous backup level. For amopaquetree, the +client-local state will include one complete copy of the tree for the lowest +level backup, plus a tree for each higher level containing only changed files, +with links to the lower level for unchanged files. Therefore, +on the client, storage is required for entire files +that have been changed, but this storage is accessed at local speeds. +From those copies, the changed regions are efficiently found, and only they +are streamed across the network to store on the server. + +Most databases and similar applications will have specific steps +that must be followed to guarantee the files are consistent before backing +them up. Those steps may take the form of a command to create a consistent +copy (svnadmin hotcopy for subversion, +db2bak for 389, etc.), or to enter and +exit a mode during which a copy may safely be made +(pg_start_backup/pg_stop_backup for +PostgreSQL, etc.). + +For cases of the first type, that create a consistent copy, an +Amanda script to run the necessary command ahead of +amopaquetree may be all that is needed. The client should +have temporary space for that consistent copy, as well as the local state space +needed by amopaquetree itself. For cases of the +second type, entering and exiting a safe-copying mode (which may require +additional steps such as copying a write-ahead log), it may be better to +create a specialized Amanda application by subclassing +Amanda::Application::AmOpaqueTree. + + +IMPLEMENTATION + +amopaquetree uses (and, therefore, requires) +the widely-available rsync tool, both in its + mode to maintain client-local state +limited in size to one full copy plus changed files in higher levels, +and in its mode to generate the +smaller data streams, reflecting changed regions only, to be sent to +the server. + + + +PROPERTIES + +This section lists the properties that control amopaquetree's +functionality. +See +for information on application properties and how they are configured. + + + + + + LOCALSTATEDIR +Amanda has a default location for client local state, but because +amopaquetree stores a larger-than-typical local state, +this property can be used to place the local state storage on a different +filesystem. Or, this property can be left at default, and RSYNCSTATESDIR +(which defaults to a subdirectory of LOCALSTATEDIR) can be set, to use a +different filesystem only for that. + + + + RSYNCEXECUTABLE +Full path to the rsync executable on the client system. +If elevated permission will be needed for rsync to read +the tree being backed up, this can point instead to a wrapper that gains +the needed privilege and then executes rsync. + + + + RSYNCSTATESDIR +The directory (defaulting to a subdirectory of LOCALSTATEDIR) for the +local copied/linked images of the tree backed up. Because this accounts +for the greatest share of local state space, it can be separately +redirected to another filesystem using this property. + + + + RSYNCTEMPBATCHDIR +The directory where rsync batch files will be +temporarily created. Temporary space must be available for the +expected size of a batch; for a level-0 backup, that will be +comparable to the size of a compressed archive of the tree being backed up. +The default is where Perl's File::Temp will put +a temporary file. + + + + + +EXAMPLE + + + define application-tool app_amopaquetree { + plugin "amopaquetree" + } + +A dumptype using this application might look like: + + define dumptype amopaquetree { + global + program "APPLICATION" + application "app_amopaquetree" + } + +Note that the program parameter must be set to +"APPLICATION" to use the application +parameter. + + + + +, + + + + + diff --git a/man/xml-source/amsvnmakehotcopy.8.xml b/man/xml-source/amsvnmakehotcopy.8.xml new file mode 100644 index 0000000000..271354cc99 --- /dev/null +++ b/man/xml-source/amsvnmakehotcopy.8.xml @@ -0,0 +1,145 @@ + + + + %global_entities; +]> + + + + +amsvnmakehotcopy +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amsvnmakehotcopy +Amanda script to make a Subversion hotcopy + + +Chapman Flack + + + +DESCRIPTION + +amsvnmakehotcopy is an Amanda script implementing the Script API. +It should not be run by users directly. It uses the +svnadmin hotcopy command to make a consistent copy +of a Subversion repository. + +The filesystem path to the Subversion repository to copy is given +in the SVNREPOSITORY property, and the consistent copy will be made into +the directory named by diskdevice in the +disklist (DLE). This is the directory where the estimate and backup application +( could be appropriate) will expect to +find the data for backup. + +The copy is made after removing all existing +content under the directory named as the +diskdevice, all of +which happens during PRE-DLE-ESTIMATE, which must be set to be executed +on the client: + + execute-on pre-dle-estimate + execute-where client + + +The script is run as the amanda user, and must be able to read the +Subversion repository. Possible arrangements include making the repository +generally readable, giving it filesystem ACLs granting the access specifically +to the amanda user, or setting the SVNADMINEXECUTABLE property not to the +path of svnadmin, but to a set-uid wrapper that obtains +the needed permission and then executes svnadmin with +the same arguments. If the wrapper approach is chosen, the wrapper should +take all appropriate precautions to avoid being misused, such as checking +the arguments to verify that they only request a hotcopy of +the intended Subversion repository. + + +PROPERTIES + +This section lists the properties that control amsvnmakehotcopy's +functionality. +See +for information on the Script API, script configuration. + + + + + CLEAN-LOGS +True if svnadmin should remove unused Berkeley DB logs +from the source repository after completing the hotcopy. + + + INCREMENTAL +True if svnadmin should perform an incremental hotcopy +(supported for FSFS repositories in Subversion 1.8 or later). + + + SVNADMINEXECUTABLE +Path to the svnadmin executable, search in $PATH by default. +If necessary to gain read access to the repository, this can be set +instead to a set-uid wrapper that (after appropriate checks) will +execute svnadmin with the same arguments. + + + SVNREPOSITORY +Filesystem path to the Subversion repository that is to be copied. + + + + + + +EXAMPLE + +This example defines a script svnmakehotcopy, +a dumptype remote-svn, and a DLE to backup a particular +repository. + +In : + + define script "svnmakehotcopy" { + plugin "amsvnmakehotcopy" + execute-where client + execute-on pre-dle-estimate + } + + define dumptype remote-svn { + global + estimate client + program "APPLICATION" + application "app_amopaquetree" + comment "backup of remote svn repository" + } + + +In : + + svnhost myrepo /tmp/svnmyrepocopy { + remote-svn + script { + "svnmakehotcopy" + property "svnrepository" "/var/www/svn/repos/myrepo" + } + } 1 enet100 + + + +BUGS +This script does not (yet) make any effort to detect failures. + + + +, +, + + + + + diff --git a/man/xml-source/amzfs-holdsend.8.xml b/man/xml-source/amzfs-holdsend.8.xml new file mode 100644 index 0000000000..30ea8514ec --- /dev/null +++ b/man/xml-source/amzfs-holdsend.8.xml @@ -0,0 +1,430 @@ + + + + %global_entities; +]> + + + + +amzfs-holdsend +8 +&rmi.source; +&rmi.version; +&rmi.manual.8; + + +amzfs-holdsend +Amanda Application for backup of +ZFS externally-created snapshots + + +Chapman Flack + + + +DESCRIPTION + +Amzfs-holdsend is an Amanda Application API script. It should not be run +by users directly. It can backup and restore ZFS filesystems opaquely +(that is, as a whole, with no option to restore just some entries), +using zfs send replication streams +based on snapshots created by other means. + +ZFS backup approaches compared + +ZFS is a sophisticated filesystem manager that lends itself to several +backup approaches depending on need (and different approaches can be used +on different datasets in the same pool). For two other approaches available +in Amanda, see and +. + + +amzfs-snapshot + +A script that can be combined with a conventional file-based archiving +application, such as , when the ability +to restore individual files will be important. The script creates a new +ZFS snapshot, the archiver copies files from the snapshot in the usual +way (but without the inefficient side effect of clobbering access times, +because the snapshot is read-only), and then the script destroys the snapshot. + +Individual files can be recovered in the usual way, though how much +advanced metadata gets preserved will depend on what +the archiving tool supports. Strictly ZFS-specific features "outside" +the filesystem, such as historical snapshots and filesystem properties, +are not preserved. + +amzfs-sendrecv + +An application that uses zfs send to faithfully +preserve the contents of a single ZFS filesystem at a single point +in time. It creates a new snapshot as of the time it is executed, +and can leave the snapshot in place when done so that a future run +can be incremental. An incremental run advances the filesystem contents +to the just-created new snapshot, but does not preserve intermediate +snapshots along the way. + +amzfs-holdsend + +This application: uses the replication +stream feature of zfs send to preserve +a ZFS filesystem or volume and any descendants. This application does +not create or destroy snapshots on its own, but selects recent snapshots +assumed to be created by other means, such as a scheduler that may take +snapshots at regular intervals. Existing historical snapshots are preserved +in the backup. + + + + +In general, approaches that use a conventional archiver have the advantage +that individual files are easily restored when needed, and (typically, at +least) a few disadvantages: they may not preserve all advanced metadata +(ACLs, security contexts, etc.) supported by the filesystem, identification +of changed files for incremental backup may require traversing the whole +filesystem making fallible comparisons, and increments may grow larger than +necessary by including entire files that have seen but small changes. + + + +This application, as well as amzfs-sendrecv, take +the complementary approach. While they do not provide +for easy recovery of individual files, they faithfully preserve all of the +filesystems' contents and metadata, identify all changed objects in an +increment without relying on fallible checks, and require only the size +of changed data, even with small changes to large files. + + + +In an environment where nothing else is creating snapshots, and there is no +desire to preserve anything but the most recent state at each backup, +amzfs-sendrecv may be a simple solution, as it takes care +of creating its own snapshot on each backup run. + + + +If snapshots are being created by other means, such as on a frequent schedule +between backups, or if intermediate snapshots between scheduled backups should +all be preserved, amzfs-holdsend fits the bill. It does not +add to or delete from the existing snapshots, but preserves them as they are. + + + +Where supported by the ZFS implementation in use, backup size and +CPU utilization can both be reduced for compressed datasets by setting +the UNCOMPRESSED property to false. + + + + +Multiple datasets in one DLE + + +ZFS arranges datasets (its generic term for both filesystems and +block-device-like volumes) in a tree. The tree structure does not have +to mirror the desired layout of filesystem mount points, but can be used +to group datasets meant to be similar in other ways, compression settings +for example, or datasets to be backed up together. + + + +For amzfs-holdsend, the disk or +device given in a disk-list entry names a subtree of +datasets to back up. It must always be a ZFS-style dataset name, not a +mountpoint, to make clear that it points into the ZFS namespace hierarchy +rather than the filesystem namespace, and the backup will include that dataset +and any others descended from it in the ZFS tree. + + + +For a subtree to be backed up at once by amzfs-holdsend, +certain conditions must hold. For a full (level 0) backup, there must be at +least one snapshot name that is present on every dataset in the subtree. An easy +way to ensure that is to use zfs snapshot -r ... to create +snapshots with identical names on a dataset and all its descendants in one +command. However, the stream generation does not depend on that: there is no +requirement that the same-named snapshots were created together, or even +closely in time; they simply have to exist. If amzfs-holdsend +will not back up a certain subtree because no common snapshot name can be found, +if there is a snapshot name that is present in most of the datasets, the remedy +can be as simple as creating snapshots by that name in the datasets lacking it. +When amzfs-holdsend can identify one or more snapshot names +present in the named dataset and all descendants, it will use the latest of +those for a level-0 backup. + + + +You could be sly and create some snapshots with matching names +everywhere in the subtree, but not always in the same order. When +amzfs-holdsend picks the "latest" of them, it uses the +one that is latest in the topmost dataset, the one that the DLE names. + + + +For an incremental backup to be produced, the snapshots +used for the next lower backup level must still be present, and must still all +have the same name. (If any has been renamed, so must they all.) For a +useful backup, there must also be some newer snapshots with their own common +name throughout the subtree; again, the "latest" will be used as the new end +point for the incremental backup. + + + +You could be sly again and arrange for a later snapshot in some +datasets to match the name of a snapshot that came earlier than the +prior-level backup snapshot in some other datasets in the same subtree. +You won't get away with it, however. + + + +If no new snapshot has been created since the last backup, a (trivial) +incremental backup can still be created, from the last-used snapshot +to itself. There are warnings from zfs send in that +case, but it successfully produces a restorable increment with no +filesystem-content changes. It will include the latest values of any +changed ZFS properties. + + + + +Holds on snapshots + + +After a backup, amzfs-holdsend places +zfs hold tags on the snapshots that must be +kept around for future increments. The tags have a recognizable prefix +followed by the backup level, and serve two purposes. For this application, +they record which backup levels used which snapshots; for ZFS itself, they +protect the snapshots from inadvertent deletion. + + + +So that a dataset or subtree can belong to more than one DLE (to back up +to more than one media series, or onsite/offsite, etc.), HOLDTAGPREFIX +is a property that can be set in a DLE or dumptype. With the different +DLEs using different prefixes, the hold tags they generate can coexist. + + + +As amzfs-holdsend releases some holds and places others +when a backup completes, there can be a moment when it has no holds on any +of the snapshots needed for future increments. That should not be a problem, +unless many snapshots have been queued up for destruction with +zfs destroy -d, in which case they could all vanish +before a new hold can be placed, making that a risky practice. + + + +When attempts to destroy a filesystem or snapshot are met with +"snapshot is busy" complaints from ZFS, the holds are doing their job. +When destroying the filesystem or snapshot really is what you want to do, +manually-issued zfs release commands (after sober +reflection) can remove the conflicting holds. + + + + +Permissions needed + + +Because amzfs-holdsend only needs to create and release +holds and invoke zfs send, it will work with just those +three ZFS permissions delegated to the Amanda backup user: + + + + zfs allow -u amandabackup hold,send,release filesystem + + + +Additional permissions are needed for recovery, but amrecover +is typically run as superuser, so there is no need to grant more permissions +to the backup user. + + + +On a platform that lacks zfs allow support, it may be +necessary to use pfexec, sudo, or a +custom setuid wrapper for the zfs command. + + + + +Estimate method + +The estimate methods CLIENT and CALCSIZE are supported. CALCSIZE is both +fast and accurate, but works only on platforms that support +zfs send -nvP as found in OpenZFS. CLIENT is slower, +but works on platforms without that support. + + + + + +PROPERTIES + +This section lists the properties that control amzfs-holdsend's +functionality. +See +for information on application properties and how they are configured. + + + + + + DEDUP +Pass the option to zfs send, +for platforms that support it to exclude duplicated blocks from the +generated stream. See for details. + + + + EMBED +Pass the option to zfs send, +for platforms that support it to generate a smaller stream from a pool that +uses the embedded_data feature. +See for details. + + + + HOLDTAGPREFIX +All hold tags placed by this application on snapshots will consist of +this prefix, a single space, and a number representing the backup level. +If one ZFS dataset or subtree is to participate in more than one DLE +(for different backup media series, onsite/offsite, etc.), this property +can be used to set distinct prefixes for the DLEs, so the holds they create +do not interfere. The default is the string "org.amanda holdsend". + + + + LARGE-BLOCK +Pass the option to zfs send, +for platforms that support it to allow blocks larger than 128 kB in the +generated stream. See for details. + + + + RAW +Pass the option to zfs send, +for platforms that support it to send data exactly as found on disk, +allowing encrypted datasets to be backed up even when their encryption keys +are not loaded, and stored securely. +See for details. + + + + UNCOMPRESSED +Defaults to true, because not all ZFS implementations suppport +compressed streams (zfs send -c where +means 'compressed', as in OpenZFS, not 'contained' as in Solaris). If +compressed-stream support is available, setting this property to false allows +the backup stream to be generated in the same compressed form used in the +dataset, saving both space and CPU cycles, as the stream will not get +uncompressed for backup and recompressed when restored. If sending from a pool +with the large_blocks feature, setting this +property false may be ineffective unless the LARGE-BLOCK property is also +set true. + + + + ZFSEXECUTABLE +Full path to the zfs executable on the client system. + + + +Properties specific to recovery + +The properties in this section apply to recovery only. +They can be set using the setproperty command within +an session. + + + + + + DESTRUCTIVE +Controls whether the is given to +zfs receive when restoring increments (it will always be +used when restoring the level 0 base, regardless of this property). +The default is true, meaning descendant datasets and snapshots that +disappear in a later increment will be destroyed as that increment is replayed, +so the final result matches the original. This property can be set to false, +if desired, so that destructions of datasets or snapshots will not be replayed. + + + + EXCLUDEPROPERTY +Takes one or more ZFS dataset property names, which will not be applied from +the recovered backup stream (so the values of those properties in the received +dataset will be defaulted or inherited in the usual way, as if no value had +been saved in the backup stream. +Not every platform supports the to +zfs receive; use of this property on platforms that do not +will cause recovery to fail. + + + + OVERRIDEPROPERTY +Takes one or more values of the form +property=value, which will be +applied in the received dataset inplace of values saved in the backup stream. +Not every platform supports the to +zfs receive; use of this property on platforms that do not +will cause recovery to fail. + + + + TARGET +The ZFS name under which to recover the data. Because ZFS recovery involves +whole filesystems at a time, to reduce the risk of clobbering wanted filesystems +by mistake, this property does not have a default; +it must be set explicitly with setproperty +ahead of an extract within amrecover. +There is no prohibition on setting it the same as the device in the DLE if +desired, though it is also easy to set it to a different name and rename it +after successful recovery. + + + + UNMOUNTED +Controls whether the is given to +zfs receive to suppress automatic mounting of +datasets received. + + + + + + +EXAMPLE + + + define application-tool app_amzfsholdsend { + plugin "amzfs-holdsend" + } + +A dumptype using this application might look like: + + define dumptype amzfshold { + global + program "APPLICATION" + application "app_amzfsholdsend" + } + +Note that the program parameter must be set to +"APPLICATION" to use the application +parameter. + + + + +, + + + + + diff --git a/perl/Amanda/Application/Abstract.pm b/perl/Amanda/Application/Abstract.pm new file mode 100644 index 0000000000..12ce283173 --- /dev/null +++ b/perl/Amanda/Application/Abstract.pm @@ -0,0 +1,1883 @@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use strict; +use warnings; + +=head1 NAME + +Amanda::Application::Abstract - A base class for Amanda applications + +=head1 SYNOPSIS + + package amMyAppl; + use base qw(Amanda::Application::Abstract) + + sub supports_message_line { return 1; } + ... + + sub inner_backup { + my ($self, $fdout) = @_; + ... + } + ... + + package main; + amMyAppl::->run(); + +=head1 DESCRIPTION + +C handles much of the common housekeeping +needed to implement Amanda's +L, so that +practical applications can be implemented in few lines of code by overriding +only a few necessary methods. + +C is itself a subclass of +C, but the latter cannot be instantiated until after +the command line has been parsed (at least the C option needs to be +passed to its constructor). Therefore, C +supplies I methods for the preliminary work of declaring the supported +options, parsing the command line, and finally calling C to get an instance +of the class. Then the actual operations of the application are carried out by +instance methods, as you would expect. + +In perl, class methods are inheritable and overridable just as instance methods +are, and the syntax C<$class-ESUPER::method(...)> works, analogously to its +familiar use in an instance method. So, the pre-instantiation behavior of an +application (declaring command options, etc.) can be tailored by simply +overriding the necessary class methods, just as its later operations can be +supplied by overriding instance methods. + +=cut + +package Amanda::Application::Abstract; +use base qw(Amanda::Application); + +use Amanda::Feature; +use Data::Dumper; +use Errno; +use File::Path qw{make_path}; +use File::Spec; +use Getopt::Long; +use IO::Handle; +use Scalar::Util qw{blessed}; +use Text::ParseWords; + +=head1 FUNDAMENTAL CLASS METHODS + +=head2 C + + appclass::->run() + +Run the application: grab the subcommand from C, be sure it is a known +one, set up for option parsing, parse the options, construct an instance +passing all the supplied options (which will include the config name, needed +by the C constructor), and finally C the subcommand, +where C is inherited from C by way of +C. + +Any Perl exception thrown from the C is caught. If it is a subclass of +C, its C method is called, which +ordinarily prints an error description to the server and exits with a nonzero +status. If it is any other kind of object, or not an object, it is passed to +C, with much the same effect. + +=cut + +# A superclass uses Amanda::Debug, which, among other things, sets its own +# __WARN__ and __DIE__ handlers. The __DIE__ handler turns an exception into +# a call to Amanda::Debug::critical, but only does that when not in 'eval' +# state, so an exception will work as expected within an eval. Outside of an +# eval, it results in a non-zero exit. + +sub run { + my ( $class ) = @_; + my $subcommand = shift @ARGV || '(empty)'; + if ( $subcommand !~ /^(?:backup|discover|estimate|index|print| + restore|selfcheck|support|validate)/x ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'subcommand', value => $subcommand, problem => 'unknown'); + } + + my %opthash = (); + my @optspecs = (); + + my $optinit = $class->can('declare_'.$subcommand.'_options'); + unless ( defined $optinit ) { + die Amanda::Application::ImplementationLimitError->transitionalError( + item => 'subcommand', value => $subcommand, problem => 'unsupported' + ); + } + + $class->$optinit(\%opthash, \@optspecs); + + unless ( GetOptions(\%opthash, @optspecs) ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'options', problem => 'invalid'); + } + + # The calling Amanda code has taken care of unwrapping any quoting/escaping + # applied to option values for transfer over the network; the options as + # passed via exec are in their proper, usable form. + + my $app = $class->new(\%opthash); + + Amanda::Debug::debug("Options: " . Data::Dumper->new([$app->{options}]) + ->Sortkeys(1)->Terse(1)->Useqq(1)->Dump()); + + eval { + $app->do($subcommand); + }; + if ( my $exc = $@ ) { + # Handling of exceptions may rely on the proper values of $app->{action} + # and $app->{mesgout} being set early by do() and remaining set + # thereafter. + if ( defined blessed($exc) and + $exc->isa('Amanda::Application::Message') ) { + $exc->on_uncaught($app); + } + else { + $app->print_to_server_and_die( + "unexpected: " . $exc, $Amanda::Script_App::FAILURE); + } + } +} + +=head2 C + + $class->new(\%opthash) + +A typical application need not call this; it is called by C after the +command line options have been declared and the command line has been parsed. +The hash reference contains option values as stored by C. A certain +convention is assumed: if there is an option/property named I, its value +will be stored in I with key exactly I, in exactly the way +C normally would, I the option-declaring method had planted +a code reference there in order to more strictly validate the option. In that +case, of course I<$opthash{o}> is the code reference, and the final value will +be stored with a different key derived from I instead, which will not collide +with any usable C option name. Option-validation code should use +C to store the final, validated value; it uses the same +convention, so the value will be properly retrieved here. + +Design to defer checks, or other operations that may throw exceptions, into +mathods called after C, such as a C or C method. +Exceptions thrown from C itself will not be caught. + +Will set C<$self-\>{cmd_from_sendbackup}> and C<$self-\>{cmd_to_sendbackup} +(to integer file descriptor numbers) if the corresponding options were passed; +the superclass expects this, and will (in C) set C<$self-\>{cmdin}> and +C<$self-\>{cmdout}> to corresponding Perl filehandles. + +=cut + +sub new { + my ( $class, $refopthash ) = @_; + my %options = (); + for ( my ($k, $v) ; ($k, $v) = each %{$refopthash} ; ) { + next if 0 == ord($k); + if ( "CODE" ne ref $v ) { + $options{$k} = $v; + } + else { + $options{$k} = $refopthash->{"\x00".$k} + } + } + my $self = $class->SUPER::new($options{'config'}); + $self->{'options'} = \%options; + + $self->{'cmd_from_sendbackup'} = $self->{'options'}->{'cmd-from-sendbackup'} + if exists $self->{'options'}->{'cmd-from-sendbackup'}; + $self->{'cmd_to_sendbackup'} = $self->{'options'}->{'cmd-to-sendbackup'} + if exists $self->{'options'}->{'cmd-to-sendbackup'}; + + return $self; +} + +=head1 CLASS METHODS IMPLEMENTING VARIOUS QUOTING RULES + +=head2 C + + $class->dquote($s) + +The purpose of this method is to quote a string in exactly the way that +C unquotes things, because C is what +C claims to use. However, the +L does not spell out the rules to use. +O'Reilly's Perl Cookbook says (1.15) that "Quotation marks and backslashes are +the only characters that have meaning backslashed. Any other use of a backslash +will be left in the output string." ... which seems to be true, looking at the +code, and is true of ' when ' is the quote mark, and true of " when " is the +quote mark. That makes it actually I like any common Unix shell, and also +not like C, so it needs its own dedicated +implementation here to be sure to get it right. + +=cut + +sub dquote { + my ( $class, $s ) = @_; + return $s if 1 eq scalar shellwords($s); + my $qs = $s; + $qs =~ s/([\\"])/\\$1/go; + $qs = '"' . $qs . '"'; + my @parsed = shellwords($qs); + return $qs if 1 eq scalar @parsed and $s eq $parsed[0]; + die 'dquote could not handle: ' . $s; +} + +=head2 C + + $class->gtar_escape_quote($s) + +This method quotes a string using the same rules documented for +GNU C in C<--quoting-style=escape> mode, as used on the index stream. +(Note: the GNU C documentation, as of April 2016 anyway, says that space +characters also get escaped, though GNU C has never actually done that, +and neither does this method.) + +=cut + +sub gtar_escape_quote { + my ( $class, $str ) = @_; + $str =~ s' + (\a)(?{"a"})|(\f)(?{"f"})|(\n)(?{"n"})|(\r)(?{"r"})|(\t)(?{"t"}) + | (\010)(?{"b"})|(\013)(?{"v"})|(\177)(?{"?"})|(\\)(?{"\\"}) + | ([[:cntrl:]])(?{sprintf("%03o",ord($^N))}) + 'q{\\}.$^R'egox; + return $str; +} + +=head2 C + + $class->gtar_escape_unquote($s) + +The inverse of C. + +=cut + +sub gtar_escape_unquote { + my ( $class, $str ) = @_; + $str =~ s'\\(?: + (a)(?{"\a"})|(f)(?{"\f"})|(n)(?{"\n"})|(r)(?{"\r"})|(t)(?{"\t"}) + | (b)(?{"\010"})|(v)(?{"\013"})|(\?)(?{"\177"})|(\\)(?{"\\"}) + | ([0-7]{1,3})(?{chr(oct($^N))}) + )'$^R'egox; + return $str; +} + +=head1 CLASS METHODS FOR VALIDATING OPTION/PROPERTY VALUES + + ..._property_setter(\%opthash) + +For a common type ... return a sub that can be placed in C<%opthash> +to validate and store a value of that type. The anonymous sub accepts the +parameters described at "User-defined subroutines to handle options" for +C, and will store the validated, converted option value in +I<%opthash> by calling C. +For example, a C method that defines a boolean +property I may contain the snippet: + + $refopthash->{'foo'} = $class->boolean_property_setter($refopthash); + +Properties with scalar or hash values can be supported (the +setter sub can tell by the number of parameters passed to it by +C), as well as multiple-valued ones accumulating into an array, +provided an array has been installed with C before the options +are parsed. + +A property setter calls C with a simple string message, not an exception +object, as it will be caught by C itself and passed to C, +and cause C to return false when finished. + + store_option(\%opthash, optname, [k,], v) + +Store into I<%opthash> a value I of option I. For an option that +accumulates in a hash (the optspec ends with C<%>), a key I (not undef) +must also be supplied. If I<%opthash> already contains an array at the +corresponding slot, the option will be assumed multivalued, and I will be +appended to the array. + +This sub implements the same convention observed by C for locating the +option value if C<$opthash{$optname}> is a code reference (validation sub). + +=cut + +sub store_option { + my ( $class, $refopthash, $optname, $k, $v); + $class = shift; + $refopthash = shift; + $optname = shift; + $k = shift if 2 == scalar(@_); + $v = shift; + my $existing_val = $refopthash->{$optname}; + if ( defined($existing_val) and "CODE" eq ref $existing_val ) { + $optname = "\x00" . $optname; + $existing_val = $refopthash->{$optname}; + } + if ( defined $k ) { + $refopthash->{$optname}->{$k} = $v; + } + elsif ( defined($existing_val) and "ARRAY" eq ref $existing_val ) { + push @{$refopthash->{$optname}}, $v; + } + else { + $refopthash->{$optname} = $v; + } +} + +=head2 C + +Allows the same boolean literals described in C: +C<1>, C, C, C, C, C or +C<0>, C, C, C, C, C, case-insensitively. + +=cut + +sub boolean_property_setter { + my ( $class, $refopthash ) = @_; + return sub { + my ( $optname, $k, $v, $b ); + $optname = shift; + $k = shift if 2 == scalar(@_); + $v = shift; + if ( $v =~ /^(?:1|y|yes|t|true|on)$/i ) { + $b = 1; + } + elsif ( $v =~ /^(?:0|n|no|f|false|off)$/i ) { + $b = 0; + } + else { + die 'invalid value ' . Amanda::Util::quote_string($v) . + ' for boolean property ' . + Amanda::Util::quote_string("$optname"); + } + $class->store_option($refopthash, $optname, $k, $b); + }; +} + +=head1 CLASS METHODS FOR DECLARING ALLOWED OPTIONS + + declare_..._options(\%opthash, \@optspecs) + +Each of these is a class method that is passed two references: a hash reference +I<\%opthash> and a list reference I<\@optspecs>. It should push onto +I<\@optspecs> the declarations of whatever options are valid for the +corresponding subcommand, using the syntax described in L. +It does not need to touch I<\%opthash>, but may store a code reference at the +name of any option, to apply stricter validation when the option is parsed. +In that case, the code reference should ultimately call C +to store the validated value. + +For each valid I, there must be a +CIC<_options> class method, which will be called +by C just before trying to parse the command line. + +=head2 C + +Declare the options common to every subcommand. This is called by +every other CIC<_options> method, and, if not overridden itself, +simply declares the C, C, C, C options that +(almost?) every subcommand ought to support. + +A subclass that has properties of its own should probably declare them here +rather than in more-specific CIC<_options> methods, as whatever +properties are set in the Amanda configuration could be passed along for any +subcommand; Amanda won't know which subcommands need them and which don't. + +=cut + +sub declare_common_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + push @$refoptspecs, ( 'config=s', 'host=s', 'disk=s', 'device=s' ); +} + +=head2 C + +Declare options for the C subcommand. Only the C ones. + +=cut + +sub declare_support_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->declare_common_options($refopthash, $refoptspecs); +} + +=head2 C + +Declare options for the C subcommand. Unless overridden, this simply +declares the C ones plus C, C, C, C, and +C, C, C, C, and +C if supported. + +=cut + +sub declare_backup_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, "target|directory=s"; + push @$refoptspecs, ( + 'message=s', 'index=s', 'level=i', 'record', 'state-stream=i' ); + push @$refoptspecs, "timestamp=s" if $class->supports('timestamp'); + + push @$refoptspecs, "cmd-to-sendbackup=i", "cmd-from-sendbackup=i" + if $class->supports('cmd_stream'); + push @$refoptspecs, "server-backup-result" + if $class->supports('want_server_backup_result'); +} + +=head2 C + +Declare options for the C subcommand. Unless overridden, this simply +declares the C ones plus C, C, C, C, and +C. + +=cut + +sub declare_restore_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, "target|directory=s"; + push @$refoptspecs, ( + 'message=s', 'index=s', 'level=i', + 'dar=s', 'recover-dump-state-file=s' ); + $refopthash->{'dar'} = sub { + my ( $optname, $optval ) = @_; + if ( $optval eq 'YES' ) { + $class->store_option($refopthash, $optname, 1); + } + elsif ( $optval eq 'NO' ) { + $class->store_option($refopthash, $optname, 0); + } + else { + die "--$optname requires YES or NO"; + } + }; +} + +=head2 C + +Declare options for the C subcommand. Unless overridden, this simply +declares the C ones plus C, C, C. + +=cut + +sub declare_index_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, ( + 'message=s', 'index=s', 'level=i' ); +} + +=head2 C + +Declare options for the C subcommand. Unless overridden, this simply +declares the C ones plus C and C. +If C<$class-Esupports('multi_estimate')> then C will be declared to +allow multiple uses. + +=cut + +sub declare_estimate_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, "target|directory=s"; + push @$refoptspecs, ( + 'message=s', 'level=i'.($class->supports('multi_estimate') ? '@' : '' ) + ); + push @$refoptspecs, "calcsize" if $class->supports('calcsize'); + push @$refoptspecs, "timestamp=s" if $class->supports('timestamp'); +} + +=head2 C + +Declare options for the C subcommand. Unless overridden, this simply +declares the C ones plus C, C, C, and C. + +=cut + +sub declare_selfcheck_options { + my ( $class, $refopthash, $refoptspecs ) = @_; + $class->declare_common_options($refopthash, $refoptspecs); + push @$refoptspecs, "target|directory=s"; + push @$refoptspecs, ('message=s', 'index=s', 'level=i', 'record'); +} + +=head2 C + +Declare options for the C subcommand. Unless overridden, this +declares no options at all. + +=cut + +sub declare_validate_options { + my ( $class, $refopthash, $refoptspecs ) = @_; +} + +=head1 CLASS METHODS SUPPORTING C SUBCOMMAND + +These class methods establish the default capabilities advertised by the +C subcommand. A particular application can supply I methods +to override them. Most are boolean and an absent one is treated as returning +C. Therefore, the only ones that need to be supplied here are the +non-booleans, and the booleans with non-C defaults. + +=head2 C + +Should return either C or C<'SMB'>. Default if not overridden +is C. + +=cut + +sub recover_mode { my ( $class ) = @_; return undef; } + +=head2 C + +Should return a list of C<'AMANDA'>, C<'DIRECTTCP'>, or both. +Default if not overridden is C<'AMANDA'>. + +=cut + +sub data_path { my ( $class ) = @_; return ("AMANDA"); } + +=head2 C + +Should return C<'CWD'> or C<'REMOTE'>. Default if not overridden is C. + +=cut + +sub recover_path { my ( $class ) = @_; return "CWD"; } + +=head2 C + +Can command pipes to and from the parent (sendbackup) be accepted? +Default if not overridden is C<1>. + +=cut + +sub supports_cmd_stream { my ( $class ) = @_; return 1; } + +=head2 C + +Can the application use a final backup result from the parent (sendbackup)? +Default if not overridden is C<1>. + +=cut + +sub supports_want_server_backup_result { my ( $class ) = @_; return 1; } + +=head2 C + + $class->supports(name) + +Returns C if there is no CIC<()> class method, +otherwise calls it and returns its result. + +=cut + +sub supports { + my ( $class, $supname ) = @_; + my $s = $class->can("supports_".$supname); + return (defined $s) ? $class->$s() : 0; +} + +=head2 C + +Returns the maximum C supported, Default if not overridden is zero, +indicating no support for incremental backup. Where incremental backup is +supported, this should return a fixed, maximum level supported (regardless +of the current state of prior levels backed up, which will limit the level +that can be requested in practice). May return a number or the string "DEFAULT", +which will be replaced with the largest value Amanda supports. + +=cut + +sub max_level { my ( $class ) = @_; return 0; } + +# INSTANCE METHODS SUPPORTING C SUBCOMMAND + +# +# Private instance method to produce a support-line for a boolean +# capability: $self->say_supports($confstring, $supname) where $confstring +# is the text to output on fd1, followed by a space and YES or NO and a +# newline, as determined by calling $class->supports($supname). +# + +my $say_supports = sub { + my ( $self, $confstring, $supname ) = @_; + my $yn = blessed($self)->supports($supname) ? "YES" : "NO"; + print $confstring . " " . $yn . "\n"; +}; + +# +# Private instance methods used only to sanity-check return values in case +# certain "supports" methods are overridden: +# + +# recover_mode (class method) should return either undef or "SMB" +my $checked_recover_mode = sub { + my ( $self ) = @_; + my $rm = blessed($self)->recover_mode(); + die Amanda::Application::ImplementationError->transitionalError( + problem => "unusable", item => "recover_mode", value => $rm + ) if defined $rm and $rm ne "SMB"; + return $rm; +}; + +# data_path (class method) should return list: "AMANDA", "DIRECTTCP", or both +my $checked_data_path = sub { + my ( $self ) = @_; + my @dp = blessed($self)->data_path(); + my $dpl = scalar @dp; + die Amanda::Application::ImplementationError->transitionalError( + problem => "unusable", item => "data_path", value => Dumper(\@dp) + ) if $dpl lt 1 or 0 lt grep(!/^(?:AMANDA|DIRECTTCP)$/, @dp); + return @dp; +}; + +# recover_path (class method) should return either "CWD" or "REMOTE" +my $checked_recover_path = sub { + my ( $self ) = @_; + my $rp = blessed($self)->recover_path(); + die Amanda::Application::ImplementationError->transitionalError( + problem => "unusable", item => "recover_path", value => $rp + ) unless defined $rp and $rp =~ /^(?:CWD|REMOTE)$/; + return $rp; +}; + +# max_level (class method) should return 0..$amanda_max or 'DEFAULT' +my $checked_max_level = sub { + my ( $self ) = @_; + my $amanda_max = 399; + my $mxl = blessed($self)->max_level(); + $mxl = $amanda_max if 'DEFAULT' eq $mxl; + die Amanda::Application::ImplementationError->transitionalError( + problem => "unusable", item => "max_level", value => $mxl + ) unless defined $mxl and $mxl =~ /^\d{1,3}$/ and 0 + $mxl <= $amanda_max; + return 0 + $mxl; +}; + +my $checked_want_server_backup_result = sub { + my ( $self ) = @_; + my $has_cmd_stream = blessed($self)->supports('cmd_stream'); + my $wants_sbr = blessed($self)->supports('want_server_backup_result'); + die Amanda::Application::ImplementationError->transitionalError( + item => "want_server_backup_result", + problem => "cannot be supported unless cmd_stream is") + if $wants_sbr and not $has_cmd_stream; + return $wants_sbr; +}; + +=head1 INSTANCE METHODS SUPPORTING LOCAL PERSISTENT STATE + +These methods establish a conventional location for a local state file, +and a default format for its contents. To avoid relying on non-core modules +(even JSON isn't a sure thing yet), the default format relies on writing +name/value pairs to the file in a form C can recover. + +By default, a state is a hash with integer keys representing levels, plus one +string key, C. The value for C is an integer, and for each +integer key I, the value is a hashref that contains +(redundantly) C<'level' =E >I plus whatever other name/value pairs +the application wants. + +=head2 C + + ($dirpart, $filepart) = $self->local_state_path() + +Supplies a reasonable location for storing local state for the subject DLE. +If not overridden, returns a I<$dirpart> based on +C<$Amanda::Paths::localstatedir> followed by Cd +components based on C, C, C, and C if present, and a +I<$filepart> based on the application name. + +=cut + +sub local_state_path { + my ( $self ) = @_; + return $self->build_state_path($Amanda::Paths::localstatedir); +} + +=head2 C + + ($dirpart, $filepart) = $self->build_state_path($basedir) + +Does the actual building of a local state path (as described above for +C) given a base directory, so that a subclass can easily +override C to use a different base directory, but otherwise +build the rest of the path the same way. + +=cut + +sub build_state_path { + my ( $self, $basedir ) = @_; + my @components = ( $basedir, $self->{'type'} ); + my $c = $self->{'options'}->{'config'}; + push @components, 'c-'.Amanda::Util::hexencode($c) if defined $c; + $c = $self->{'options'}->{'host'}; + push @components, 'h-'.Amanda::Util::hexencode($c) if defined $c; + $c = $self->{'options'}->{'disk'}; + push @components, 'd-'.Amanda::Util::hexencode($c) if defined $c; + $c = $self->{'options'}->{'device'}; + push @components, 'v-'.Amanda::Util::hexencode($c) if defined $c; + return File::Spec->catdir(@components), + Amanda::Util::hexencode($self->{'name'}); +} + +=head2 C + + $hashref = $self->read_local_state(\@optspecs) + +Read a local state, if present, returning it in I<$hashref>. In the returned +state, the value for C represents the maximum level of incremental +I that could be requested, namely, one plus the maximum previous level +represented in the state. If no stored state is found, a state hash is returned +with C of zero and no other data. Otherwise, the file is parsed for +one or more levels of saved state, by repeatedly applying C with +I<\@optspecs> to declare the application's allowed options. + +An application that uses local state should read it into +C<$self->{'localstate'}>, as C will call C +on that member automatically if the caller has asked to record state. + +=cut + +sub read_local_state { + my ( $self, $optspecs ) = @_; + my ( $dirpart, $filepart ) = $self->local_state_path(); + my $fn = File::Spec->catfile($dirpart, $filepart); + my $ret = open my $fh, '<', $fn; + if ( ! $ret ) { + if ( $!{ENOENT} ) { + my %empty = ( 'maxlevel' => 0 ); + return \%empty; + } + die Amanda::Application::EnvironmentError->transitionalError( + item => 'local state file', value => $fn, problem => 'read', + errno => $!); + } + my $slurped; + { + local $/; + $slurped = <$fh>; + } + close $fh; + my $opthash = {}; + my $maxlevel = 0; + my %result; + my $remain; + ( $ret, $remain ) = + Getopt::Long::GetOptionsFromString($slurped, $opthash, @$optspecs); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'local state file', value => $fn, problem => 'malformed') + unless $ret and (0 == scalar(@$remain) or $remain->[0] =~ m/^--/); + my $lev = $opthash->{'level'}; + $maxlevel = $lev if $lev > $maxlevel; + $result{$lev} = $opthash; + while ( 0 < scalar @$remain ) { + $opthash = {}; + $ret = Getopt::Long::GetOptionsFromArray($remain, $opthash, @$optspecs); + die Amanda::Application::EnvironmentError->transitionalError( + item => 'local state file', value => $fn, problem => 'malformed') + unless $ret; + $lev = $opthash->{'level'}; + $maxlevel = $lev if $lev > $maxlevel; + $result{$lev} = $opthash; + } + $result{'maxlevel'} = 1 + $maxlevel; + return \%result; +} + +=head2 C + + $self->write_local_state(\%levhash) + +Save the local state represented by I<\%levhash>, creating the file and +intermediate directories if necessary. C +is used to avoid leaving partially-overwritten state. + +This is normally called by C as part of successful completion. +For unsuccessful completion, C should be called instead. + +=cut + +sub write_local_state { + my ( $self, $levhash ) = @_; + my $state = ''; + for ( my ($level, $opthash); ($level, $opthash) = each %$levhash; ) { + next if 'maxlevel' eq $level; + $state .= '--level ' . $level . "\n"; + for ( my ($k, $v); ($k, $v) = each %$opthash; ) { + next if 'level' eq $k; + $state .= $self->dquote('--'.$k).' '.$self->dquote($v)."\n"; + } + $state .= "--\n"; + } + my ( $dirpart, $filepart ) = $self->local_state_path(); + make_path($dirpart); + my $fn = File::Spec->catfile($dirpart, $filepart); + Amanda::Util::safe_overwrite_file($fn, $state) + or die Amanda::Application::EnvironmentError->transitionalError( + errno => $!, item => 'local state file', value => $fn, + problem => 'overwrite'); +} + +=head2 C + + $self->repair_local_state() + +If an application might allocate resources during backup that would normally +be referred to as part of the saved local state, but an unsuccessful exit +(that does not call C) would leave those resources leaked +(not referred to by the state, and not reclaimed), then the application should +override this method to reclaim them. + +If not overridden, this method does nothing. + +=cut + +sub repair_local_state { + my ( $self ) = @_; +} + +=head2 C + + $self->update_local_state(\%state, $level, \%opthash) + +Given a I<\%state> (which contains C<'maxlevel' =E >(I+1) as well as +IC< =E >I<(opthash for level k)> for I in 0..n), and a new +I<$level> and corresponding I<\%opthash> for that level, change +C to 1 + I<$level>, drop all entries for levels above +I<$level>, and install the new I<\%opthash> at I<$level>. + +=cut + +sub update_local_state { + my ( $self, $state, $level, $opthash ) = @_; + for ( my ($l, $oh); ($l, $oh) = each %$state; ) { + next if 'maxlevel' eq $l or (0 + $l) le $level; + delete $state->{$l}; + } + $state->{'maxlevel'} = 1 + $level; + $state->{$level} = $opthash; +} + +=head1 INSTANCE METHOD SUPPORTING POST-PARSE OPTION/PROPERTY CHECKS + +Option/property checks can happen two ways. The C +methods can set a coarse limit on what options are accepted, and +C<..._property_setter> methods can further restrict just what values +are accepted for a particular option. Those will not catch other kinds +of errors, such as combinations of options that make no sense together, +missing ones that are needed, etc. Those higher-level checks can be made +in C instance methods, after the options have been +parsed and the app has been instantiated. + +=head2 C + + $self->check(boolean-expr, message) + +If I is false, pass I to C with a +severity of C, and remember that a check failed. + + $self->check() + +If any previous check (invocation of this method with arguments) failed, +throw an exception. By dividing the labor this way, several checks can be +performed, and all detected errors reported to the server, before the final +C that terminates execution by throwing the exception. + +=cut + +sub check { + my ( $self, $bval, $message ) = @_; + unless ( 1 == scalar(@_) ) { + $self->{'checkstate'} = 1 unless exists $self->{'checkstate'}; + return if $bval; + $self->{'checkstate'} = 0; + $self->print_to_server($message, $Amanda::Script_App::ERROR); + return; + } + unless ( exists $self->{'checkstate'} ) { + die Amanda::Application::ImplementationError->transitionalError( + item => 'check', problem => 'called before anything checked'); + } + unless ( $self->{'checkstate'} ) { + die Amanda::Application::InvocationError->transitionalError( + item => 'check', value => $self->{'action'}, + problem => 'error(s) detected'); + } +} + +=head1 INSTANCE METHODS USABLE IN SUBCOMMANDS + +=head2 Checked conversions + +=head3 C + + $self->int2big(n) + +Return a C version of I, throwing an exception if it is already +too late (something that could happen in a perl with 32-bit ints if it has +already widened I to a float and lost precision). + +=cut + +sub int2big { + my ( $self, $n ) = @_; + die Amanda::Application::PrecisionLossError->transitionalError( + item => 'numeric value', value => $n) + if $n - 1 == $n or $n == $n + 1; + my $big = Math::BigInt->new($n); + die Amanda::Application::PrecisionLossError->transitionalError( + item => 'numeric value', value => $n) + unless $big->is_int(); + return $big; +} + +=head3 C + + $self->big2int(n) + +Return the native perl number corresponding to the C I, +throwing an exception if I gets widened to something (a perl float, say) +that does not retain precision to the units place. + +=cut + +sub big2int { + my ( $self, $big ) = @_; + my $n = $big->numify(); + die Amanda::Application::PrecisionLossError->transitionalError( + item => 'numeric value', value => $big->bstr()) + if $n - 1 == $n or $n == $n + 1; + return $n; +} + +=head2 Determining target of operation + +=head3 C + + $self->target([default]) + +Return the target of the operation, to be interpreted as a directory name +by applications that act on directory trees, or as a file name by an application +that acts on a single object. + +The value returned will be that of the TARGET property, if that has been given. +Otherwise, for actions other than restore, it will be taken from the DEVICE +property, if that has been given. If not determined by either of those rules, +it will be the value of I if provided; otherwise an exception will be +thrown. Pass C as I in order to simply return C rather +than throwing the exception. + +This provides the usual behavior for backup, using DEVICE unless explicitly +overridden by TARGET. When restoring, if C is returned, restoration +should be into the directory named by C; when +not overridden, C ensures the current directory is that one. +For an application that acts on a directory tree, C should be +called to override that location in case a TARGET property has been given. +An application that acts on a single object should supply a default file name +(not an absolute path). It will be returned if there is no explicit TARGET, +which should leave the application creating a file by that default name in +the current directory. + +=cut + +sub target { + my ( $self, $default ) = @_; + + if ( exists $self->{'options'}->{'target'} ) { + return $self->{'options'}->{'target'}; + } + if ( 'restore' ne $self->{'action'} and + exists $self->{'options'}->{'device'} ) { + return $self->{'options'}->{'device'}; + } + return $default if 1 < scalar(@_); + + die Amanda::Application::InvocationError->transitionalError( + item => 'target', problem => 'must be specified; there is no default'); +} + +=head3 C, if it is not C. + +If C returns C, no directory change is done. + +Throw an exception if the directory change fails. This assumes that the target +should be interpreted as a directory, so only applications that act on directory +trees should call this method, not an application that acts on a single object. + +=cut + +sub chdir_to_target { + my ( $self ) = @_; + + my $tg = $self->target(undef); + + return unless defined $tg; + + chdir $tg + or die Amanda::Application::EnvironmentError->transitionalError( + item => 'target', value => $tg, errno => $!); +} + +=head1 INSTANCE METHODS IMPLEMENTING SUBCOMMANDS + +=head2 C + +As long as the application class overrides the methods indicating its +support for these capabilities, this default implementation will take care +of producing output in the proper format. + +=cut + +sub command_support { + my ( $self ) = @_; + + $self->$say_supports( "CONFIG", "config"); + $self->$say_supports( "HOST", "host"); + $self->$say_supports( "DISK", "disk"); + $self->$say_supports( "INDEX-LINE", "index_line"); + $self->$say_supports( "INDEX-XML", "index_xml"); + $self->$say_supports( "MESSAGE-LINE", "message_line"); + $self->$say_supports( "MESSAGE-XML", "message_xml"); + $self->$say_supports( "RECORD", "record"); + $self->$say_supports( "INCLUDE-FILE", "include_file"); + $self->$say_supports( "INCLUDE-LIST", "include_list"); + $self->$say_supports("INCLUDE-LIST-GLOB", "include_list_glob"); + $self->$say_supports( "INCLUDE-OPTIONAL", "include_optional"); + $self->$say_supports( "EXCLUDE-FILE", "exclude_file"); + $self->$say_supports( "EXCLUDE-LIST", "exclude_list"); + $self->$say_supports("EXCLUDE-LIST-GLOB", "exclude_list_glob"); + $self->$say_supports( "EXCLUDE-OPTIONAL", "exclude_optional"); + $self->$say_supports( "COLLECTION", "collection"); + $self->$say_supports( "CALCSIZE", "calcsize"); + $self->$say_supports( "CLIENT-ESTIMATE", "client_estimate"); + $self->$say_supports( "MULTI-ESTIMATE", "multi_estimate"); + + print "MAX-LEVEL ".($self->$checked_max_level())."\n"; + + my $rcvrmode = $self->$checked_recover_mode(); + print "RECOVER-MODE ".$rcvrmode."\n" if defined $rcvrmode; + + for my $dp ($self->$checked_data_path()) { + print "DATA-PATH ".$dp."\n"; + } + + print "RECOVER-PATH ".($self->$checked_recover_path())."\n"; + + $self->$say_supports( "AMFEATURES", "amfeatures"); + + if ( defined $Amanda::Feature::fe_amidxtaped_dar ) { + $self->$say_supports("RECOVER-DUMP-STATE-FILE", + "recover_dump_state_file"); + $self->$say_supports( "DAR", "dar"); + $self->$say_supports( "STATE-STREAM", "state_stream"); + } + + if ( defined $Amanda::Feature::fe_req_options_timestamp ) { + $self->$say_supports("TIMESTAMP", "timestamp"); + } + + if ( defined $Amanda::Feature::fe_sendbackup_stream_cmd ) { + $self->$say_supports("CMD-STREAM", "cmd_stream"); + } + + if ( defined $Amanda::Feature::fe_sendbackup_stream_cmd_get_dumper_result ){ + print "WANT-SERVER-BACKUP-RESULT ". + ($self->$checked_want_server_backup_result()? 'YES': 'NO')."\n"; + } +} + +=head2 METHODS FOR THE C SUBCOMMAND + +=head3 C + +Check sanity of the (standard) options parsed from the command line +for C<--message> and C<--index>. + +=cut + +sub check_message_index_options { + my ( $self ) = @_; + + my $msg = $self->{'options'}->{'message'}; + if ( ! defined $msg and blessed($self)->supports('message_line') ) { + $self->{'options'}->{'message'} = 'line'; + } + elsif ( 'line' eq $msg and blessed($self)->supports('message_line') ) { + # jolly + } + elsif ( 'xml' ne $msg or ! blessed($self)->supports('message_xml') ) { + $self->check(0, 'invalid --message '.$msg,); + } + + my $idx = $self->{'options'}->{'index'}; + if ( ! defined $idx and blessed($self)->supports('index_line') ) { + $self->{'options'}->{'index'} = 'line'; + } + elsif ( 'line' eq $idx and blessed($self)->supports('index_line') ) { + # jolly + } + elsif ( 'xml' ne $idx or ! blessed($self)->supports('index_xml') ) { + $self->check(0, 'invalid --index '.$idx); + } +} + +=head3 C + +Check sanity of the (standard) options parsed from the command line +for a C operation. Override to check any additional properties +for the application. + +=cut + +sub check_backup_options { + my ( $self ) = @_; + + $self->check_message_index_options(); + $self->check_level_option(); + + $self->check( + ! $self->{'options'}->{'record'} or blessed($self)->supports('record'), + 'not supported --record'); + + $self->check( + ! $self->{'options'}->{'server-backup-result'} or + defined $self->{cmd_to_sendbackup} and + defined $self->{cmd_from_sendbackup}, + '--server-backup-result without --cmd-to/from-sendbackup'); +} + +=head3 C + +Check sanity of C<--level> option(s) parsed from the command line. +In order to work in cases that do or don't support C, +this will check either a single value that isn't an array, or every +element of a value that's an array reference. + +=cut + +sub check_level_option { + my ( $self ) = @_; + + my $lvls = $self->{'options'}->{'level'}; + return if !defined($lvls); + $lvls = [ $lvls ] if ref($lvls) ne 'ARRAY'; + + for my $lvl (@$lvls) { + $self->check(0 <= $lvl and $lvl <= $self->$checked_max_level(), + 'out of range --level '.$lvl); + } +} + +=head3 C + + $self->emit_index_entry($name) + +Write I<$name> to the index stream. The caller is responsible to make sure +that I<$name> begins with a C, is relative to the I<--device>, and ends +with a C if it is a directory. Otherwise it should be the exact name that +the OS would be given to open the file. This method will handle quoting +any special characters in the name as needed within the index file. It will use +the same quoting rules as GNU C in C<--quoting-style=escape> mode. + +=cut + +sub emit_index_entry { + my ( $self, $name ) = @_; + die Amanda::Application::ImplementationLimitError->transitionalError( + item => 'emit_index_entry', value => $self->{'options'}->{'index'}, + problem => 'only "line" supported') + unless 'line' eq $self->{'options'}->{'index'}; + $self->{'index_out'}->print(blessed($self)->gtar_escape_quote($name)."\n"); +} + +=head3 C + + $self->backup_succeeded() + +If the parent has not passed C<--server-backup-result>, return true as an +optimistic assumption; otherwise, wait for the parent to report the final +success of storing the backup, and return true for a result of success or false +for a result of failure, or throw an exception for any other response. + +=cut + +sub backup_succeeded { + my ( $self ) = @_; + + return 1 unless $self->{'options'}->{'server-backup-result'}; + + my $report = $self->{'cmdin'}->getline(); + chomp $report; + return 1 if 'SUCCESS' eq $report; + return 0 if 'FAILED' eq $report; + die Amanda::Application::EnvironmentError->transitionalError( + item => 'backup result from server', value => $report, + problem => 'neither SUCCESS nor FAILED'); +} + +=head3 C + +Performs common housekeeping tasks, calling C to do the +application's real work. First calls C to verify +the invocation, creates the C file handle, and calls +C passing the fd to use for output. + +On return from C, closes the output fd and the index handle, +and writes the C line based on the size in bytes returned +by C (unless the returned size is negative, in which case no +C line is written). + +If C<--record> is supported and requested, and C<$self->{'localstate'}> is +defined, calls C{'localstate'})>. + +=cut + +sub command_backup { + my ( $self ) = @_; + $self->check_backup_options(); + $self->check(); + + $self->{'index_out'} = IO::Handle->new_from_fd(4, 'w'); + + my $fdout = fileno(STDOUT); + my $size = $self->inner_backup($fdout); + POSIX::close($fdout); + + $self->{'index_out'}->close; + if ($size->bcmp(0) >= 0) { + my $ksize = $size->copy()->badd(1023)->bdiv(1024); + if ($ksize->bcmp(32) < 0) { + $ksize = 32; # no need to represent 32 as a BigInt.... + } else { + $ksize = $ksize->bstr(); + } + print {$self->{mesgout}} "sendbackup: size $ksize\n"; + } + + if ( $self->{'options'}->{'record'} and defined $self->{'localstate'} ) { + if ( $self->backup_succeeded() ) { + $self->write_local_state($self->{'localstate'}); + } else { + $self->repair_local_state(); # app may need to roll something back + } + } + + # Here's a kludge for you ... the same commit that created the + # sendbackup_crc feature (a4e01f9) also shifted the responsibility for + # sending the "end" line, so it will be done for us in sendbackup.c. + # If using this module in an older Amanda without that feature, it still + # has to be sent here. + + if ( !defined $Amanda::Feature::fe_sendbackup_crc ) { + print {$self->{mesgout}} "sendbackup: end\n"; + } +} + +=head3 C + + $size = $self->inner_backup($fdout) + +In many cases, the application should only need to override this method to +perform a backup. The backup stream should be written to I<$fdout>, and the +number of bytes written should be returned, as a C. + +If not overridden, this default implementation writes nothing and returns zero. + +=cut + +sub inner_backup { + my ( $self, $fdout ) = @_; + return Math::BigInt->bzero(); +} + +=head3 C + + $size = $self->shovel($fdfrom, $fdto) + +Shovel all the available bytes from I<$fdfrom> to I<$fdto>, returning the +number of bytes shoveled as a C. + +=cut + +sub shovel { + my ( $self, $fdfrom, $fdto ) = @_; + my $size = Math::BigInt->bzero(); + my $rd; + my $wn; + my $buffer; + while (($rd = POSIX::read($fdfrom, $buffer, 32768)) > 0) { + $wn = Amanda::Util::full_write($fdto, $buffer, $rd); + die Amanda::Application::EnvironmentError->transitionalError( + errno => $!, item => 'fd', value => $fdto, problem => 'write') + unless defined $wn; + $size->badd($wn); + } + die Amanda::Application::EnvironmentError->transitionalError( + errno => $!, item => 'fd', value => $fdfrom, problem => 'read') + unless defined $rd; + return $size; +} + +=head2 METHODS FOR THE C SUBCOMMAND + +=head3 C + +Check sanity of the (standard) options parsed from the command line +for a C operation. Override to check any additional properties +for the application. + +=cut + +sub check_restore_options { + my ( $self ) = @_; + + $self->check_message_index_options(); + + $self->check_level_option(); + + my $dar = $self->{'options'}->{'dar'}; + my $rdsf = $self->{'options'}->{'recover-dump-state-file'}; + + $self->check( ! $dar or blessed($self)->supports('dar'), + 'not supported --dar'); + + $self->check(! $rdsf or blessed($self)->supports('recover_dump_state_file'), + 'not supported --recover-dump-state-file'); + + $self->check( !( $dar xor $rdsf ), + '--dar=YES and --recover-dump-state-file only make sense together'); +} + +=head3 C + + $self->emit_dar_request($offset, $size) + +A C subcommand that supports Direct Access Recovery can be passed +C<--dar YES> and C<--recover-dump-state-file >I, read the I +to recover the backup stream position information saved at backup time, then +look up the names requested for restoration to determine what ranges of the +backup stream will actually be needed to complete the recovery. It calls +C once for each contiguous range needed; I<$offset> and +I<$size> are both in units of bytes. + +=cut + +sub emit_dar_request { + my ( $self, $offset, $size ) = @_; + $self->{'dar_out'}->print('DAR '.$offset->bstr().':'.$size->bstr()."\n"); +} + +=head3 C + +Performs common housekeeping tasks, calling C to do the +application's real work. First calls C to verify +the invocation. If DAR is supported and requested, opens the +recover-state-file for reading and the DAR stream for writing. +Changes directory to C, or to the value of +a C<--directory> property if the application declared such an option and it +was seen on the command line. + +Calls C passing the input stream fileno, the recover-state-file +handle (or C if DAR was not requested), and the command-line arguments +(indicating objects to be restored); that is, C has a +variable-length argument list after the first two. + +On return from C, closes the recover-state-file and DAR handles +if used. + +=cut + +sub command_restore { + my ( $self ) = @_; + $self->check_restore_options(); + $self->check(); + + my $dsf; + my $rdsf = $self->{'options'}->{'recover-dump-state-file'}; + if ( defined $rdsf ) { + open $dsf, '<', $rdsf + or die Amanda::Application::EnvironmentError->transitionalError( + item => 'dump state file', value => $rdsf, errno => $!); + $self->{'dar_out'} = IO::Handle->new_from_fd(3, 'w'); + } + + chdir(Amanda::Util::get_original_cwd()); + + # XXX at this point, names-to-restore in @ARGV may need some unescaping + # done. First get signs of life, then test how Amanda is in fact passing + # them, to determine what is needed. + + $self->inner_restore(fileno(STDIN), $dsf, @ARGV); + + if ( defined $dsf ) { + close $dsf; + close $self->{'dar_out'}; + } +} + +=head3 C + + $self->inner_restore($fdin, $dsf, $filetorestore...) + +Should be overridden to do the actual restoration. Reads stream from I<$fdin>, +restores objects represented by the I<$filetorestore> arguments. If the +application supports DAR, should check I<$dsf>: if it is defined, it is a +readable file handle; read the state from it and then call C +to request the needed backup stream regions to recover the wanted objects. + +If not overridden, this default implementation reads, and does, nothing. + +=cut + +sub inner_restore { + my $self = shift; + my $fdin = shift; + my $dsf = shift; +} + +=head2 METHODS FOR THE C SUBCOMMAND + +=head3 C + +Check sanity of the (standard) options parsed from the command line +for an C operation. Override to check any additional properties +for the application. + +=cut + +sub check_index_options { + my ( $self ) = @_; + + $self->check_message_index_options(); + $self->check_level_option(); +} + +=head3 C + +Performs common housekeeping tasks, calling C to do the +application's real work. First calls C to verify +the invocation. + +Calls C, which is expected to read from C. + +On return from C, closes the index-out stream. + +=cut + +sub command_index { + my ( $self ) = @_; + $self->check_index_options(); + $self->check(); + + $self->{'index_out'} = IO::Handle->new_from_fd(1, 'w'); + $self->inner_index(); + + $self->{'index_out'}->close; +} + +=head3 C + + $self->inner_index() + +Should be overridden to read from C and call C for +each user object represented. + +If not overridden, this default implementation reads everything from C +and emits a single entry for C. + +=cut + +sub inner_index { + my $self = shift; + $self->emit_index_entry('/'); + $self->default_validate(); # which happens to read all STDIN and do nothing +} + +=head2 METHODS FOR THE C SUBCOMMAND + +=head3 C + +Check sanity of the (standard) options parsed from the command line +for an C operation. Override to check any additional properties +for the application. + +=cut + +sub check_estimate_options { + my ( $self ) = @_; + + $self->check_message_index_options(); + $self->check_level_option(); + + $self->check(exists $self->{'options'}->{'level'}, + "Can't estimate without --level"); +} + +=head3 C + +Performs common housekeeping tasks, calling C to do the +application's real work. First calls C to verify +the invocation. + +Calls C once (for each level, in case of C), +which should return a C size in bytes. Writes each size to the +output stream in the required format, assuming a block size of 1K. + +=cut + +sub command_estimate { + my ( $self ) = @_; + $self->check_estimate_options(); + $self->check(); + + my $lvls = $self->{'options'}->{'level'}; # existence already checked + my $isArray = ref($lvls) eq 'ARRAY'; + + # This next is a Should Never Happen, assuming declare_estimate_options has + # done its job; hence the ImplementationError rather than simply check(). + if ( $isArray xor blessed($self)->supports('multi_estimate') ){ + die Amanda::Application::ImplementationError->transitionalError( + item => 'option parsing', value => 'level', + problem => 'mismatch with multi_estimate support'); + } + $lvls = [ $lvls ] if ! $isArray; + + my $anyLevelSucceeded = 0; + my @failReasons; + + for my $lvl ( @$lvls ) { + my ( $size, $validsize ); + eval { + $size = $self->inner_estimate($lvl); + $validsize = ($size->isa('Math::BigInt') and $size->bcmp(0) >= 0); + }; + if ( $@ or ! $validsize ) { + push @failReasons, $@; + if ( Amanda::Application::DiscontiguousLevelError->captures($@) ) { + print "$lvl -2 -2\n"; + } + else { + print "$lvl -1 -1\n"; + } + } else { + $anyLevelSucceeded = 1; + my $ksize = $size->copy()->badd(1023)->bdiv(1024); + $ksize = ($ksize->bcmp(32) < 0) ? 32 : $ksize->bstr(); + print "$lvl $ksize 1\n"; + } + } + + die Amanda::Application::MultipleError->transitionalError( + exceptions => \@failReasons) unless ( $anyLevelSucceeded ); +} + +=head3 C + +Takes one parameter (the level) and returns a C estimating +the backup size in bytes. + +If not overridden, this default implementation does nothing but return zero. +Hey, it's an estimate. + +=cut + +sub inner_estimate { + my ($self, $level) = @_; + return Math::BigInt->bzero(); +} + +=head2 METHODS FOR THE C SUBCOMMAND + +=head3 C + +Check sanity of the (standard) options parsed from the command line +for a C operation. Override to check any additional properties +for the application. If not overridden, this checks the same things as +C. + +=cut + +sub check_selfcheck_options { + my ( $self ) = @_; + + $self->check_backup_options(); +} + +=head3 C + +If not overridden, this simply checks the options and writes a GOOD message. + +=cut + +sub command_selfcheck { + my ( $self ) = @_; + $self->check_selfcheck_options(); + $self->check(); + $self->print_to_server("$self->{name} (non-overridden selfcheck)", + $Amanda::Script_App::GOOD); +} + +=head2 METHODS FOR THE C SUBCOMMAND + +=head3 C + +If not overridden, this simply reads the complete data stream +and does nothing with it. (Which is just what would happen if this sub +were absent and the invocation fell back to C, but +providing this sub allows subclasses to refer to it with SUPER and not +have to think about whether C or C +is the right thing to call.) + +=cut + +sub command_validate { + my ( $self ) = @_; + $self->default_validate(); +} + +package Amanda::Application::Message; +use base qw(Amanda::Message); +use Scalar::Util qw{blessed}; + +# CLASS method to determine if a given thing is an exception that's an +# instance of this class or a subclass (sort of isa in reverse). +sub captures { + my ( $class, $thing ) = @_; + return ( blessed($thing) and $thing->isa($class) ); +} + +# Private sub to relieve application authors of the need to mention __FILE__ +# and __LINE__ everywhere an exception might be thrown. Return a usually-best +# file and line by following the call stack outward until (a) out of any code +# from Amanda::Message or subclasses and (b) out of this package itself (unless +# (c) that unwinds all the way out to Script_App, in which case use the +# innermost file/line found in this package). It is still possible to pass +# source_filename and source_line explicitly to the constructor at particular +# call sites. +my $callerfill = sub { + my ( $package, $sourcefile, $sourceline ); + my ( $myfile, $myline ); + # can start at depth 1; zero is just the call to this private routine + for ( my $depth = 1; my @frameinfo = caller($depth); ++ $depth ) { + ( $package, $sourcefile, $sourceline ) = @frameinfo; + next if $package->isa('Amanda::Message'); + last unless 'Amanda::Application::Abstract' eq $package; + ( $myfile, $myline ) = ( $sourcefile, $sourceline ) + unless defined $myfile or defined $myline; + } + return ($myfile, $myline) if 'Amanda::Script_App' eq $package + and defined $myfile and defined $myline; + return ($sourcefile, $sourceline); +}; + +sub new { + my ( $class, %params ) = @_; + unless (defined $params{'source_filename'}&&defined $params{'source_line'}){ + ($params{'source_filename'},$params{'source_line'}) = $callerfill->(); + } + return $class->SUPER::new(%params); +} + +# For development transition to using ::Message but before every message has +# a code assigned; this allows code to be omitted, and uses 0 to indicate a +# generic 'good' message. It will also default severity to INFO. +sub transitionalGood { + my ( $class, %params ) = @_; + $params{'code'} = 0 unless exists $params{'code'}; + $params{'severity'} = $Amanda::Message::INFO + unless exists $params{'severity'}; + return $class->new(%params); +} + +# For development transition to using ::Message but before every message has +# a code assigned; this allows code to be omitted, and uses 1 to indicate a +# generic 'error' message. The severity, as usual, defaults to CRITICAL. +sub transitionalError { + my ( $class, %params ) = @_; + $params{'code'} = 1 unless exists $params{'code'}; + return $class->new(%params); +} + +# Make this do something useful, even when not overridden. +sub local_full_message { my ( $self ) = @_; return $self->local_message(); } + +sub on_uncaught { + my ( $self, $app ) = @_; + $app->print_to_server_and_die($self . '', $Amanda::Script_App::FAILURE); +} + +# An exception indicating the application has not been properly invoked. +package Amanda::Application::InvocationError; +use base 'Amanda::Application::Message'; +sub local_message { + my ( $self ) = @_; + my $lm = 'usage error: ' . $self->{'item'}; + $lm .= ' (' . $self->{'value'} . ')' if defined $self->{'value'}; + $lm .= ': ' . $self->{'problem'}; + return $lm; +} + +# An exception indicating some limit of the implementation has been hit (this +# includes where functionality that should be implemented simply isn't yet). +package Amanda::Application::ImplementationLimitError; +use base 'Amanda::Application::Message'; +sub local_message { + my ( $self ) = @_; + my $lm = 'implementation limit: ' . $self->{'item'}; + $lm .= ' (' . $self->{'value'} . ')' if defined $self->{'value'}; + $lm .= ': ' . $self->{'problem'}; + return $lm; +} + +# An exception indicating something is probably wrong in the implementation +# itself; an assertion/should-not-happen kind of thing. +package Amanda::Application::ImplementationError; +use base 'Amanda::Application::Message'; +sub local_message { + my ( $self ) = @_; + my $lm = 'should not happen: ' . $self->{'item'}; + $lm .= ' (' . $self->{'value'} . ')' if defined $self->{'value'}; + $lm .= ': ' . $self->{'problem'}; + return $lm; +} + +# An exception that reflects some condition in the environment where the +# application runs: missing files, I/O errors, the usual stuff that happens. +package Amanda::Application::EnvironmentError; +use base 'Amanda::Application::Message'; +sub local_message { + my ( $self ) = @_; + my $lm = $self->{'item'}; + $lm .= ' (' . $self->{'value'} . ')' if defined $self->{'value'}; + $lm .= ': ' . $self->{'problem'} if defined $self->{'problem'}; + $lm .= ': ' . $self->{'errnostr'} if defined $self->{'errnostr'}; + return $lm; +} + +# An exception reflecting a loss of numeric precision where that isn't ok. +package Amanda::Application::PrecisionLossError; +use base 'Amanda::Application::ImplementationLimitError'; +sub new { + my ( $class, %params ) = @_; + $params{'problem'} = 'Loss of numeric precision' + unless exists $params{'problem'}; + $class->SUPER::new(%params); +} + +# An exception reflecting the status of a called process. +package Amanda::Application::CalledProcessError; +use base 'Amanda::Application::EnvironmentError'; +sub new { + my ( $class, %params ) = @_; + $params{'item'} = 'External process' + unless exists $params{'item'}; + if ( exists $params{'cmd'} and not exists $params{'value'} ) { + $params{'value'} = Data::Dumper->new([$params{'cmd'}])-> + Terse(1)->Useqq(1)->Indent(0)->Dump(); + } + $class->SUPER::new(%params); +} +sub local_message { + my ( $self ) = @_; + my $lm = $self->SUPER::local_message(); + $lm .= ': exit status ' . $self->{'returncode'} + if defined $self->{'returncode'}; + return $lm; +} + +# An exception refusing a requested backup level because the prior level is +# not recorded. +package Amanda::Application::DiscontiguousLevelError; +use base 'Amanda::Application::EnvironmentError'; +sub new { + my ( $class, %params ) = @_; + $params{'item'} = 'Requested level' + unless exists $params{'item'}; + $params{'problem'} = 'Prior level not recorded'; + $class->SUPER::new(%params); +} + +# An exception bundling one or more other exceptions as an array {'exceptions'}. +package Amanda::Application::MultipleError; +use base 'Amanda::Application::Message'; +sub local_message { + my ( $self ) = @_; + return 'Collected errors: ' . scalar(@{$self->{'exceptions'}}); +} +sub on_uncaught { + my ( $self, $app ) = @_; + for my $exc ( @{$self->{'exceptions'}} ) { + $app->print_to_server($exc . '', $Amanda::Script_App::ERROR); + } + $app->print_to_server_and_die($self . '', $Amanda::Script_App::FAILURE); +} + +# An exception indicating a dump should be retried. +package Amanda::Application::RetryDumpError; +use base 'Amanda::Application::EnvironmentError'; +use Amanda::Feature; +use Amanda::Util; +sub local_message { + my ( $self ) = @_; + my $lm = 'Should retry'; + $lm .= ' in ' . (0 + $self->{delay}) . ' seconds' if exists $self->{delay}; + $lm .= ' at level ' . (0 + $self->{level}) if exists $self->{level}; + $lm .= ': ' . $self->{problem} if exists $self->{problem}; + return $lm; +} +sub on_uncaught { + my ( $self, $app ) = @_; + if ( defined $Amanda::Feature::fe_sendbackup_retry ) { + if ( 'backup' eq $app->{'action'} ) { + my $pm = 'sendbackup: retry'; + $pm .= ' delay ' . (0 + $self->{delay}) if exists $self->{delay}; + $pm .= ' level ' . (0 + $self->{level}) if exists $self->{level}; + $pm .= ' message ' . Amanda::Util::quote_string($self->{problem}) + if exists $self->{problem}; + print {$app->{mesgout}} $pm . "\n"; + return; + } + } + # if action isn't 'backup' or feature isn't supported, fall back and behave + # as an ordinary error. + $self->SUPER::on_uncaught($app); +} + +1; diff --git a/perl/Amanda/Debug.swg b/perl/Amanda/Debug.swg index 863e03936b..f1da93229a 100644 --- a/perl/Amanda/Debug.swg +++ b/perl/Amanda/Debug.swg @@ -64,7 +64,7 @@ sub _my_die { my ($msg) = @_; chomp $msg; suppress_error_traceback(); - critical(@_); + critical($msg . ''); # Equal rights for exception objects! } }; $SIG{__DIE__} = \&_my_die; @@ -73,7 +73,7 @@ sub _my_warn { foreach my $msg (@_) { my $msg1 = $msg; chomp $msg1; - warning($msg1); + warning($msg1 . ''); # Equal rights for exception objects! } }; $SIG{__WARN__} = \&_my_warn; diff --git a/perl/Amanda/Script/Abstract.pm b/perl/Amanda/Script/Abstract.pm new file mode 100644 index 0000000000..0ab61ab13d --- /dev/null +++ b/perl/Amanda/Script/Abstract.pm @@ -0,0 +1,629 @@ +# This copyright apply to all codes written by authors that made contribution +# made under the BSD license. Read the AUTHORS file to know which authors made +# contribution made under the BSD license. +# +# The 3-Clause BSD License + +# Copyright 2017 Purdue University +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +use strict; +use warnings; + +=head1 NAME + +Amanda::Script::Abstract - A base class for Amanda scripts + +=head1 SYNOPSIS + + package amMyScript; + use base qw(Amanda::Script::Abstract) + + package main; + amMyScript::->run(); + +=head1 DESCRIPTION + +C handles much of the common housekeeping +needed to implement Amanda's +L