Add Archive module

The Archive module add the possibility to rsync the the files after a dump. Also you can enfore clean up with a very simple retention rule.
adjust · Dec 16, 2014 · 34981d4 · 34981d4
1 parent 6a2b501
commit 34981d4
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 28 deletions.
diff --git a/Build.PL b/Build.PL
@@ -13,6 +13,7 @@ my $build = Module::Build->new(
         'DBD::Pg'                    => 0,
         'Getopt::Long'               => 0,
         'File::Path'                 => 0,
+        'File::Rsync'                => 0,
         'DateTime'                   => 0,
         'DateTime::Format::Strptime' => 0,
         'Parallel::ForkManager'      => 0,

diff --git a/MANIFEST b/MANIFEST
@@ -1,10 +1,12 @@
+bin/analyze.pl
 bin/diff.pl
 bin/dump.pl
 bin/dump93.pl
 bin/restore.pl
 bin/restore93.pl
 Build.PL
 lib/PostgresTools.pm
+lib/PostgresTools/Archive.pm
 lib/PostgresTools/Database.pm
 lib/PostgresTools/Date.pm
 LICENSE

diff --git a/bin/dump.pl b/bin/dump.pl
@@ -15,21 +15,27 @@
 my $host = 'localhost';
 my $user = 'postgres';
 my $db;
+my $base_dir = './base';
+my $dst;
 my $pretend  = 0;
 my $verbose  = 0;
 my $progress = 0;
 my $jobs     = 1;
 my $offset   = 0;
+my $rsync    = 0;
 
 GetOptions(
     "host|h=s"   => \$host,
     "user|U=s"   => \$user,
     "db=s"       => \$db,
+    "base_dir=s" => \$base_dir,
+    "dst=s"      => \$dst,
     "jobs|j=i"   => \$jobs,
     "offset|o=i" => \$offset,
     "pretend|p"  => \$pretend,
     "verbose|v"  => \$verbose,
     "progress"   => \$progress,
+    "rsync"      => \$rsync,
 );
 
 unless ( defined($db) ) {
@@ -41,15 +47,23 @@
     exit(1);
 }
 
+if ( $rsync && !defined($dst) ) {
+    say "you need to define <dst> if rsync is used";
+    exit(1);
+}
+
 my $tools = PostgresTools->new(
     host     => $host,
     user     => $user,
     db       => $db,
+    base_dir => $base_dir,
     pretend  => $pretend,
     verbose  => $verbose,
     offset   => $offset,
     forks    => $jobs,
     progress => $progress,
+    rsync    => $rsync,
+    dst      => $dst,
 );
 
 $tools->dump;
diff --git a/lib/PostgresTools.pm b/lib/PostgresTools.pm
@@ -11,33 +11,37 @@ use DateTime::Format::Strptime;
 use Term::ProgressBar;
 use autodie;
 
+use PostgresTools::Archive;
 use PostgresTools::Database;
 use PostgresTools::Date;
 
-has user     => ( is => 'rw' );
-has user2    => ( is => 'rw' );
-has host     => ( is => 'rw' );
-has host2    => ( is => 'rw' );
-has db       => ( is => 'ro', required => 1 );
-has db2      => ( is => 'ro' );
-has dbh      => ( is => 'rw' );
-has dbh2     => ( is => 'rw' );
-has date     => ( is => 'rw' );
-has base_dir => ( is => 'rw' );
-has dump_dir => ( is => 'rw' );
-has forks    => ( is => 'rw' );
-has offset   => ( is => 'rw' );
-has exclude  => ( is => 'rw' );
-has excludes => ( is => 'rw' );
-has pretend  => ( is => 'rw' );
-has verbose  => ( is => 'rw' );
-has progress => ( is => 'rw' );
-has iter     => ( is => 'rw' );
-has bar      => ( is => 'rw' );
-has count    => ( is => 'rw' );
-has restore  => ( is => 'rw' );
-has schema   => ( is => 'rw' );
-has quiet    => ( is => 'rw' );
+has user      => ( is => 'rw' );
+has user2     => ( is => 'rw' );
+has host      => ( is => 'rw' );
+has host2     => ( is => 'rw' );
+has db        => ( is => 'ro', required => 1 );
+has db2       => ( is => 'ro' );
+has dbh       => ( is => 'rw' );
+has dbh2      => ( is => 'rw' );
+has date      => ( is => 'rw' );
+has base_dir  => ( is => 'rw' );
+has dump_dir  => ( is => 'rw' );
+has forks     => ( is => 'rw' );
+has offset    => ( is => 'rw' );
+has exclude   => ( is => 'rw' );
+has excludes  => ( is => 'rw' );
+has pretend   => ( is => 'rw' );
+has verbose   => ( is => 'rw' );
+has progress  => ( is => 'rw' );
+has iter      => ( is => 'rw' );
+has bar       => ( is => 'rw' );
+has count     => ( is => 'rw' );
+has restore   => ( is => 'rw' );
+has schema    => ( is => 'rw' );
+has quiet     => ( is => 'rw' );
+has rsync     => ( is => 'rw' );
+has dst       => ( is => 'rw' );
+has keep_days => ( is => 'rw' );
 
 sub BUILD {
     my $self = shift;
@@ -56,6 +60,8 @@ sub BUILD {
     $self->offset(0)              unless $self->offset;
     $self->pretend(0)             unless $self->pretend;
     $self->restore( $self->{db} ) unless $self->restore;
+    $self->keep_days(30)          unless $self->keep_days;
+    $self->rsync(0)               unless $self->rsync;
     $self->_create_excludes;
 }
 
@@ -75,6 +81,19 @@ sub analyze {
     }
 }
 
+sub archive {
+    my $self     = shift;
+    my $src      = "$self->{dump_dir}/$self->{db}/";
+    my $dst      = $self->{dst};
+    my $archiver = PostgresTools::Archive->new(
+        dst       => $dst,
+        base_dir  => $self->{base_dir},
+        keep_days => $self->{keep_days},
+    );
+    $archiver->backup;
+    $archiver->clean;
+}
+
 sub dump93 {
     my $self = shift;
     $self->_make_base;
@@ -108,6 +127,9 @@ sub dump {
     $self->_dump_partitions;
     $self->_dump_tables;
     $self->_dump_sequences;
+    if ( $self->rsync ) {
+        $self->archive;
+    }
 }
 
 sub restore93 {
@@ -312,9 +334,12 @@ sub _create_excludes {
 }
 
 sub _set_date {
-    my $self = shift;
+    my $self      = shift;
     my $formatter = DateTime::Format::Strptime->new( pattern => '%Y%m%d' );
-    $self->date( DateTime->now( formatter => $formatter ) );
+    my $date      = PostgresTools::Date->new(
+        formatter => $formatter,
+    );
+    $self->date( $date->offset2date(0) );
     $self->dump_dir( $self->{base_dir} . "/$self->{date}" );
 }
 

diff --git a/lib/PostgresTools/Archive.pm b/lib/PostgresTools/Archive.pm
@@ -0,0 +1,46 @@
+package PostgresTools::Archive;
+
+use strict;
+use warnings;
+use 5.012;
+
+use Moo;
+use File::Path;
+use File::Rsync;
+use PostgresTools::Date;
+
+has keep_days => ( is => 'rw' );
+has dst       => ( is => 'ro', required => 1 );
+has base_dir  => ( is => 'ro', required => 1 );
+
+sub BUILD {
+    my $self = shift;
+    unless ( $self->{dst} ) {
+        die "archive destination is undefined or empty string";
+    }
+}
+
+sub backup {
+    my $self = shift;
+    my $rsync = File::Rsync->new( { archive => 1 } );
+
+    $rsync->exec( { src => $self->{base_dir}, dest => $self->{dst} } ) or die $!;
+}
+
+sub clean {
+    my $self      = shift;
+    my $formatter = DateTime::Format::Strptime->new( pattern => '%Y%m%d' );
+    my $date      = PostgresTools::Date->new(
+        formatter => $formatter,
+    );
+    my $delete_date = $date->offset2date( $self->{keep_days} );
+    my $to_clean    = "$self->{base_dir}/$delete_date";
+    if ( -d $to_clean ) {
+        rmtree $to_clean or warn $!;
+    }
+    my $rsync = File::Rsync->new( { archive => 1, delete => 1 } );
+
+    $rsync->exec( { src => $self->{base_dir}, dest => $self->{dst} } ) or die $!;
+}
+
+1;
diff --git a/lib/PostgresTools/Date.pm b/lib/PostgresTools/Date.pm
@@ -8,11 +8,13 @@ use Moo;
 use DateTime;
 use Storable qw(dclone);
 
-has _now_ => ( is => 'rw' );
+has _now_     => ( is => 'rw' );
+has formatter => ( is => 'rw' );
 
 sub BUILD {
     my $self = shift;
-    $self->_now_( DateTime->today( formatter => DateTime::Format::Strptime->new( pattern => '%Y_%m_%d' ) ) ) unless $self->_now_;
+    $self->formatter( DateTime::Format::Strptime->new( pattern => '%Y_%m_%d' ) ) unless $self->formatter;
+    $self->_now_( DateTime->today( formatter => $self->{formatter} ) ) unless $self->_now_;
 }
 
 sub date_from_string {