Finalizing sftp backup

.gitignore

@@ -82,8 +82,15 @@ ssh deployment/pg/postgres_data/9.3
 deployment/pg/postgres_staging_data/9.3
 deployment/pg/postgres_dev_data/9.3
 deployment/pg/dbbackup/
-deployment/pg/sftp_backup/backups/
-deployment/pg/sftp_backup/target/
+deployment/sftp_backup/backups/media/
+deployment/sftp_backup/backups/pg/
+deployment/sftp_backup/target/pg/
+
+!deployment/pg/dbbackup/backups/README.md
+!deployment/pg/dbbackup/devbackups/README.md
+!deployment/sftp_backup/target/pg/.gitkeep
+!deployment/sftp_backup/backups/media/README.md
+!deployment/sftp_backup/backups/pg/README.md
 
 # apt-cacher-ng
 71-apt-cacher-ng

deployment/Makefile

@@ -128,16 +128,27 @@ dbbackup:
 	@# - prefix causes command to continue even if it fails
 	docker exec -t -i $(PROJECT_ID)_dbbackup_1 /backups.sh
 	@docker exec -t -i $(PROJECT_ID)_dbbackup_1 cat /var/log/cron.log | tail -2 | head -1 | awk '{print $4}'
-	-@docker exec -t -i $(PROJECT_ID)_dbbackup_1 /bin/sh -c "if [ -f /backups/latest.dmp ]; then echo 'remove symlink file'; rm /backups/latest.dmp; fi"
 	# backups is intentionally missing from front of first clause below otherwise symlink comes
 	# out with wrong path...
-	@echo "creating a link in docker"
-	@docker exec -t -i $(PROJECT_ID)_dbbackup_1 /bin/sh -c "ln -s `date +%Y`/`date +%B`/PG_$(PROJECT_ID)_gis.`date +%d-%B-%Y`.dmp /backups/latest.dmp"
-	@echo "Backup should be at: backups/`date +%Y`/`date +%B`/PG_$(PROJECT_ID)_gis.`date +%d-%B-%Y`.dmp"
-	@echo "Copy backup for sftp-backup"
-	@docker exec -t -it $(PROJECT_ID)_dbbackup_1 /bin/sh -c "cp /backups/latest.dmp /target/latest.dmp"
 	# trigger sftp backups
-	@docker exec -t -i $(PROJECT_ID)_sftpbackup_1 /backups.sh
+	@docker exec -t -i $(PROJECT_ID)_sftppgbackup_1 /backups.sh
+	@docker exec -t -i $(PROJECT_ID)_sftpmediabackup_1 /backups.sh
+
+pushbackup:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Push local backup in sftpbackup to sftp remote server"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_sftppgbackup_1 /start.sh push-to-remote-sftp
+	@docker exec -t -i $(PROJECT_ID)_sftpmediabackup_1 /start.sh push-to-remote-sftp
+
+pullbackup:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Pull remote sftp backup to local backup"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)_sftppgbackup_1 /start.sh pull-from-remote-sftp
+	@docker exec -t -i $(PROJECT_ID)_sftpmediabackup_1 /start.sh pull-from-remote-sftp
 
 reload:
 	@echo
@@ -385,13 +396,24 @@ dev-dbbackup:
 	@# - prefix causes command to continue even if it fails
 	docker exec -t -i $(PROJECT_ID)dev_devdbbackup_1 /backups.sh
 	@docker exec -t -i $(PROJECT_ID)dev_devdbbackup_1 cat /var/log/cron.log | tail -2 | head -1 | awk '{print $4}'
-	-@docker exec -t -i $(PROJECT_ID)dev_devdbbackup_1 /bin/sh -c "if [ -f /backups/latest.dmp ]; then echo 'remove symlink file'; rm /backups/latest.dmp; fi"
 	# backups is intentionally missing from front of first clause below otherwise symlink comes
 	# out with wrong path...
-	@echo "creating a link in docker"
-	@docker exec -t -i $(PROJECT_ID)dev_devdbbackup_1 /bin/sh -c "ln -s `date +%Y`/`date +%B`/PG_$(PROJECT_ID)_gis.`date +%d-%B-%Y`.dmp /backups/latest.dmp"
-	@echo "Backup should be at: backups/`date +%Y`/`date +%B`/PG_$(PROJECT_ID)_gis.`date +%d-%B-%Y`.dmp"
-	@echo "Copy backup for sftp-backup"
-	@docker exec -t -it $(PROJECT_ID)dev_devdbbackup_1 /bin/sh -c "cp /backups/latest.dmp /target/latest.dmp"
 	# trigger sftp backups
-	@docker exec -t -i $(PROJECT_ID)dev_devsftpbackup_1 /backups.sh
+	@docker exec -t -i $(PROJECT_ID)dev_devsftppgbackup_1 /backups.sh
+	@docker exec -t -i $(PROJECT_ID)dev_devsftpmediabackup_1 /backups.sh
+
+dev-pushbackup:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Push local backup in sftpbackup to sftp remote server"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)dev_devsftppgbackup_1 /start.sh push-to-remote-sftp
+	@docker exec -t -i $(PROJECT_ID)dev_devsftpmediabackup_1 /start.sh push-to-remote-sftp
+
+dev-pullbackup:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Pull remote sftp backup to local backup"
+	@echo "------------------------------------------------------------------"
+	@docker exec -t -i $(PROJECT_ID)dev_devsftppgbackup_1 /start.sh pull-from-remote-sftp
+	@docker exec -t -i $(PROJECT_ID)dev_devsftpmediabackup_1 /start.sh pull-from-remote-sftp

deployment/docker-backup/Dockerfile

@@ -0,0 +1,6 @@
+FROM kartoza/pg-backup:9.3
+MAINTAINER [email protected]
+
+ADD backups.sh /backups.sh
+
+CMD ["/start.sh"]

deployment/docker-backups/backups.sh → deployment/docker-backup/backups.sh

@@ -9,6 +9,7 @@ MYDATE=`date +%d-%B-%Y`
 MONTH=$(date +%B)
 YEAR=$(date +%Y)
 MYBASEDIR=/backups
+TARGETDIR=/pg_backup
 MYBACKUPDIR=${MYBASEDIR}/${YEAR}/${MONTH}
 mkdir -p ${MYBACKUPDIR}
 cd ${MYBACKUPDIR}
@@ -26,6 +27,9 @@ do
   echo "Backing up $DB"  >> /var/log/cron.log
   FILENAME=${MYBACKUPDIR}/${DUMPPREFIX}_${DB}.${MYDATE}.dmp
   pg_dump -i -Fc -f ${FILENAME} -x -O ${DB}
-  echo "push backup to remote server" >> /var/log/cron.log
-  /usr/bin/python /sftp_remote.py ${FILENAME} >> /var/log/cron.log
+
+  # create link for latest dump
+  echo "Create link for latest dump"
+  ln -sf $FILENAME $MYBASEDIR/latest.dmp
+  cp $MYBASEDIR/latest.dmp $TARGETDIR/latest.dmp
 done