From 4cad5e9605ccf05ed8b4a87ffe3964854886d5fa Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Thu, 22 Apr 2021 13:34:54 -0400 Subject: [PATCH] Import shared TAR helpers from common.utils --- storage_service/common/utils.py | 75 ++++++++++++++++ storage_service/locations/models/gpg.py | 71 +--------------- .../models/location_helpers/__init__.py | 0 .../models/location_helpers/helpers.py | 85 ------------------- .../locations/models/tape_archive.py | 8 +- 5 files changed, 82 insertions(+), 157 deletions(-) delete mode 100644 storage_service/locations/models/location_helpers/__init__.py delete mode 100644 storage_service/locations/models/location_helpers/helpers.py diff --git a/storage_service/common/utils.py b/storage_service/common/utils.py index 9ddc02a9c..f52d16234 100644 --- a/storage_service/common/utils.py +++ b/storage_service/common/utils.py @@ -11,6 +11,7 @@ import os import shutil import subprocess +import tarfile import uuid import scandir @@ -553,6 +554,80 @@ def set_compression_transforms(aip, compression, transform_order): return version, extension, program_name +# ########### TAR Packaging ############ + + +class TARException(Exception): + pass + + +def _abort_create_tar(path, tarpath): + fail_msg = _( + "Failed to create a tarfile at %(tarpath)s for dir at %(path)s" + % {"tarpath": tarpath, "path": path} + ) + LOGGER.error(fail_msg) + raise TARException(fail_msg) + + +def create_tar(path): + """Create a tarfile from the directory at ``path`` and overwrite + ``path`` with that tarfile. + """ + path = path.rstrip("/") + tarpath = "{}.tar".format(path) + changedir = os.path.dirname(tarpath) + source = os.path.basename(path) + cmd = ["tar", "-C", changedir, "-cf", tarpath, source] + LOGGER.info( + "creating archive of %s at %s, relative to %s", source, tarpath, changedir + ) + try: + subprocess.check_output(cmd) + except (OSError, subprocess.CalledProcessError): + _abort_create_tar(path, tarpath) + + # Providing the TAR is successfully created then remove the original. + if os.path.isfile(tarpath) and tarfile.is_tarfile(tarpath): + try: + shutil.rmtree(path) + except OSError: + # Remove a file-path as We're likely packaging a file, e.g. 7z. + os.remove(path) + os.rename(tarpath, path) + else: + _abort_create_tar(path, tarpath) + try: + assert tarfile.is_tarfile(path) + assert not os.path.exists(tarpath) + except AssertionError: + _abort_create_tar(path, tarpath) + + +def _abort_extract_tar(tarpath, newtarpath, err): + fail_msg = _( + "Failed to extract %(tarpath)s: %(error)s" % {"tarpath": tarpath, "error": err} + ) + LOGGER.error(fail_msg) + os.rename(newtarpath, tarpath) + raise TARException(fail_msg) + + +def extract_tar(tarpath): + """Extract tarfile at ``path`` to a directory at ``path``.""" + newtarpath = "{}.tar".format(tarpath) + os.rename(tarpath, newtarpath) + changedir = os.path.dirname(newtarpath) + cmd = ["tar", "-xf", newtarpath, "-C", changedir] + try: + subprocess.check_output(cmd) + except (OSError, subprocess.CalledProcessError) as err: + _abort_extract_tar(tarpath, newtarpath, err) + # TODO: GPG treats this differently because it only ever expects to + # TAR a directory but we actually want to TAR file-types as well. + os.remove(newtarpath) + + # ########### OTHER ############ diff --git a/storage_service/locations/models/gpg.py b/storage_service/locations/models/gpg.py index e791ed444..41999d718 100644 --- a/storage_service/locations/models/gpg.py +++ b/storage_service/locations/models/gpg.py @@ -4,8 +4,6 @@ import datetime import logging import os -import shutil -import subprocess import tarfile # Core Django, alphabetical @@ -242,7 +240,7 @@ def _gpg_encrypt(path, key_fingerprint): """ tar_created = False if os.path.isdir(path): - _create_tar(path) + utils.create_tar(path) tar_created = True encr_path, result = gpgutils.gpg_encrypt_file(path, key_fingerprint) if os.path.isfile(encr_path) and result.ok: @@ -252,7 +250,7 @@ def _gpg_encrypt(path, key_fingerprint): return path, result else: if tar_created: - _extract_tar(path) + utils.extract_tar(path) fail_msg = _( "An error occured when attempting to encrypt" " %(path)s" % {"path": path} ) @@ -291,69 +289,6 @@ def _encr_path2key_fingerprint(encr_path): raise GPGException(fail_msg) -def _abort_create_tar(path, tarpath): - fail_msg = _( - "Failed to create a tarfile at %(tarpath)s for dir at %(path)s" - % {"tarpath": tarpath, "path": path} - ) - LOGGER.error(fail_msg) - raise GPGException(fail_msg) - - -def _create_tar(path): - """Create a tarfile from the directory at ``path`` and overwrite ``path`` - with that tarfile. - """ - path = path.rstrip("/") - tarpath = "{}.tar".format(path) - changedir = os.path.dirname(tarpath) - source = os.path.basename(path) - cmd = ["tar", "-C", changedir, "-cf", tarpath, source] - LOGGER.info( - "creating archive of %s at %s, relative to %s", source, tarpath, changedir - ) - try: - subprocess.check_output(cmd) - except (OSError, subprocess.CalledProcessError): - _abort_create_tar(path, tarpath) - if os.path.isfile(tarpath) and tarfile.is_tarfile(tarpath): - shutil.rmtree(path) - os.rename(tarpath, path) - else: - _abort_create_tar(path, tarpath) - try: - assert tarfile.is_tarfile(path) - assert not os.path.exists(tarpath) - except AssertionError: - _abort_create_tar(path, tarpath) - - -def _abort_extract_tar(tarpath, newtarpath): - fail_msg = _( - "Failed to extract %(tarpath)s to a directory at the same" - " location." % {"tarpath": tarpath} - ) - LOGGER.error(fail_msg) - os.rename(newtarpath, tarpath) - raise GPGException(fail_msg) - - -def _extract_tar(tarpath): - """Extract tarfile at ``path`` to a directory at ``path``.""" - newtarpath = "{}.tar".format(tarpath) - os.rename(tarpath, newtarpath) - changedir = os.path.dirname(newtarpath) - cmd = ["tar", "-xf", newtarpath, "-C", changedir] - try: - subprocess.check_output(cmd) - except (OSError, subprocess.CalledProcessError): - _abort_extract_tar(tarpath, newtarpath) - if os.path.isdir(tarpath): - os.remove(newtarpath) - else: - _abort_extract_tar(tarpath, newtarpath) - - def _parse_gpg_version(raw_gpg_version): return ".".join(str(i) for i in raw_gpg_version) @@ -392,7 +327,7 @@ def _gpg_decrypt(path): # using an uncompressed AIP as input. We extract those here. if tarfile.is_tarfile(path) and os.path.splitext(path)[1] == "": LOGGER.info("%s is a tarfile so we are extracting it", path) - _extract_tar(path) + utils.extract_tar(path) return path diff --git a/storage_service/locations/models/location_helpers/__init__.py b/storage_service/locations/models/location_helpers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/storage_service/locations/models/location_helpers/helpers.py b/storage_service/locations/models/location_helpers/helpers.py deleted file mode 100644 index 675024826..000000000 --- a/storage_service/locations/models/location_helpers/helpers.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import logging -import os -import shutil -import subprocess -import tarfile - -from django.utils.translation import ugettext_lazy as _ - - -LOGGER = logging.getLogger(__name__) - - -class TARException(Exception): - pass - - -def _abort_create_tar(path, tarpath): - fail_msg = _( - "Failed to create a tarfile at %(tarpath)s for dir at %(path)s" - % {"tarpath": tarpath, "path": path} - ) - LOGGER.error(fail_msg) - raise TARException(fail_msg) - - -def create_tar(path): - """Create a tarfile from the directory at ``path`` and overwrite - ``path`` with that tarfile. - """ - path = path.rstrip("/") - tarpath = "{}.tar".format(path) - changedir = os.path.dirname(tarpath) - source = os.path.basename(path) - cmd = ["tar", "-C", changedir, "-cf", tarpath, source] - LOGGER.info( - "creating archive of %s at %s, relative to %s", source, tarpath, changedir - ) - try: - subprocess.check_output(cmd) - except (OSError, subprocess.CalledProcessError): - _abort_create_tar(path, tarpath) - - # Providing the TAR is successfully created then remove the original. - if os.path.isfile(tarpath) and tarfile.is_tarfile(tarpath): - try: - shutil.rmtree(path) - except OSError: - # Remove a file-path as We're likely packaging a file, e.g. 7z. - os.remove(path) - os.rename(tarpath, path) - else: - _abort_create_tar(path, tarpath) - try: - assert tarfile.is_tarfile(path) - assert not os.path.exists(tarpath) - except AssertionError: - _abort_create_tar(path, tarpath) - - -def _abort_extract_tar(tarpath, newtarpath, err): - fail_msg = _( - "Failed to extract %(tarpath)s: %(error)s" % {"tarpath": tarpath, "error": err} - ) - LOGGER.error(fail_msg) - os.rename(newtarpath, tarpath) - raise TARException(fail_msg) - - -def extract_tar(tarpath): - """Extract tarfile at ``path`` to a directory at ``path``.""" - newtarpath = "{}.tar".format(tarpath) - os.rename(tarpath, newtarpath) - changedir = os.path.dirname(newtarpath) - cmd = ["tar", "-xf", newtarpath, "-C", changedir] - try: - subprocess.check_output(cmd) - except (OSError, subprocess.CalledProcessError) as err: - _abort_extract_tar(tarpath, newtarpath, err) - # TODO: GPG treats this differently because it only ever expects to - # TAR a directory but we actually want to TAR file-types as well. - os.remove(newtarpath) diff --git a/storage_service/locations/models/tape_archive.py b/storage_service/locations/models/tape_archive.py index bfb64d203..937ef5a37 100644 --- a/storage_service/locations/models/tape_archive.py +++ b/storage_service/locations/models/tape_archive.py @@ -4,12 +4,12 @@ from __future__ import absolute_import - from django.db import models from django.utils.translation import ugettext_lazy as _ +from common import utils + from .location import Location -from .location_helpers.helpers import create_tar, extract_tar class TAR(models.Model): @@ -31,12 +31,12 @@ def move_to_storage_service(self, src_path, dest_path, dest_space): """ Moves src_path to dest_space.staging_path/dest_path. """ self.space.create_local_directory(dest_path) self.space.move_rsync(src_path, dest_path, try_mv_local=True) - extract_tar(dest_path) + utils.extract_tar(dest_path) def move_from_storage_service(self, src_path, dest_path, package=None): """ Moves self.staging_path/src_path to dest_path. """ self.space.create_local_directory(dest_path) self.space.move_rsync(src_path, dest_path) - create_tar(dest_path) + utils.create_tar(dest_path) if package.should_have_pointer_file(): """Update the pointer file to represent the TAR packaging."""