diff --git a/dds_web/commands.py b/dds_web/commands.py index 0ed76e33a..14c0574fa 100644 --- a/dds_web/commands.py +++ b/dds_web/commands.py @@ -308,24 +308,10 @@ def list_lost_files(project_id: str): # Check which Safespring storage location to use # Use sto4 if project created after sto4 info added try: - if use_sto4(unit_object=project.responsible_unit, project_object=project): - flask.current_app.logger.info( - f"Safespring location for project '{project_id}': sto4" - ) - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - project.responsible_unit.sto4_endpoint, - project.responsible_unit.sto4_access, - project.responsible_unit.sto4_secret, - ) - else: - flask.current_app.logger.info( - f"Safespring location for project '{project_id}': sto2" - ) - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - project.responsible_unit.sto2_endpoint, - project.responsible_unit.sto2_access, - project.responsible_unit.sto2_secret, - ) + sto4: bool = use_sto4(unit_object=project.responsible_unit, project_object=project) + flask.current_app.logger.info( + f"Safespring location for project '{project_id}': {'sto4' if sto4 else 'sto2'}" + ) except S3InfoNotFoundError as err: flask.current_app.logger.error(str(err)) sys.exit(1) @@ -333,16 +319,17 @@ def list_lost_files(project_id: str): # Connect to S3 resource = session.resource( service_name="s3", - endpoint_url=endpoint_url, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, + endpoint_url=project.responsible_unit.sto4_endpoint + if sto4 + else project.responsible_unit.sto2_endpoint, + aws_access_key_id=project.responsible_unit.sto4_access + if sto4 + else project.responsible_unit.sto2_access, + aws_secret_access_key=project.responsible_unit.sto4_secret + if sto4 + else project.responsible_unit.sto2_secret, ) - # Clean up information - del endpoint_url - del aws_access_key_id - del aws_secret_access_key - # List the lost files try: in_db_but_not_in_s3, in_s3_but_not_in_db = list_lost_files_in_project( @@ -387,24 +374,10 @@ def list_lost_files(project_id: str): # Check which Safespring storage location to use # Use sto4 if roject created after sto4 info added try: - if use_sto4(unit_object=unit, project_object=proj): - flask.current_app.logger.info( - f"Safespring location for project '{proj.public_id}': sto4" - ) - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - proj.responsible_unit.sto4_endpoint, - proj.responsible_unit.sto4_access, - proj.responsible_unit.sto4_secret, - ) - else: - flask.current_app.logger.info( - f"Safespring location for project '{proj.public_id}': sto2" - ) - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - proj.responsible_unit.sto2_endpoint, - proj.responsible_unit.sto2_access, - proj.responsible_unit.sto2_secret, - ) + sto4: bool = use_sto4(unit_object=unit, project_object=proj) + flask.current_app.logger.info( + f"Safespring location for project '{proj.public_id}': {'sto4' if sto4 else 'sto2'}" + ) except S3InfoNotFoundError as err: flask.current_app.logger.error(str(err)) continue @@ -412,16 +385,17 @@ def list_lost_files(project_id: str): # Connect to S3 resource_unit = session.resource( service_name="s3", - endpoint_url=endpoint_url, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, + endpoint_url=proj.responsible_unit.sto4_endpoint + if sto4 + else proj.responsible_unit.sto2_endpoint, + aws_access_key_id=proj.responsible_unit.sto4_access + if sto4 + else proj.responsible_unit.sto2_access, + aws_secret_access_key=proj.responsible_unit.sto4_secret + if sto4 + else proj.responsible_unit.sto2_secret, ) - # Clean up information - del endpoint_url - del aws_access_key_id - del aws_secret_access_key - # List the lost files try: in_db_but_not_in_s3, in_s3_but_not_in_db = list_lost_files_in_project( @@ -445,9 +419,6 @@ def list_lost_files(project_id: str): f"Project errors: {num_proj_errors}\n" ) - # Garbage collect - gc.collect() - @lost_files_s3_db.command(name="add-missing-bucket") @click.option("--project-id", "-p", type=str, required=True) @@ -477,20 +448,10 @@ def add_missing_bucket(project_id: str): # Use sto4 if project created after sto4 info added try: - if use_sto4(unit_object=project.responsible_unit, project_object=project): - flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto4") - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - project.responsible_unit.sto4_endpoint, - project.responsible_unit.sto4_access, - project.responsible_unit.sto4_secret, - ) - else: - flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto2") - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - project.responsible_unit.sto2_endpoint, - project.responsible_unit.sto2_access, - project.responsible_unit.sto2_secret, - ) + sto4 = use_sto4(unit_object=project.responsible_unit, project_object=project) + flask.current_app.logger.info( + f"Safespring location for project '{project_id}': {'sto4' if sto4 else 'sto2'}" + ) except S3InfoNotFoundError as err: flask.current_app.logger.error(str(err)) sys.exit(1) @@ -498,16 +459,17 @@ def add_missing_bucket(project_id: str): # Connect to S3 resource = session.resource( service_name="s3", - endpoint_url=endpoint_url, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, + endpoint_url=project.responsible_unit.sto4_endpoint + if sto4 + else project.responsible_unit.sto2_endpoint, + aws_access_key_id=project.responsible_unit.sto4_access + if sto4 + else project.responsible_unit.sto2_access, + aws_secret_access_key=project.responsible_unit.sto4_secret + if sto4 + else project.responsible_unit.sto2_secret, ) - # Delete info - del endpoint_url - del aws_access_key_id - del aws_secret_access_key - # Check if bucket exists try: resource.meta.client.head_bucket(Bucket=project.bucket) @@ -529,9 +491,6 @@ def add_missing_bucket(project_id: str): f"Bucket for project '{project.public_id}' found; Bucket not missing. Will not create bucket." ) - # Collect garbage - gc.collect() - @lost_files_s3_db.command(name="delete") @click.option("--project-id", "-p", type=str, required=True) @@ -555,20 +514,10 @@ def delete_lost_files(project_id: str): # Use sto4 if project created after sto4 info added try: - if use_sto4(unit_object=project.responsible_unit, project_object=project): - flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto4") - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - project.responsible_unit.sto4_endpoint, - project.responsible_unit.sto4_access, - project.responsible_unit.sto4_secret, - ) - else: - flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto2") - endpoint_url, aws_access_key_id, aws_secret_access_key = ( - project.responsible_unit.sto2_endpoint, - project.responsible_unit.sto2_access, - project.responsible_unit.sto2_secret, - ) + sto4: bool = use_sto4(unit_object=project.responsible_unit, project_object=project) + flask.current_app.logger.info( + f"Safespring location for project '{project_id}': {'sto4' if sto4 else 'sto2'}" + ) except S3InfoNotFoundError as err: flask.current_app.logger.error(str(err)) sys.exit(1) @@ -576,16 +525,17 @@ def delete_lost_files(project_id: str): # Connect to S3 resource = session.resource( service_name="s3", - endpoint_url=endpoint_url, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, + endpoint_url=project.responsible_unit.sto4_endpoint + if sto4 + else project.responsible_unit.sto2_endpoint, + aws_access_key_id=project.responsible_unit.sto4_access + if sto4 + else project.responsible_unit.sto2_access, + aws_secret_access_key=project.responsible_unit.sto4_secret + if sto4 + else project.responsible_unit.sto2_secret, ) - # Delete info - del endpoint_url - del aws_access_key_id - del aws_secret_access_key - # Get list of lost files in_db_but_not_in_s3, in_s3_but_not_in_db = list_lost_files_in_project( project=project, s3_resource=resource @@ -624,8 +574,6 @@ def delete_lost_files(project_id: str): flask.current_app.logger.info(f"Files deleted from S3: {len(in_s3_but_not_in_db)}") flask.current_app.logger.info(f"Files deleted from DB: {len(in_db_but_not_in_s3)}") - gc.collect() - @click.command("set-available-to-expired") @flask.cli.with_appcontext