Skip to content

Commit

Permalink
refactored commands
Browse files Browse the repository at this point in the history
  • Loading branch information
@i-oden
i-oden committed Jul 28, 2023
1 parent 206ad91 commit 7f94008
Showing 1 changed file with 52 additions and 104 deletions.
156 changes: 52 additions & 104 deletions dds_web/commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,41 +308,28 @@ def list_lost_files(project_id: str):
# Check which Safespring storage location to use
# Use sto4 if project created after sto4 info added
try:
if use_sto4(unit_object=project.responsible_unit, project_object=project):
flask.current_app.logger.info(
f"Safespring location for project '{project_id}': sto4"
)
endpoint_url, aws_access_key_id, aws_secret_access_key = (
project.responsible_unit.sto4_endpoint,
project.responsible_unit.sto4_access,
project.responsible_unit.sto4_secret,
)
else:
flask.current_app.logger.info(
f"Safespring location for project '{project_id}': sto2"
)
endpoint_url, aws_access_key_id, aws_secret_access_key = (
project.responsible_unit.sto2_endpoint,
project.responsible_unit.sto2_access,
project.responsible_unit.sto2_secret,
)
sto4: bool = use_sto4(unit_object=project.responsible_unit, project_object=project)
flask.current_app.logger.info(
f"Safespring location for project '{project_id}': {'sto4' if sto4 else 'sto2'}"
)
except S3InfoNotFoundError as err:
flask.current_app.logger.error(str(err))
sys.exit(1)

# Connect to S3
resource = session.resource(
service_name="s3",
endpoint_url=endpoint_url,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
endpoint_url=project.responsible_unit.sto4_endpoint
if sto4
else project.responsible_unit.sto2_endpoint,
aws_access_key_id=project.responsible_unit.sto4_access
if sto4
else project.responsible_unit.sto2_access,
aws_secret_access_key=project.responsible_unit.sto4_secret
if sto4
else project.responsible_unit.sto2_secret,
)

# Clean up information
del endpoint_url
del aws_access_key_id
del aws_secret_access_key

# List the lost files
try:
in_db_but_not_in_s3, in_s3_but_not_in_db = list_lost_files_in_project(
Expand Down Expand Up @@ -387,41 +374,28 @@ def list_lost_files(project_id: str):
# Check which Safespring storage location to use
# Use sto4 if roject created after sto4 info added
try:
if use_sto4(unit_object=unit, project_object=proj):
flask.current_app.logger.info(
f"Safespring location for project '{proj.public_id}': sto4"
)
endpoint_url, aws_access_key_id, aws_secret_access_key = (
proj.responsible_unit.sto4_endpoint,
proj.responsible_unit.sto4_access,
proj.responsible_unit.sto4_secret,
)
else:
flask.current_app.logger.info(
f"Safespring location for project '{proj.public_id}': sto2"
)
endpoint_url, aws_access_key_id, aws_secret_access_key = (
proj.responsible_unit.sto2_endpoint,
proj.responsible_unit.sto2_access,
proj.responsible_unit.sto2_secret,
)
sto4: bool = use_sto4(unit_object=unit, project_object=proj)
flask.current_app.logger.info(
f"Safespring location for project '{proj.public_id}': {'sto4' if sto4 else 'sto2'}"
)
except S3InfoNotFoundError as err:
flask.current_app.logger.error(str(err))
continue

# Connect to S3
resource_unit = session.resource(
service_name="s3",
endpoint_url=endpoint_url,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
endpoint_url=proj.responsible_unit.sto4_endpoint
if sto4
else proj.responsible_unit.sto2_endpoint,
aws_access_key_id=proj.responsible_unit.sto4_access
if sto4
else proj.responsible_unit.sto2_access,
aws_secret_access_key=proj.responsible_unit.sto4_secret
if sto4
else proj.responsible_unit.sto2_secret,
)

# Clean up information
del endpoint_url
del aws_access_key_id
del aws_secret_access_key

# List the lost files
try:
in_db_but_not_in_s3, in_s3_but_not_in_db = list_lost_files_in_project(
Expand All @@ -445,9 +419,6 @@ def list_lost_files(project_id: str):
f"Project errors: {num_proj_errors}\n"
)

# Garbage collect
gc.collect()


@lost_files_s3_db.command(name="add-missing-bucket")
@click.option("--project-id", "-p", type=str, required=True)
Expand Down Expand Up @@ -477,37 +448,28 @@ def add_missing_bucket(project_id: str):

# Use sto4 if project created after sto4 info added
try:
if use_sto4(unit_object=project.responsible_unit, project_object=project):
flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto4")
endpoint_url, aws_access_key_id, aws_secret_access_key = (
project.responsible_unit.sto4_endpoint,
project.responsible_unit.sto4_access,
project.responsible_unit.sto4_secret,
)
else:
flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto2")
endpoint_url, aws_access_key_id, aws_secret_access_key = (
project.responsible_unit.sto2_endpoint,
project.responsible_unit.sto2_access,
project.responsible_unit.sto2_secret,
)
sto4 = use_sto4(unit_object=project.responsible_unit, project_object=project)
flask.current_app.logger.info(
f"Safespring location for project '{project_id}': {'sto4' if sto4 else 'sto2'}"
)
except S3InfoNotFoundError as err:
flask.current_app.logger.error(str(err))
sys.exit(1)

# Connect to S3
resource = session.resource(
service_name="s3",
endpoint_url=endpoint_url,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
endpoint_url=project.responsible_unit.sto4_endpoint
if sto4
else project.responsible_unit.sto2_endpoint,
aws_access_key_id=project.responsible_unit.sto4_access
if sto4
else project.responsible_unit.sto2_access,
aws_secret_access_key=project.responsible_unit.sto4_secret
if sto4
else project.responsible_unit.sto2_secret,
)

# Delete info
del endpoint_url
del aws_access_key_id
del aws_secret_access_key

# Check if bucket exists
try:
resource.meta.client.head_bucket(Bucket=project.bucket)
Expand All @@ -529,9 +491,6 @@ def add_missing_bucket(project_id: str):
f"Bucket for project '{project.public_id}' found; Bucket not missing. Will not create bucket."
)

# Collect garbage
gc.collect()


@lost_files_s3_db.command(name="delete")
@click.option("--project-id", "-p", type=str, required=True)
Expand All @@ -555,37 +514,28 @@ def delete_lost_files(project_id: str):

# Use sto4 if project created after sto4 info added
try:
if use_sto4(unit_object=project.responsible_unit, project_object=project):
flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto4")
endpoint_url, aws_access_key_id, aws_secret_access_key = (
project.responsible_unit.sto4_endpoint,
project.responsible_unit.sto4_access,
project.responsible_unit.sto4_secret,
)
else:
flask.current_app.logger.info(f"Safespring location for project '{project_id}': sto2")
endpoint_url, aws_access_key_id, aws_secret_access_key = (
project.responsible_unit.sto2_endpoint,
project.responsible_unit.sto2_access,
project.responsible_unit.sto2_secret,
)
sto4: bool = use_sto4(unit_object=project.responsible_unit, project_object=project)
flask.current_app.logger.info(
f"Safespring location for project '{project_id}': {'sto4' if sto4 else 'sto2'}"
)
except S3InfoNotFoundError as err:
flask.current_app.logger.error(str(err))
sys.exit(1)

# Connect to S3
resource = session.resource(
service_name="s3",
endpoint_url=endpoint_url,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
endpoint_url=project.responsible_unit.sto4_endpoint
if sto4
else project.responsible_unit.sto2_endpoint,
aws_access_key_id=project.responsible_unit.sto4_access
if sto4
else project.responsible_unit.sto2_access,
aws_secret_access_key=project.responsible_unit.sto4_secret
if sto4
else project.responsible_unit.sto2_secret,
)

# Delete info
del endpoint_url
del aws_access_key_id
del aws_secret_access_key

# Get list of lost files
in_db_but_not_in_s3, in_s3_but_not_in_db = list_lost_files_in_project(
project=project, s3_resource=resource
Expand Down Expand Up @@ -624,8 +574,6 @@ def delete_lost_files(project_id: str):
flask.current_app.logger.info(f"Files deleted from S3: {len(in_s3_but_not_in_db)}")
flask.current_app.logger.info(f"Files deleted from DB: {len(in_db_but_not_in_s3)}")

gc.collect()


@click.command("set-available-to-expired")
@flask.cli.with_appcontext
Expand Down

0 comments on commit 7f94008

Please sign in to comment.