diff --git a/batch-setup/batch_setup.py b/batch-setup/batch_setup.py index a5a7689..d3f977c 100644 --- a/batch-setup/batch_setup.py +++ b/batch-setup/batch_setup.py @@ -148,34 +148,6 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen boto_iam, boto_ec2, instanceRoleName, instanceRoleName) print("Using ECS instance profile %s" % instanceProfileArn) - # Create the spot fleet role - # https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html - spotIamFleetRoleName = "AmazonEC2SpotFleetRole" - spotIamRoleDocument = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "spotfleet.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] - } - spotIamFleetRoleArn = get_or_create_role( - boto_iam, spotIamFleetRoleName, spotIamRoleDocument) - print("Using EC2 Spot Fleet role %s" % spotIamFleetRoleArn) - - spotFleetTaggingRolePolicy = find_policy( - boto_iam, 'AmazonEC2SpotFleetTaggingRole') - boto_iam.attach_role_policy( - PolicyArn=spotFleetTaggingRolePolicy['Arn'], - RoleName=spotIamFleetRoleName, - ) - print("Attached Spot Fleet Tagging Role to EC2 Spot Fleet role %s" - % spotIamFleetRoleArn) - # Create a compute environment for raw tile rendering try: response = boto_batch.describe_compute_environments( @@ -192,7 +164,7 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen state="ENABLED", serviceRole=serviceRoleArn, computeResources=dict( - type='SPOT', + type='EC2', minvCpus=0, maxvCpus=max_vcpus, desiredvCpus=0, @@ -205,7 +177,6 @@ def batch_setup(region_name, run_id, vpc_id, securityGroupIds, computeEnvironmen 'cost_resource_group': run_id, }, bidPercentage=60, - spotIamFleetRole=spotIamFleetRoleArn, subnets=subnet_ids, securityGroupIds=securityGroupIds, ) diff --git a/batch-setup/make_meta_tiles.py b/batch-setup/make_meta_tiles.py index 3a22594..450779a 100644 --- a/batch-setup/make_meta_tiles.py +++ b/batch-setup/make_meta_tiles.py @@ -378,16 +378,17 @@ def viable_container_overrides(mem_mb): :return: the amount of mem you need to request for AWS batch to honor it, the amount of vcpus you must request """ if mem_mb < 512: - return 512 + return 512, 1 if mem_mb % 1024 == 0: - return mem_mb + return mem_mb, 1 - # truncate number / 1024, then bump by 1 - desired_mem_mb = (int(mem_mb) / 1024 + 1) * 1024 + mem_gb_truncated = mem_mb / 1024 + next_gb = mem_gb_truncated + 1 + desired_mem_mb = next_gb * 1024 max_mem_per_vcpu = 8 * 1024 - vcpus = (desired_mem_mb - 1) / max_mem_per_vcpu + 1 + vcpus = 1 + (desired_mem_mb - 1)/max_mem_per_vcpu return desired_mem_mb, vcpus @@ -413,7 +414,7 @@ def enqueue_tiles(config_file, tile_list_file, check_metatile_exists, tile_speci mem_mb = int(tile_specifier.get_mem_reqs_mb(coord_line)) adjusted_mem = mem_mb * mem_multiplier - # now that we know what we think we want, pick something AWS actually supports + # now that we know what we want, pick something AWS actually supports viable_mem_request, required_min_cpus = viable_container_overrides(adjusted_mem) update_container_overrides(cfg, viable_mem_request, mem_max, required_min_cpus)