diff --git a/.gitignore b/.gitignore index 9c45b24..33e7db7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ buildNumber.properties .project .classpath .settings +.factorypath +.java-version ### Idea IDE .idea -*.iml \ No newline at end of file +*.iml diff --git a/README.md b/README.md index f73b423..d299d0e 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ Add this to the `` section of your pom.xml: ``` + Notes: * If you don't access AWS via an https proxy then leave those configuration settings out. @@ -204,6 +205,60 @@ export AWS_SECRET_ACCESS_KEY= mvn package aws:deployS3 ``` +### Empty an S3 bucket +* Empties an S3 bucket for future deploys to s3 +* Configurable to have a list of excluded regular expressions to ignore certain s3 objects + +Add this to the `` section of your pom.xml: + +```xml + + com.github.davidmoten + aws-maven-plugin + [LATEST_VERSION] + + + + aws + + YOUR_AWS_ACCESS_KEY + YOUR_AWS_SECRET_ACCESS_KEY + + + ap-southeast-2 + + the_bucket + + + + 60 + + + + \\\\*.jpg + config/* + + + + + + false + + + proxy.mycompany + 8080 + user + pass + + +``` + +and call + +```bash +mvn package aws:emptyS3 +``` + ### Create/Update CloudfFormation stack To create or update a stack in CloudFormation (bulk create/modify resources in AWS using a declarative definition) specify the name of the stack, the template and its parameters to the plugin as below. diff --git a/src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucket.java b/src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucket.java new file mode 100644 index 0000000..70719e9 --- /dev/null +++ b/src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucket.java @@ -0,0 +1,151 @@ +package com.github.davidmoten.aws.maven; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.google.common.annotations.VisibleForTesting; + +import org.apache.maven.plugin.logging.Log; + +final class S3EmptyBucket { + + private final Log log; + private final AmazonS3 s3Client; + + S3EmptyBucket(Log log, AmazonS3 s3Client) { + this.log = log; + this.s3Client = s3Client; + } + + public void empty(String bucketName, List excludes, boolean isDryRun, int maxKeys) { + + + /* + * Successfully return if the bucket does not exist so that other + * maven jobs can create the bucket and/or upload content to it first. + */ + if (!s3Client.doesBucketExistV2(bucketName)) { + return; + } + + validateRegex(excludes); + + try { + + /* S3 OBJECT FETCHING */ + + List bucketObjectKeys = new ArrayList<>(); + int excludedObjectCount = 0; + + // Request 30 objects at a time from the bucket + ListObjectsV2Request req = new ListObjectsV2Request() + .withBucketName(bucketName) + .withMaxKeys(maxKeys); + + ListObjectsV2Result result; + + do { + result = s3Client.listObjectsV2(req); + + List summaries = result.getObjectSummaries(); + + // filter + map the keys we want to delete based on exclusion list + List keys = filterS3Objects(summaries, excludes); + + // count how many objects were excluded. + excludedObjectCount += summaries.size() - keys.size(); + + bucketObjectKeys.addAll(keys); + + // If there are more than maxKeys keys in the bucket, get a continuation token + // and list the next objects. + String token = result.getNextContinuationToken(); + req.setContinuationToken(token); + + } while (result.isTruncated()); + + log.info(String.format("Found %d objects (excluding %d) from bucket %s. ", bucketObjectKeys.size(), excludedObjectCount, bucketName)); + + /* S3 OBJECT DELETION */ + + if (bucketObjectKeys.isEmpty()) { + log.info("No objects to remove from bucket " + bucketName + "!"); + } else { + if (!isDryRun) { + + // Delete the objects. + DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName) + .withKeys(bucketObjectKeys) + .withQuiet(false); + + // Verify that the objects were deleted successfully. + DeleteObjectsResult delObjRes = s3Client.deleteObjects(multiObjectDeleteRequest); + int successfulDeletes = delObjRes.getDeletedObjects().size(); + log.info(successfulDeletes + " objects successfully deleted."); + } else { + log.info("[Dry Run] Deleting the following objects:"); + for (KeyVersion kv: bucketObjectKeys) { + log.info(String.format("[Dry Run] - will delete %s/%s", bucketName, kv.getKey())); + } + } + } + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + + private void validateRegex(List regexes) { + + if (regexes == null) { + return; + } + + for (String regex: regexes) { + try { + Pattern.compile(regex); + } catch (PatternSyntaxException e) { + throw new RuntimeException("Invalid regular expression: " + regex); + } + } + } + + @VisibleForTesting + static boolean excludeObjectFromDelete(String objKey, List excludes) { + + if (excludes == null) { + return false; + } + + // check the exclusion regexes + for (String exclude: excludes) { + final Matcher m = Pattern.compile(exclude).matcher(objKey); + if (m.find()) { + return true; + } + } + + return false; + } + + @VisibleForTesting + static List filterS3Objects(List objects, List excludes){ + return objects + .stream() + .filter(s -> !excludeObjectFromDelete(s.getKey(), excludes)) + .map(s -> new KeyVersion(s.getKey())) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucketMojo.java b/src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucketMojo.java new file mode 100644 index 0000000..851bd63 --- /dev/null +++ b/src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucketMojo.java @@ -0,0 +1,36 @@ +package com.github.davidmoten.aws.maven; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.apache.maven.plugins.annotations.Mojo; +import org.apache.maven.plugins.annotations.Parameter; +import java.util.List; + +@Mojo(name = "emptyS3") +public final class S3EmptyBucketMojo extends AbstractDeployAwsMojo { + + @Parameter(property = "region") + private String region; + + @Parameter(property = "bucketName") + private String bucketName; + + @Parameter(property = "excludes") + private List excludes; + + @Parameter(property = "maxObjects", defaultValue = "900") + private int maxObjects; + + @Parameter(property = "dryRun", defaultValue = "false") + private boolean isDryRun; + + public S3EmptyBucketMojo() { + super(AmazonS3ClientBuilder.standard()); + } + + @Override + protected void execute(AmazonS3 s3Client) { + S3EmptyBucket emptyBucket = new S3EmptyBucket(getLog(), s3Client); + emptyBucket.empty(bucketName, excludes, isDryRun, maxObjects); + } +} diff --git a/src/test/java/com/github/davidmoten/aws/maven/S3EmptyBucketTest.java b/src/test/java/com/github/davidmoten/aws/maven/S3EmptyBucketTest.java new file mode 100644 index 0000000..1687281 --- /dev/null +++ b/src/test/java/com/github/davidmoten/aws/maven/S3EmptyBucketTest.java @@ -0,0 +1,86 @@ +package com.github.davidmoten.aws.maven; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; + +import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +import org.junit.Test; + +public final class S3EmptyBucketTest { + + @Test + public void testExcludeObjectFromDelete() { + + String key = "foo/bar/test.jpg"; + + List regexes = Arrays.asList( + "\\\\*.jpg" + ); + + boolean result = S3EmptyBucket.excludeObjectFromDelete(key, regexes); + + assertTrue(result); + + List regexes2 = Arrays.asList( + "bar/*" + ); + + boolean result2 = S3EmptyBucket.excludeObjectFromDelete(key, regexes2); + + assertTrue(result2); + } + + @Test + public void testFilterS3Objects() { + + List regexes = Arrays.asList( + "config/*" + ); + + S3ObjectSummary o1 = new S3ObjectSummary(); + o1.setKey("config/config.json"); + o1.setBucketName("test"); + + S3ObjectSummary o2 = new S3ObjectSummary(); + o2.setKey("a.js"); + o2.setBucketName("test"); + + S3ObjectSummary o3 = new S3ObjectSummary(); + o3.setKey("b.html"); + o3.setBucketName("test"); + + S3ObjectSummary o4 = new S3ObjectSummary(); + o4.setKey("asset/a.jpg"); + o4.setBucketName("test"); + + S3ObjectSummary o5 = new S3ObjectSummary(); + o5.setKey("config/other.json"); + o5.setBucketName("test"); + + List summaries = Arrays.asList( + o1, o2, o3, o4, o5 + ); + + List keyVersions = S3EmptyBucket.filterS3Objects(summaries, regexes); + + List expected = Arrays.asList( + "a.js", + "b.html", + "asset/a.jpg" + ); + + for (KeyVersion kv: keyVersions) { + assertTrue(expected.contains(kv.getKey())); + } + + assertEquals(expected.size(), keyVersions.size()); + + } + + +}