New Feature of Emptying S3 Bucket (preferably used before the deploy to s3)

.gitignore

@@ -10,6 +10,8 @@ buildNumber.properties
 .project
 .classpath
 .settings
+.factorypath
+.java-version
 ### Idea IDE
 .idea
-*.iml
+*.iml

README.md

@@ -195,6 +195,7 @@ Add this to the `<plugins>` section of your pom.xml:
     </configuration>
 </plugin>
 ```
+
 Notes:
 * If you don't access AWS via an https proxy then leave those configuration settings out.
 
@@ -204,6 +205,60 @@ export AWS_SECRET_ACCESS_KEY=<your_secret>
 mvn package aws:deployS3
 ```
 
+### Empty an S3 bucket
+* Empties an S3 bucket for future deploys to s3
+* Configurable to have a list of excluded regular expressions to ignore certain s3 objects
+
+Add this to the `<plugins>` section of your pom.xml:
+
+```xml
+<plugin>
+    <groupId>com.github.davidmoten</groupId>
+    <artifactId>aws-maven-plugin</artifactId>
+    <version>[LATEST_VERSION]</version>
+    <configuration>
+        <!-- Optional authentication configuration. The default credential provider chain is used if the configuration is omitted -->
+        <!-- if you have serverId then exclude awsAccessKey and awsSecretAccessKey parameters -->
+        <serverId>aws</serverId>
+        <!-- if you omit serverId then put explicit keys here as below -->
+        <awsAccessKey>YOUR_AWS_ACCESS_KEY</awsAccessKey>
+        <awsSecretAccessKey>YOUR_AWS_SECRET_ACCESS_KEY</awsSecretAccessKey>
+
+        <!-- The default region provider chain is used if the region is omitted -->
+        <region>ap-southeast-2</region>
+
+        <bucketName>the_bucket</bucketName>
+
+        <!-- The maximum number of objects you want to fetch from s3 at a time -->
+        <!-- Max is 1000. Default is 900 -->
+        <maxObjects>60<maxObjects>
+
+        <!-- optional: java regex to exclude certain s3 objects that match any of these patterns -->
+        <excludes>
+            <exclude>\\\\*.jpg</exclude>
+            <exclude>config/*</exclude>
+        </excludes>
+
+        <!-- optional: will not actually delete objects in s3 -->
+        <!-- Used for testing purposes -->
+        <!-- default is false -->
+        <dryRun>false</dryRun>
+
+        <!-- optional proxy config -->
+        <httpsProxyHost>proxy.mycompany</httpsProxyHost>
+        <httpsProxyPort>8080</httpsProxyPort>
+        <httpsProxyUsername>user</httpsProxyUsername>
+        <httpsProxyPassword>pass</httpsProxyPassword>
+    </configuration>
+</plugin>
+```
+
+and call 
+
+```bash
+mvn package aws:emptyS3
+```
+
 ### Create/Update CloudfFormation stack
 
 To create or update a stack in CloudFormation (bulk create/modify resources in AWS using a declarative definition) specify the name of the stack, the template and its parameters to the plugin as below.

src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucket.java

@@ -0,0 +1,151 @@
+package com.github.davidmoten.aws.maven;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
+import com.amazonaws.services.s3.model.DeleteObjectsResult;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.maven.plugin.logging.Log;
+
+final class S3EmptyBucket {
+
+    private final Log log;
+    private final AmazonS3 s3Client;
+
+    S3EmptyBucket(Log log, AmazonS3 s3Client) {
+        this.log = log;
+        this.s3Client = s3Client;
+    }
+
+    public void empty(String bucketName, List<String> excludes, boolean isDryRun, int maxKeys) {
+
+
+        /* 
+        * Successfully return if the bucket does not exist so that other
+        * maven jobs can create the bucket and/or upload content to it first.
+        */ 
+        if (!s3Client.doesBucketExistV2(bucketName)) {
+            return;
+        }
+
+        validateRegex(excludes);
+
+        try {
+
+            /* S3 OBJECT FETCHING */
+
+            List<KeyVersion> bucketObjectKeys = new ArrayList<>();
+            int excludedObjectCount = 0;
+
+            // Request 30 objects at a time from the bucket
+            ListObjectsV2Request req = new ListObjectsV2Request()
+                    .withBucketName(bucketName)
+                    .withMaxKeys(maxKeys);
+
+            ListObjectsV2Result result;
+
+            do {
+                result = s3Client.listObjectsV2(req);
+
+                List<S3ObjectSummary> summaries = result.getObjectSummaries();
+
+                // filter + map the keys we want to delete based on exclusion list
+                List<KeyVersion> keys = filterS3Objects(summaries, excludes);
+
+                // count how many objects were excluded.
+                excludedObjectCount += summaries.size() - keys.size();
+
+                bucketObjectKeys.addAll(keys);
+
+                // If there are more than maxKeys keys in the bucket, get a continuation token
+                // and list the next objects.
+                String token = result.getNextContinuationToken();
+                req.setContinuationToken(token);
+
+            } while (result.isTruncated());
+
+            log.info(String.format("Found %d objects (excluding %d) from bucket %s. ", bucketObjectKeys.size(), excludedObjectCount, bucketName));
+
+            /* S3 OBJECT DELETION */
+
+            if (bucketObjectKeys.isEmpty()) {
+                log.info("No objects to remove from bucket " + bucketName + "!");
+            } else {
+                if (!isDryRun) {
+
+                    // Delete the objects.
+                    DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName)
+                        .withKeys(bucketObjectKeys)
+                        .withQuiet(false);
+
+                    // Verify that the objects were deleted successfully.
+                    DeleteObjectsResult delObjRes = s3Client.deleteObjects(multiObjectDeleteRequest);
+                    int successfulDeletes = delObjRes.getDeletedObjects().size();
+                    log.info(successfulDeletes + " objects successfully deleted.");
+                } else {
+                    log.info("[Dry Run] Deleting the following objects:");
+                    for (KeyVersion kv: bucketObjectKeys) {
+                        log.info(String.format("[Dry Run] - will delete %s/%s", bucketName, kv.getKey()));
+                    }   
+                }
+            }
+
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
+    private void validateRegex(List<String> regexes) {
+
+        if (regexes == null) {
+            return;
+        }
+
+        for (String regex: regexes) {
+            try {
+                Pattern.compile(regex);
+            } catch (PatternSyntaxException e) {
+                throw new RuntimeException("Invalid regular expression: " + regex);
+            }
+        }
+    }
+
+    @VisibleForTesting
+    static boolean excludeObjectFromDelete(String objKey, List<String> excludes) {
+
+        if (excludes == null) {
+            return false;
+        }
+
+        // check the exclusion regexes
+        for (String exclude: excludes) {
+            final Matcher m = Pattern.compile(exclude).matcher(objKey);
+            if (m.find()) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    @VisibleForTesting
+    static List<KeyVersion> filterS3Objects(List<S3ObjectSummary> objects, List<String> excludes){
+        return objects
+                .stream()
+                .filter(s -> !excludeObjectFromDelete(s.getKey(), excludes))
+                .map(s -> new KeyVersion(s.getKey()))
+                .collect(Collectors.toList());
+    }
+}

src/main/java/com/github/davidmoten/aws/maven/S3EmptyBucketMojo.java

@@ -0,0 +1,36 @@
+package com.github.davidmoten.aws.maven;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import org.apache.maven.plugins.annotations.Mojo;
+import org.apache.maven.plugins.annotations.Parameter;
+import java.util.List;
+
+@Mojo(name = "emptyS3")
+public final class S3EmptyBucketMojo extends AbstractDeployAwsMojo<AmazonS3ClientBuilder, AmazonS3> {
+
+    @Parameter(property = "region")
+    private String region;
+
+    @Parameter(property = "bucketName")
+    private String bucketName;
+
+    @Parameter(property = "excludes")
+    private List<String> excludes;
+
+    @Parameter(property = "maxObjects", defaultValue = "900")
+    private int maxObjects;
+
+    @Parameter(property = "dryRun", defaultValue = "false")
+    private boolean isDryRun;
+
+    public S3EmptyBucketMojo() {
+        super(AmazonS3ClientBuilder.standard());
+    }
+
+    @Override
+    protected void execute(AmazonS3 s3Client) {
+        S3EmptyBucket emptyBucket = new S3EmptyBucket(getLog(), s3Client);
+        emptyBucket.empty(bucketName, excludes, isDryRun, maxObjects);
+    }
+}

src/test/java/com/github/davidmoten/aws/maven/S3EmptyBucketTest.java

@@ -0,0 +1,86 @@
+package com.github.davidmoten.aws.maven;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+
+import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import org.junit.Test;
+
+public final class S3EmptyBucketTest {
+
+    @Test
+    public void testExcludeObjectFromDelete() {
+
+        String key = "foo/bar/test.jpg";
+
+        List<String> regexes = Arrays.asList(
+            "\\\\*.jpg"
+        );
+
+        boolean result = S3EmptyBucket.excludeObjectFromDelete(key, regexes);
+
+        assertTrue(result);
+
+        List<String> regexes2 = Arrays.asList(
+            "bar/*"
+        );
+
+        boolean result2 = S3EmptyBucket.excludeObjectFromDelete(key, regexes2);
+
+        assertTrue(result2);
+    }
+
+    @Test
+    public void testFilterS3Objects() {
+
+        List<String> regexes = Arrays.asList(
+            "config/*"
+        );
+
+        S3ObjectSummary o1 = new S3ObjectSummary();
+        o1.setKey("config/config.json");
+        o1.setBucketName("test");
+
+        S3ObjectSummary o2 = new S3ObjectSummary();
+        o2.setKey("a.js");
+        o2.setBucketName("test");
+
+        S3ObjectSummary o3 = new S3ObjectSummary();
+        o3.setKey("b.html");
+        o3.setBucketName("test");
+
+        S3ObjectSummary o4 = new S3ObjectSummary();
+        o4.setKey("asset/a.jpg");
+        o4.setBucketName("test");
+
+        S3ObjectSummary o5 = new S3ObjectSummary();
+        o5.setKey("config/other.json");
+        o5.setBucketName("test");
+
+        List<S3ObjectSummary> summaries = Arrays.asList(
+            o1, o2, o3, o4, o5
+        );
+
+        List<KeyVersion> keyVersions = S3EmptyBucket.filterS3Objects(summaries, regexes);
+
+        List<String> expected = Arrays.asList(
+            "a.js",
+            "b.html",
+            "asset/a.jpg"
+        );
+
+        for (KeyVersion kv: keyVersions) {
+            assertTrue(expected.contains(kv.getKey()));
+        }
+
+        assertEquals(expected.size(), keyVersions.size());
+
+    }
+
+
+}