Merge pull request #6 from hms-dbmi/QA

1.1 Final

.gitignore

@@ -0,0 +1,19 @@
+/target/
+target/
+
+.classpath
+
+.project
+
+.settings/org.eclipse.jdt.core.prefs
+
+.settings/org.eclipse.m2e.core.prefs
+*.class
+
+*.component
+
+*.class
+
+*.class
+
+*.xml

pom.xml

@@ -0,0 +1,114 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>edu.harvard.hms.dbmi.bd2k.irct</groupId>
+	<artifactId>IRCT-EXT</artifactId>
+	<version>1.1</version>
+	<name>INTER-RESOURCE COMMUNICATION TOOL : EXTENSION</name>
+
+	<dependencies>
+		<!-- AWS -->
+		<dependency>
+			<groupId>com.amazonaws</groupId>
+			<artifactId>aws-java-sdk</artifactId>
+			<version>1.11.7</version>
+		</dependency>
+		<!-- JODA Time -->
+		<dependency>
+			<groupId>joda-time</groupId>
+			<artifactId>joda-time</artifactId>
+			<version>2.9.4</version>
+		</dependency>
+		<!--JSON -->
+		<dependency>
+			<groupId>com.fasterxml.jackson.dataformat</groupId>
+			<artifactId>jackson-dataformat-cbor</artifactId>
+			<version>2.8.1</version>
+		</dependency>
+
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-databind</artifactId>
+			<version>2.8.1</version>
+		</dependency>
+		<dependency>
+			<groupId>com.fasterxml.jackson.core</groupId>
+			<artifactId>jackson-core</artifactId>
+			<version>2.8.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.glassfish</groupId>
+			<artifactId>javax.json</artifactId>
+			<version>1.0.4</version>
+		</dependency>
+		<dependency>
+			<groupId>javax</groupId>
+			<artifactId>javaee-api</artifactId>
+			<version>7.0</version>
+			<scope>provided</scope>
+		</dependency>
+		<!-- IRCT API -->
+		<dependency>
+			<groupId>edu.harvard.hms.dbmi.bd2k.irct</groupId>
+			<artifactId>IRCT-API</artifactId>
+			<version>1.1</version>
+		</dependency>
+		<!-- JAVA EE -->
+		<dependency>
+			<groupId>javax</groupId>
+			<artifactId>javaee-api</artifactId>
+			<version>7.0</version>
+			<scope>provided</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<finalName>IRCT-EXT</finalName>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-eclipse-plugin</artifactId>
+				<configuration>
+					<downloadSources>true</downloadSources>
+					<downloadJavadocs>true</downloadJavadocs>
+					<source>1.7</source>
+					<target>1.7</target>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<configuration>
+					<source>1.7</source>
+					<target>1.7</target>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-dependency-plugin</artifactId>
+				<executions>
+					<execution>
+						<id>copy-dependencies</id>
+						<phase>package</phase>
+						<goals>
+							<goal>copy-dependencies</goal>
+						</goals>
+						<configuration>
+							<includeScope>runtime</includeScope>
+							<outputDirectory>${project.build.directory}/dependency-jars/</outputDirectory>
+							<source>1.7</source>
+							<target>1.7</target>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>

src/main/java/edu/harvard/hms/dbmi/bd2k/irct/aws/event/result/S3AfterGetResult.java

@@ -0,0 +1,105 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+package edu.harvard.hms.dbmi.bd2k.irct.aws.event.result;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import edu.harvard.hms.dbmi.bd2k.irct.event.result.AfterGetResult;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.Result;
+
+public class S3AfterGetResult implements AfterGetResult {
+
+	private AmazonS3 s3client;
+	private String bucketName;
+	private Log log;
+	private String irctSaveLocation;
+	private String s3Folder;
+
+	@Override
+	public void init(Map<String, String> parameters) {
+		log = LogFactory.getLog("AWS S3 Monitoring");
+		bucketName = parameters.get("Bucket Name");
+		irctSaveLocation = parameters.get("resultDataFolder");
+		s3Folder = parameters.get("s3Folder");
+
+		s3client = new AmazonS3Client(new InstanceProfileCredentialsProvider());
+
+	}
+
+	@Override
+	public void fire(Result result) {
+		if (!result.getResultSetLocation().startsWith("S3://")) {
+			File temp = new File(result.getResultSetLocation());
+			if (temp.exists()) {
+				return;
+			} else {
+				result.setResultSetLocation("S3://"
+						+ s3Folder
+						+ result.getResultSetLocation().replaceAll(
+								irctSaveLocation, ""));
+			}
+		}
+		String location = result.getResultSetLocation().substring(5);
+		// List the files in that bucket path
+		try {
+
+			final ListObjectsV2Request req = new ListObjectsV2Request()
+					.withBucketName(bucketName).withPrefix(location);
+
+			// Loop Through all the files
+			ListObjectsV2Result s3Files;
+			do {
+				s3Files = s3client.listObjectsV2(req);
+				for (S3ObjectSummary objectSummary : s3Files
+						.getObjectSummaries()) {
+					// Download the files to the directory specified
+					String keyName = objectSummary.getKey();
+					String fileName = irctSaveLocation
+							+ keyName.replace(location, "");
+					log.info("Downloading: " + keyName + " --> " + fileName);
+					s3client.getObject(
+							new GetObjectRequest(bucketName, keyName),
+							new File(fileName));
+				}
+				req.setContinuationToken(s3Files.getNextContinuationToken());
+			} while (s3Files.isTruncated() == true);
+
+			// Update the result set id
+			result.setResultSetLocation(irctSaveLocation + "/"
+					+ location.replace(s3Folder, ""));
+		} catch (AmazonServiceException ase) {
+			log.warn("Caught an AmazonServiceException, which "
+					+ "means your request made it "
+					+ "to Amazon S3, but was rejected with an error response"
+					+ " for some reason.");
+			log.warn("Error Message:    " + ase.getMessage());
+			log.warn("HTTP Status Code: " + ase.getStatusCode());
+			log.warn("AWS Error Code:   " + ase.getErrorCode());
+			log.warn("Error Type:       " + ase.getErrorType());
+			log.warn("Request ID:       " + ase.getRequestId());
+		} catch (AmazonClientException ace) {
+			log.warn("Caught an AmazonClientException, which "
+					+ "means the client encountered "
+					+ "an internal error while trying to "
+					+ "communicate with S3, "
+					+ "such as not being able to access the network.");
+			log.warn("Error Message: " + ace.getMessage());
+		}
+	}
+
+}

src/main/java/edu/harvard/hms/dbmi/bd2k/irct/aws/event/result/S3AfterSaveResult.java

@@ -0,0 +1,94 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+package edu.harvard.hms.dbmi.bd2k.irct.aws.event.result;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+
+import edu.harvard.hms.dbmi.bd2k.irct.event.action.AfterExecutionPlan;
+import edu.harvard.hms.dbmi.bd2k.irct.exception.ResourceInterfaceException;
+import edu.harvard.hms.dbmi.bd2k.irct.executable.Executable;
+import edu.harvard.hms.dbmi.bd2k.irct.executable.ExecutableStatus;
+import edu.harvard.hms.dbmi.bd2k.irct.model.result.Result;
+import edu.harvard.hms.dbmi.bd2k.irct.model.security.SecureSession;
+
+/**
+ * Copies the results of an execution plan to an AWS S3 bucket. The bucket is
+ * name is set by the Bucket Name parameter. The Executable Status must be set
+ * to COMPLETED in order for the result to be saved remotely.
+ * 
+ * @author Jeremy R. Easton-Marks
+ *
+ */
+public class S3AfterSaveResult implements AfterExecutionPlan {
+	private AmazonS3 s3client;
+	private Log log;
+	private String bucketName;
+	private String s3Folder;
+
+	@Override
+	public void init(Map<String, String> parameters) {
+		log = LogFactory.getLog("AWS S3 Monitoring");
+		bucketName = parameters.get("Bucket Name");
+		s3Folder = parameters.get("s3Folder");
+
+		s3client = new AmazonS3Client(new InstanceProfileCredentialsProvider());
+	}
+
+	@Override
+	public void fire(SecureSession session, Executable executable) {
+
+		try {
+			if (executable.getStatus() != ExecutableStatus.COMPLETED) {
+				return;
+			}
+
+			Result result = executable.getResults();
+			for (File resultFile : result.getData().getFileList()) {
+				String keyName = s3Folder + result.getId() + "/"
+						+ resultFile.getName();
+				// Copy the result into S3 if bucketName is not empty or null
+				s3client.putObject(new PutObjectRequest(bucketName, keyName,
+						resultFile));
+				log.info("Moved " + result.getResultSetLocation() + " to "
+						+ bucketName + "/" + keyName);
+				// Delete File
+				resultFile.delete();
+				log.info("Deleted " + resultFile.getName());
+			}
+			result.setResultSetLocation("S3://" + s3Folder + result.getId());
+
+		} catch (AmazonServiceException ase) {
+			log.warn("Caught an AmazonServiceException, which "
+					+ "means your request made it "
+					+ "to Amazon S3, but was rejected with an error response"
+					+ " for some reason.");
+			log.warn("Error Message:    " + ase.getMessage());
+			log.warn("HTTP Status Code: " + ase.getStatusCode());
+			log.warn("AWS Error Code:   " + ase.getErrorCode());
+			log.warn("Error Type:       " + ase.getErrorType());
+			log.warn("Request ID:       " + ase.getRequestId());
+		} catch (AmazonClientException ace) {
+			log.warn("Caught an AmazonClientException, which "
+					+ "means the client encountered "
+					+ "an internal error while trying to "
+					+ "communicate with S3, "
+					+ "such as not being able to access the network.");
+			log.warn("Error Message: " + ace.getMessage());
+		} catch (ResourceInterfaceException e) {
+			log.warn("Error Message: " + e.getMessage());
+		}
+	}
+
+}

src/main/java/edu/harvard/hms/dbmi/bd2k/irct/monitoring/event/action/MonitoringBeforeProcess.java

@@ -0,0 +1,31 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+package edu.harvard.hms.dbmi.bd2k.irct.monitoring.event.action;
+
+
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import edu.harvard.hms.dbmi.bd2k.irct.event.action.BeforeProcess;
+import edu.harvard.hms.dbmi.bd2k.irct.model.process.IRCTProcess;
+import edu.harvard.hms.dbmi.bd2k.irct.model.security.SecureSession;
+
+public class MonitoringBeforeProcess implements BeforeProcess {
+
+	private Log log;
+
+	public void init(Map<String, String> parameters) {
+		log = LogFactory.getLog("Action Monitoring");
+	}
+
+	@Override
+	public void fire(SecureSession session, IRCTProcess process) {
+		log.info("PROCESS: " + session.getUser().getName() + " : " + process.toString());
+
+	}
+
+
+}