diff --git a/pom.xml b/pom.xml index c3e3147..cb2a06c 100644 --- a/pom.xml +++ b/pom.xml @@ -388,6 +388,21 @@ vecmath 1.6.0-scijava-2 + + org.apache.httpcomponents + httpclient + 4.5.5 + + + org.json + json + 20180130 + + + com.github.lookfirst + sardine + 5.8 + diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/Exception/NextcloudApiException.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/Exception/NextcloudApiException.java new file mode 100644 index 0000000..3b95c92 --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/Exception/NextcloudApiException.java @@ -0,0 +1,13 @@ +package org.seagrid.desktop.connectors.NextcloudStorage.Exception; + +public class NextcloudApiException extends RuntimeException { + private static final long serialVersionUID = 8088239559973590632L; + + public NextcloudApiException(Throwable cause) { + super(cause); + } + + public NextcloudApiException(String message) { + super(message); + } +} diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextCloudFolderdownloadtask.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextCloudFolderdownloadtask.java new file mode 100644 index 0000000..c45a44e --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextCloudFolderdownloadtask.java @@ -0,0 +1,126 @@ +package org.seagrid.desktop.connectors.NextcloudStorage; + +import com.github.sardine.DavResource; +import com.jcraft.jsch.SftpException; +import org.seagrid.desktop.connectors.NextcloudStorage.Exception.NextcloudApiException; +import org.seagrid.desktop.util.SEAGridContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.LinkedList; +import java.util.List; + +public class NextCloudFolderdownloadtask extends NextcloudFileTask { + + private final static Logger logger = LoggerFactory.getLogger(NextCloudFolderdownloadtask.class); + + private String rootpath; + private String remoteDirPath, localDirPath; + private double totalBytesRead = 0, totalSize = 0; + + public NextCloudFolderdownloadtask(String remoteDirPath, String localDirPath) throws IOException { + super(); + rootpath = (isusehttps ? "https" : "http") +"://"+servername+"/"+basepath+ "/" + SEAGridContext.getInstance().getUserName(); + this.remoteDirPath = remoteDirPath; + this.localDirPath = localDirPath; + } + + @Override + protected Boolean call() throws Exception { + calculateNextcloudTotalSize(remoteDirPath); + boolean status = downloadFolder(remoteDirPath, localDirPath); + return status; + } + + /** + * Downloads the folder at the specified remotepath to the rootdownloadirpath + * + * @param remotepath the path in the nextcloud server with respect to the specific folder + * @param rootdownloadirpath the local path in the system where the folder needs be saved + * @return + * @throws IOException + */ + + public boolean downloadFolder(String remotepath, String rootdownloadirpath) throws IOException { + int depth=1; + String newdownloadir = rootdownloadirpath; + File localDir = new File(newdownloadir); + + if(!localDir.exists()){ + localDir.mkdirs(); + } + + String rootpathnew = rootpath + remotepath ; + + int count = 0; + String filepath; + + List retVal= new LinkedList<>(); + List resources; + try { + resources = sardine.list(rootpathnew, depth); + } catch (IOException e) { + throw new NextcloudApiException(e); + } + + for (DavResource res : resources) + { + if(count != 0) { + if(res.equals(".") || res.equals("..")){ + continue; + } + + else if(res.isDirectory()) { + String subFoldername = res.getName(); + String downloadDirtosend = newdownloadir + "/" + subFoldername; + String pathToSend = remotepath + "/" + subFoldername; + downloadFolder(pathToSend,downloadDirtosend); + } + + else { + String filename = res.getName(); + filepath = rootpathnew + "/" + filename; + retVal.add(res.getName()); + InputStream in = null; + if (sardine.exists(filepath)) { + in = sardine.get(filepath); + byte[] buffer = new byte[in.available()]; + File targetFile = new File(newdownloadir + "/" + filename); + OutputStream outStream = new FileOutputStream(targetFile); + int bytesRead = -1; + while ((bytesRead = in.read(buffer)) != -1) { + outStream.write(buffer); + totalBytesRead += bytesRead; + updateMessage("Downloaded " + totalBytesRead + " bytes"); + updateProgress(totalBytesRead, totalSize); + } + in.close(); + outStream.close(); + } + } + } + count ++; + } + return true; + } + + private void calculateNextcloudTotalSize(String remoteDirPath) throws SftpException, IOException { + List davResource = listDirectories(remoteDirPath); + int count = 0; + for (DavResource res : davResource) { + if(count != 0) { + if (res.getName().equals(".") || res.getName().equals("..")) { + continue; + } + if (res.isDirectory()) { + String tempRemoteDir = remoteDirPath + "/" + res.getName(); + calculateNextcloudTotalSize(tempRemoteDir); + } else { + totalSize += res.getContentLength().intValue(); + } + } + count++; + } + } +} diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileDownloadTask.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileDownloadTask.java new file mode 100644 index 0000000..12e018d --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileDownloadTask.java @@ -0,0 +1,49 @@ +package org.seagrid.desktop.connectors.NextcloudStorage; + +import org.seagrid.desktop.connectors.NextcloudStorage.Exception.NextcloudApiException; +import org.seagrid.desktop.util.SEAGridContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; + +public class NextcloudFileDownloadTask extends NextcloudFileTask { + + private final static Logger logger = LoggerFactory.getLogger(NextcloudFileDownloadTask.class); + + private String remoteFilePath, localFilePath; + private String remoterootpath; + + public NextcloudFileDownloadTask(String remoteFilePath, String localFilePath) throws IOException { + super(); + this.remoteFilePath = remoteFilePath; + this.localFilePath = localFilePath; + remoterootpath = (isusehttps ? "https" : "http") + "://" + servername + "/" + basepath + "/" + SEAGridContext.getInstance().getUserName(); + } + + @Override + protected Boolean call() throws Exception { + return downloadFile(localFilePath, remoteFilePath); + } + + public boolean downloadFile(String sourceFile, String destFile) { + String path = remoterootpath + destFile; + File downloadFilepath = new File(sourceFile); + if(!downloadFilepath.getParentFile().exists()) { + downloadFilepath.getParentFile().mkdirs(); + } + try { + InputStream in = sardine.get(path); + byte[] buffer = new byte[in.available()]; + in.read(buffer); + File targetFile = new File(sourceFile); + OutputStream outStream = new FileOutputStream(targetFile); + outStream.write(buffer); + outStream.close(); + in.close(); + return true; + } catch (IOException e) { + throw new NextcloudApiException(e); + } + } +} diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileTask.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileTask.java new file mode 100644 index 0000000..0e7bef1 --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileTask.java @@ -0,0 +1,67 @@ +package org.seagrid.desktop.connectors.NextcloudStorage; + +import com.github.sardine.DavResource; +import com.github.sardine.Sardine; +import com.github.sardine.SardineFactory; +import javafx.concurrent.Task; +import org.seagrid.desktop.connectors.NextcloudStorage.Exception.NextcloudApiException; +import org.seagrid.desktop.util.SEAGridContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.util.List; + +public abstract class NextcloudFileTask extends Task { + private final static Logger logger = LoggerFactory.getLogger(org.seagrid.desktop.connectors.NextcloudStorage.NextcloudFileTask.class); + + protected Sardine sardine = SardineFactory.begin(); + + protected String servername; + protected String basepath; + protected boolean isusehttps; + private String token; + private String rootremotepath; + + public NextcloudFileTask() throws IOException { + servername = SEAGridContext.getInstance().getNextcloudServername(); + basepath = SEAGridContext.getInstance().getDavBasepath(); + isusehttps = SEAGridContext.getInstance().isUseHttps(); + rootremotepath = (isusehttps ? "https" : "http") + "://" + servername + "/" + basepath + "/" + SEAGridContext.getInstance().getUserName(); + token = SEAGridContext.getInstance().getOAuthToken(); + sardine.setCredentials(SEAGridContext.getInstance().getUserName(), token); + sardine.enablePreemptiveAuthentication(SEAGridContext.getInstance().getNextcloudServername()); + } + + /** + * Create the folder at the specified path + * @param remotepath + */ + public void createFolder(String remotepath){ + String path= rootremotepath+remotepath; + + try { + sardine.createDirectory(path); + } catch (IOException e) { + throw new NextcloudApiException(e); + } + } + + public List listDirectories(String remotepath) throws IOException { + String path = rootremotepath + remotepath; + List resources; + if(sardine.exists(path)) { + try { + resources = sardine.list(path, 1); + return resources; + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } +} + + diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileUploadTask.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileUploadTask.java new file mode 100644 index 0000000..e0a0052 --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFileUploadTask.java @@ -0,0 +1,73 @@ +package org.seagrid.desktop.connectors.NextcloudStorage; + +import org.seagrid.desktop.connectors.NextcloudStorage.Exception.NextcloudApiException; +import org.seagrid.desktop.util.SEAGridContext; + +import java.io.*; +import java.util.Map; + +public class NextcloudFileUploadTask extends NextcloudFileTask { + + private String remoterootpath; + private Map uploadFiles; + /** + * Constructor + */ + public NextcloudFileUploadTask(Map uploadFiles) throws IOException { + super(); + this.uploadFiles = uploadFiles; + remoterootpath = (isusehttps ? "https" : "http") + "://" + servername + "/" + basepath + "/" + SEAGridContext.getInstance().getUserName(); + } + + @Override + protected Boolean call() throws Exception { + return uploadToNextcloud(); + } + + public boolean uploadToNextcloud() throws IOException { + for(String remoteFilePath : uploadFiles.keySet()){ + int numberOfFiles = uploadFiles.size(); + int index = 1; + remoteFilePath = remoteFilePath.replace("\\","/"); + File localFile = uploadFiles.get(remoteFilePath); + String localpath = localFile.getPath(); + String remotepath = remoteFilePath; + boolean status = false; + InputStream inputStream = new FileInputStream(localpath); + String path; + try { + //createpath if not exists + String[] segments = remotepath.split("/"); + String appendpath=""; + for(int i = 1; i < segments.length - 1 ; i++) + { + appendpath = appendpath + "/" + segments[i]; + if(!sardine.exists(remoterootpath + "/" + appendpath)) { + createFolder(appendpath); + } + } + path = remoterootpath + remotepath; + long fileSize = localFile.length(); + byte[] buffer = new byte[inputStream.available()]; + int bytesRead = -1; + int totalBytesRead = 0; + double percentCompleted = 0; + while ((bytesRead = inputStream.read(buffer)) != -1) { + totalBytesRead += bytesRead; + sardine.put(path, buffer); + percentCompleted = ((double)totalBytesRead) / fileSize * index / numberOfFiles; + updateMessage("Uploaded " + totalBytesRead + " bytes"); + updateProgress(percentCompleted, 1); + } + status = true; + } catch (IOException e) { + throw new NextcloudApiException(e); + } finally { + inputStream.close(); + return status; + } + } + return true; + } + +} diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFolderUploadTask.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFolderUploadTask.java new file mode 100644 index 0000000..98978c6 --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudFolderUploadTask.java @@ -0,0 +1,77 @@ +package org.seagrid.desktop.connectors.NextcloudStorage; + +import org.seagrid.desktop.connectors.Nextcloud.Exception.NextcloudApiException; +import org.seagrid.desktop.util.SEAGridContext; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class NextcloudFolderUploadTask extends NextcloudFileTask { + + private String remoterootpath; + private String remoteDirPath, localDirPath; + + public NextcloudFolderUploadTask(String remoteDirPath, String localDirPath) throws IOException { + super(); + this.remoteDirPath = remoteDirPath; + this.localDirPath = localDirPath; + remoterootpath = (isusehttps ? "https" : "http") + "://" + servername + "/" + basepath + "/" + SEAGridContext.getInstance().getUserName(); + } + + @Override + protected Boolean call() throws Exception { + return uploadFolder(localDirPath, remoteDirPath); + } + + /** + * Upload the folder at the specified path, if the folder doesn't exist a new folder with + * the specific foldername is created + * + * @param localpath + * @param remotepath + * @throws IOException + */ + public Boolean uploadFolder(String localpath, String remotepath) throws IOException { + String path; + path = remoterootpath + remotepath; + String ip = localpath; + File f = new File(ip); + try { + if (f.exists() && f.isDirectory()) { + //Extract the Foldername from the path + String[] segments = ip.split("/"); + String foldername = segments[segments.length - 1]; + String folderemotepath = remotepath + "/" + foldername; + String checkpath = remoterootpath + "/" + folderemotepath; + + //if the folder doesn't exist in the remote server then create the folder + if (!sardine.exists(checkpath)) { + createFolder(folderemotepath); + } + + File[] listfil = f.listFiles(); + if (listfil != null) { + for (File child : listfil) { + if(child.isDirectory()) { + String childfoldername = child.getName(); + String newremotepath = folderemotepath; + String newlocalpath = localpath + "/" + childfoldername; + uploadFolder(newlocalpath, newremotepath); + } else { + String filename = child.getName(); + String newpath = path + "/" + foldername + "/" + filename; + InputStream input = new FileInputStream(child.getAbsolutePath()); + sardine.put(newpath, input); + input.close(); + } + } + } + } + } catch (IOException e) { + throw new NextcloudApiException(e); + } + return true; + } +} diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudSingleFileUploadTask.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudSingleFileUploadTask.java new file mode 100644 index 0000000..e24026f --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudSingleFileUploadTask.java @@ -0,0 +1,63 @@ +package org.seagrid.desktop.connectors.NextcloudStorage; + +import org.seagrid.desktop.connectors.NextcloudStorage.Exception.NextcloudApiException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +public class NextcloudSingleFileUploadTask extends NextcloudFileTask{ + + private final static Logger logger = LoggerFactory.getLogger(NextcloudSingleFileUploadTask.class); + + private String remoteFilePath, localFilePath; + private String remoterootpath; + + public NextcloudSingleFileUploadTask(String remoteFilePath, String localFilePath) throws IOException { + super(); + this.remoteFilePath = remoteFilePath; + this.localFilePath = localFilePath; + } + + + @Override + protected Boolean call() throws Exception { + return uploadFile(remoteFilePath, localFilePath); + } + + /** + * Upload the inputstream to the remote file path as specified to the nextcloud server + * + * @param localpath + * @param remotepath + * @throws IOException + * + */ + protected boolean uploadFile(String localpath, String remotepath) throws IOException { + boolean status = false; + InputStream inputStream = new FileInputStream(localpath); + String path; + try { + //createpath if not exists + String[] segments = remotepath.split("/"); + String appendpath=""; + for(int i = 1; i < segments.length - 1 ; i++) + { + appendpath = appendpath + "/" + segments[i]; + if(!sardine.exists(remoterootpath + "/" + appendpath)) { + createFolder(appendpath); + } + } + path = remoterootpath + remotepath; + sardine.put(path, inputStream); + status = true; + } catch (IOException e) { + throw new NextcloudApiException(e); + } finally { + inputStream.close(); + return status; + } + } +} diff --git a/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudStorageManager.java b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudStorageManager.java new file mode 100644 index 0000000..7caa120 --- /dev/null +++ b/src/main/java/org/seagrid/desktop/connectors/NextcloudStorage/NextcloudStorageManager.java @@ -0,0 +1,95 @@ +package org.seagrid.desktop.connectors.NextcloudStorage; + +import com.github.sardine.DavResource; +import com.github.sardine.Sardine; +import com.github.sardine.SardineFactory; +import com.jcraft.jsch.JSchException; +import org.seagrid.desktop.connectors.NextcloudStorage.Exception.NextcloudApiException; +import org.seagrid.desktop.util.SEAGridContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +public class NextcloudStorageManager { + private final static Logger logger = LoggerFactory.getLogger(NextcloudStorageManager.class); + + private static NextcloudStorageManager instance; + + protected Sardine sardine = SardineFactory.begin(); + + private String servername; + private String basepath; + private boolean isusehttps; + private String token; + private String rootremotepath; + + public NextcloudStorageManager() throws IOException, JSchException { + connect(); + } + + private void connect() throws JSchException, IOException { + servername = SEAGridContext.getInstance().getNextcloudServername(); + basepath = SEAGridContext.getInstance().getDavBasepath(); + isusehttps = SEAGridContext.getInstance().isUseHttps(); + rootremotepath = (isusehttps ? "https" : "http") + "://" + servername + "/" + basepath + "/" + SEAGridContext.getInstance().getUserName(); + token = SEAGridContext.getInstance().getOAuthToken(); + sardine.setCredentials(SEAGridContext.getInstance().getUserName(), token); + sardine.enablePreemptiveAuthentication(SEAGridContext.getInstance().getNextcloudServername()); + } + + public static NextcloudStorageManager getInstance() throws JSchException, IOException { + if(instance==null) { + instance = new NextcloudStorageManager(); + } + return instance; + } + + public List listDirectories(String remotepath) throws IOException { + String path = rootremotepath + remotepath; + List resources; + if(sardine.exists(path)) { + try { + resources = sardine.list(path, 1); + return resources; + } catch (IOException e) { + e.printStackTrace(); + } + } + return null; + } + + /** + * Creates the folder if the folder is not present at the remote path of the nextcloud storage + * @param remotepath + * @throws IOException + */ + public void createFolderifNotExist(String remotepath) throws IOException { + String[] segments = remotepath.split("/"); + + String tempath = ""; + for (int i = 1; i < segments.length; i++) { + tempath = tempath + "/" + segments[i]; + String path = rootremotepath + tempath; + if (!sardine.exists(path)) { + createFolder(tempath); + } + } + } + + /** + * Create the folder at the specified path + * @param remotepath + */ + public void createFolder(String remotepath) + { + String path= rootremotepath+remotepath; + + try { + sardine.createDirectory(path); + } catch (IOException e) { + throw new NextcloudApiException(e); + } + } +} diff --git a/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/ExperimentCreateController.java b/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/ExperimentCreateController.java index fb83fbf..09b45c2 100644 --- a/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/ExperimentCreateController.java +++ b/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/ExperimentCreateController.java @@ -52,9 +52,9 @@ import org.apache.airavata.model.scheduling.ComputationalResourceSchedulingModel; import org.apache.airavata.model.workspace.Project; import org.apache.thrift.TException; +import org.seagrid.desktop.connectors.NextcloudStorage.NextcloudFileDownloadTask; +import org.seagrid.desktop.connectors.NextcloudStorage.NextcloudFileUploadTask; import org.seagrid.desktop.connectors.airavata.AiravataManager; -import org.seagrid.desktop.connectors.storage.GuiBulkFileUploadTask; -import org.seagrid.desktop.connectors.storage.GuiFileDownloadTask; import org.seagrid.desktop.ui.commons.ImageButton; import org.seagrid.desktop.ui.commons.SEAGridDialogHelper; import org.seagrid.desktop.util.SEAGridContext; @@ -65,10 +65,7 @@ import org.slf4j.LoggerFactory; import java.awt.*; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.PrintWriter; +import java.io.*; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; @@ -243,6 +240,8 @@ public ApplicationInterfaceDescription fromString(String string) { createExperiment(false); } catch (TException e) { SEAGridDialogHelper.showExceptionDialog(e,"Caught Exception",null, "Unable to create experiment"); + } catch (IOException e) { + e.printStackTrace(); } }); @@ -251,6 +250,8 @@ public ApplicationInterfaceDescription fromString(String string) { createExperiment(true); } catch (TException e) { SEAGridDialogHelper.showExceptionDialog(e, "Caught Exception", null, "Unable to create experiment"); + } catch (IOException e) { + e.printStackTrace(); } }); @@ -796,7 +797,7 @@ private String showSelectRemoteFile() throws IOException { return controller.getSelectedFilePath(); } - private void createExperiment(boolean launch) throws TException { + private void createExperiment(boolean launch) throws TException, IOException { if(validateExperimentFields()){ //FIXME Hardcoded value String projectId = ((Project)expCreateProjField.getSelectionModel().getSelectedItem()).getProjectID(); @@ -868,6 +869,7 @@ private void createExperiment(ExperimentModel experimentModel, boolean launch){ } private ExperimentModel assembleExperiment(String remoteDataDir, String experimentDataDir) throws TException { + ExperimentModel experimentModel = new ExperimentModel(); experimentModel.setExperimentName(expCreateNameField.getText()); experimentModel.setDescription(expCreateDescField.getText() == null ? "" : expCreateDescField.getText()); @@ -888,6 +890,7 @@ private ExperimentModel assembleExperiment(String remoteDataDir, String experime userConfigurationDataModel.setAiravataAutoSchedule(false); userConfigurationDataModel.setOverrideManualScheduledParams(false); userConfigurationDataModel.setStorageId(SEAGridContext.getInstance().getGatewayaStorageId()); + userConfigurationDataModel.setExperimentDataDir(remoteDataDirRoot + experimentDataDir); userConfigurationDataModel.setUseUserCRPref(useMyCRAccount.isSelected()); @@ -1026,7 +1029,15 @@ private Service getFileUploadService(Map uploadFiles){ @Override protected Task createTask() { try { - return new GuiBulkFileUploadTask(uploadFiles); + /*for (Map.Entry entry : uploadFiles.entrySet()) { + //System.out.println(entry.getKey() + ":" + entry.getValue().toString()); + Alert alert = new Alert(Alert.AlertType.CONFIRMATION, "Content of the keys" + entry.getKey() + " ?" +"Value of the key"+entry.getValue(), ButtonType.YES, ButtonType.NO, ButtonType.CANCEL); + alert.showAndWait(); + if (alert.getResult() == ButtonType.YES) { + //do stuff + } + }*/ + return new NextcloudFileUploadTask(uploadFiles); } catch (Exception e) { e.printStackTrace(); SEAGridDialogHelper.showExceptionDialogAndWait(e, "Exception Dialog", expCreateInputsGridPane.getScene().getWindow(), @@ -1050,7 +1061,8 @@ private void downloadFile(Path remotePath, String destParentPath){ @Override protected Task createTask() { try { - return new GuiFileDownloadTask(remotePath.toString(), localPath); + //return new GuiFileDownloadTask(remotePath.toString(), localPath); + return new NextcloudFileDownloadTask(remotePath.toString(), localPath); } catch (Exception e) { e.printStackTrace(); SEAGridDialogHelper.showExceptionDialogAndWait(e, "Exception Dialog", expCreateNameField.getScene().getWindow(), diff --git a/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/RemoteFilePickerController.java b/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/RemoteFilePickerController.java index 00e9589..6a00b9f 100644 --- a/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/RemoteFilePickerController.java +++ b/src/main/java/org/seagrid/desktop/ui/experiment/create/controller/RemoteFilePickerController.java @@ -20,6 +20,7 @@ */ package org.seagrid.desktop.ui.experiment.create.controller; +import com.github.sardine.DavResource; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.SftpException; @@ -37,16 +38,18 @@ import javafx.scene.input.TransferMode; import javafx.scene.layout.HBox; import javafx.stage.Stage; -import org.seagrid.desktop.connectors.storage.StorageManager; +import org.seagrid.desktop.connectors.NextcloudStorage.NextcloudStorageManager; import org.seagrid.desktop.ui.commons.SEAGridDialogHelper; import org.seagrid.desktop.ui.storage.model.FileListModel; import org.seagrid.desktop.util.SEAGridContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; +import java.util.List; import java.util.Vector; public class RemoteFilePickerController { @@ -102,7 +105,7 @@ private void initialiseColumnWidths(){ fbRemoteFileTblLastMod.prefWidthProperty().bind(fbRemoteFileTable.widthProperty().divide(3)); } - private void initializeRemoteFileTable() throws SftpException, JSchException { + private void initializeRemoteFileTable() throws SftpException, JSchException, IOException { String remoteHome = "/"; this.currentRemotePath = Paths.get(remoteHome); @@ -203,8 +206,9 @@ public void updateItem(String item, boolean empty){ populateRemoteFileTable(); } - private void populateRemoteFileTable() throws JSchException, SftpException { + private void populateRemoteFileTable() throws JSchException, SftpException, IOException { currentRemoteFileList.clear(); + int count = 0; fbRemotePath.setText(SEAGridContext.getInstance().getUserName() + currentRemotePath.toString()); FileListModel fileListModel; if(currentRemotePath.getParent() != null){ @@ -212,14 +216,18 @@ private void populateRemoteFileTable() throws JSchException, SftpException { currentRemotePath.getParent().toString()); currentRemoteFileList.add(fileListModel); } - Vector children = StorageManager.getInstance().getDirectoryListing(currentRemotePath.toString()); - for(ChannelSftp.LsEntry lsEntry : children){ - if(lsEntry.getFilename().equals(".") || lsEntry.getFilename().equals("..")) continue; - fileListModel = new FileListModel(lsEntry.getFilename(), lsEntry.getAttrs().isDir() == false - ? FileListModel.FileListModelType.FILE : FileListModel.FileListModelType.DIR, lsEntry.getAttrs().getSize(), - lsEntry.getAttrs().getATime() * 1000L, FileListModel.FileLocation.REMOTE, currentRemotePath.toString() - + "/" + lsEntry.getFilename()); - currentRemoteFileList.add(fileListModel); + List resources = NextcloudStorageManager.getInstance().listDirectories(currentRemotePath.toString()); + + for (DavResource res : resources) { + if (count != 0) { + if (res.getName().equals(".") || res.getName().equals("..")) continue; + fileListModel = new FileListModel(res.getName(), res.isDirectory() == false + ? FileListModel.FileListModelType.FILE : FileListModel.FileListModelType.DIR, res.getContentLength().intValue(), + res.getModified().getTime(), FileListModel.FileLocation.REMOTE, currentRemotePath.toString() + + "/" + res.getName()); + currentRemoteFileList.add(fileListModel); + } + count++; } } diff --git a/src/main/java/org/seagrid/desktop/ui/experiment/summary/controller/ExperimentSummaryController.java b/src/main/java/org/seagrid/desktop/ui/experiment/summary/controller/ExperimentSummaryController.java index 024b428..aa991b7 100644 --- a/src/main/java/org/seagrid/desktop/ui/experiment/summary/controller/ExperimentSummaryController.java +++ b/src/main/java/org/seagrid/desktop/ui/experiment/summary/controller/ExperimentSummaryController.java @@ -53,9 +53,9 @@ import org.apache.airavata.model.status.JobStatus; import org.apache.airavata.model.workspace.Project; import org.apache.thrift.TException; +import org.seagrid.desktop.connectors.NextcloudStorage.NextcloudFileDownloadTask; +import org.seagrid.desktop.connectors.NextcloudStorage.NextcloudStorageManager; import org.seagrid.desktop.connectors.airavata.AiravataManager; -import org.seagrid.desktop.connectors.storage.GuiFileDownloadTask; -import org.seagrid.desktop.connectors.storage.StorageManager; import org.seagrid.desktop.ui.commons.SEAGridDialogHelper; import org.seagrid.desktop.ui.home.model.ExperimentListModel; import org.seagrid.desktop.ui.storage.MassStorageBrowserWindow; @@ -65,7 +65,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; @@ -190,7 +193,7 @@ public void initialize(){ List inputDataObjectTypes = experimentModel.getExperimentInputs(); String experimentDataDir = "/" + experimentModel.getProjectId().substring(0, experimentModel.getProjectId().length()-37).replaceAll("[^A-Za-z0-9 ]", "_") + "/" + experimentNameLabel.getText().replaceAll("[^A-Za-z0-9]","_")+"."+System.currentTimeMillis(); - StorageManager.getInstance().createDirIfNotExists(experimentDataDir); + NextcloudStorageManager.getInstance().createFolderifNotExist(experimentDataDir); String expId = AiravataManager.getInstance().cloneExperiment(experimentModel.getExperimentId(), "Clone of " + experimentModel.getExperimentName(), experimentModel.getProjectId()); ExperimentModel clonedExperimentModel = AiravataManager.getInstance().getExperiment(expId); @@ -217,7 +220,7 @@ public void initialize(){ SEAGridEventBus.getInstance().register(this); } - public void initExperimentInfo(ExperimentModel experimentModel) throws TException, URISyntaxException { + public void initExperimentInfo(ExperimentModel experimentModel) throws TException, URISyntaxException, IOException { if(experimentModel != null){ experimentIdLabel.setText(experimentModel.getExperimentId()); experimentNameLabel.setText(experimentModel.getExperimentName()); @@ -292,7 +295,7 @@ public void initExperimentInfo(ExperimentModel experimentModel) throws TExceptio } } - public void initExperimentInfo(String experimentId) throws TException, URISyntaxException { + public void initExperimentInfo(String experimentId) throws TException, URISyntaxException, IOException { experimentModel = AiravataManager.getInstance().getExperiment(experimentId); initExperimentInfo(experimentModel); } @@ -365,6 +368,7 @@ private void showStatus(ExperimentModel experimentModel) throws TException { } private void showExperimentInputs(ExperimentModel experimentModel) throws TException, URISyntaxException { + List inputDataObjectTypes = experimentModel.getExperimentInputs(); int rowIndex = EXPERIMENT_INPUT_START_ROW; experimentInfoGridPane.add(new Label("Inputs"), 0, rowIndex); @@ -400,6 +404,7 @@ else if(o2.getName().startsWith("Optional-File-Inputs")) String filePath1 = (new URI(fileUri)).getPath(); hyperlink = new Hyperlink(Paths.get(filePath1).getFileName().toString()); uriOutputLabel = new TextFlow(new Text(input.getName() + " : "), hyperlink); + hyperlink.setOnAction(event -> { downloadFile(Paths.get(filePath1.toString().replaceAll(dataRoot, "")), experimentModel); }); @@ -457,7 +462,7 @@ private void showExperimentErrors(ExperimentModel experimentModel) { } } - private void showExperimentOutputs(ExperimentModel experimentModel) throws TException, URISyntaxException { + private void showExperimentOutputs(ExperimentModel experimentModel) throws TException, URISyntaxException, IOException { int rowIndex = experimentInfoGridPane.getRowConstraints().size(); experimentInfoGridPane.add(new Label("Outputs"), 0, rowIndex); List outputDataObjectTypes = experimentModel.getExperimentOutputs(); @@ -469,6 +474,7 @@ private void showExperimentOutputs(ExperimentModel experimentModel) throws TExce case STDERR: case STDOUT: String dataRoot = remoteDataDirRoot; + try{ List replicas = AiravataManager.getInstance().getDataReplicas(output.getValue()); String fileUri = ""; @@ -510,7 +516,7 @@ private void downloadFile(Path remotePath, ExperimentModel experimentModel){ @Override protected Task createTask() { try { - return new GuiFileDownloadTask(remotePath.toString(), localPath); + return new NextcloudFileDownloadTask(remotePath.toString(), localPath); } catch (Exception e) { e.printStackTrace(); SEAGridDialogHelper.showExceptionDialogAndWait(e, "Exception Dialog", experimentInfoGridPane.getScene().getWindow(), @@ -555,7 +561,7 @@ private void updateButtonOptions(ExperimentModel experimentModel){ @SuppressWarnings("unused") @Subscribe - public void listenSEAGridEvents(SEAGridEvent event) throws TException { + public void listenSEAGridEvents(SEAGridEvent event) throws TException, IOException { if (event.getEventType().equals(SEAGridEvent.SEAGridEventType.EXPERIMENT_UPDATED)) { ExperimentModel updatedExperimentModel = (ExperimentModel) event.getPayload(); if(updatedExperimentModel.getExperimentId().equals(this.experimentModel.getExperimentId())){ diff --git a/src/main/java/org/seagrid/desktop/ui/storage/controller/MassStorageBrowserController.java b/src/main/java/org/seagrid/desktop/ui/storage/controller/MassStorageBrowserController.java index 242927a..62e53cf 100644 --- a/src/main/java/org/seagrid/desktop/ui/storage/controller/MassStorageBrowserController.java +++ b/src/main/java/org/seagrid/desktop/ui/storage/controller/MassStorageBrowserController.java @@ -20,6 +20,7 @@ */ package org.seagrid.desktop.ui.storage.controller; +import com.github.sardine.DavResource; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.SftpException; @@ -42,7 +43,7 @@ import javafx.scene.input.TransferMode; import javafx.scene.layout.HBox; import javafx.stage.Stage; -import org.seagrid.desktop.connectors.storage.*; +import org.seagrid.desktop.connectors.NextcloudStorage.*; import org.seagrid.desktop.ui.commons.SEAGridDialogHelper; import org.seagrid.desktop.ui.storage.model.FileListModel; import org.seagrid.desktop.util.SEAGridContext; @@ -51,10 +52,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; +import java.util.List; import java.util.Vector; public class MassStorageBrowserController { @@ -257,7 +262,7 @@ private void populateLocalFileList(){ } } - private void initializeRemoteFileTable() throws SftpException, JSchException { + private void initializeRemoteFileTable() throws SftpException, JSchException, IOException { this.currentRemotePath = "/"; fbRemoteFileTblFileName.setCellValueFactory(cellData-> new SimpleObjectProperty(cellData.getValue())); @@ -379,7 +384,7 @@ public void updateItem(String item, boolean empty){ populateRemoteFileTable(); } - private void populateRemoteFileTable() throws JSchException, SftpException { + private void populateRemoteFileTable() throws JSchException, SftpException, IOException { currentRemoteFileList.clear(); fbRemotePath.setText(SEAGridContext.getInstance().getUserName() + currentRemotePath.toString()); FileListModel fileListModel; @@ -388,14 +393,21 @@ private void populateRemoteFileTable() throws JSchException, SftpException { (new File(currentRemotePath).getParent())); currentRemoteFileList.add(fileListModel); } - Vector children = StorageManager.getInstance().getDirectoryListing(currentRemotePath.toString()); - for(ChannelSftp.LsEntry lsEntry : children){ - if(lsEntry.getFilename().equals(".") || lsEntry.getFilename().equals("..")) continue; - fileListModel = new FileListModel(lsEntry.getFilename(), lsEntry.getAttrs().isDir() == false - ? FileListModel.FileListModelType.FILE : FileListModel.FileListModelType.DIR, lsEntry.getAttrs().getSize(), - lsEntry.getAttrs().getATime() * 1000L, FileListModel.FileLocation.REMOTE, currentRemotePath.toString() - + "/" + lsEntry.getFilename()); - currentRemoteFileList.add(fileListModel); + + List resources = NextcloudStorageManager.getInstance().listDirectories(currentRemotePath.toString()); + int count = 0; + if(resources!= null) { + for (DavResource res : resources) { + if(count != 0) { + if (res.getName().equals(".") || res.getName().equals("..")) continue; + fileListModel = new FileListModel(res.getName(), res.isDirectory() == false + ? FileListModel.FileListModelType.FILE : FileListModel.FileListModelType.DIR, res.getContentLength().intValue(), + res.getModified().getTime(), FileListModel.FileLocation.REMOTE, currentRemotePath.toString() + + "/" + res.getName()); + currentRemoteFileList.add(fileListModel); + } + count++; + } } } @@ -404,7 +416,7 @@ private void uploadFile(String localFile, String remotePath, FileListModel upldF @Override protected Task createTask() { try { - return new GuiFileUploadTask(remotePath, localFile); + return new NextcloudSingleFileUploadTask(remotePath, localFile); } catch (Exception e) { e.printStackTrace(); SEAGridDialogHelper.showExceptionDialogAndWait(e, "Exception Dialog", fbRemoteFileTable.getScene().getWindow(), @@ -441,7 +453,7 @@ private void uploadDir(String localDir, String remoteDir, FileListModel upldFile @Override protected Task createTask() { try { - return new GuiDirUploadTask(remoteDir, localDir); + return new NextcloudFolderUploadTask(remoteDir, localDir); } catch (Exception e) { e.printStackTrace(); SEAGridDialogHelper.showExceptionDialogAndWait(e, "Exception Dialog", fbRemoteFileTable.getScene().getWindow(), @@ -478,7 +490,7 @@ private void downloadFile(String remoteFile, String localFile, FileListModel dow @Override protected Task createTask() { try { - return new GuiFileDownloadTask(remoteFile, localFile); + return new NextcloudFileDownloadTask(remoteFile, localFile); } catch (Exception e) { e.printStackTrace(); SEAGridDialogHelper.showExceptionDialogAndWait(e, "Exception Dialog", fbLocalFileTable.getScene().getWindow(), @@ -515,7 +527,7 @@ private void downloadDir(String remoteDir, String localDir, FileListModel downFi @Override protected Task createTask() { try { - return new GuiDirDownloadTask(remoteDir, localDir); + return new NextCloudFolderdownloadtask(remoteDir, localDir); } catch (Exception e) { e.printStackTrace(); SEAGridDialogHelper.showExceptionDialogAndWait(e, "Exception Dialog", fbLocalFileTable.getScene().getWindow(), @@ -547,9 +559,10 @@ protected Task createTask() { service.start(); } - public void gotoRemoteDir(String path) throws JSchException, SftpException { + public void gotoRemoteDir(String path) throws JSchException, SftpException, IOException { currentRemoteFileList.clear(); currentRemotePath = path; + int count = 0; fbRemotePath.setText(SEAGridContext.getInstance().getUserName() + currentRemotePath.toString()); FileListModel fileListModel; if((new File(currentRemotePath)).getParent() != null && !(new File(currentRemotePath)).getParent().isEmpty()){ @@ -557,14 +570,21 @@ public void gotoRemoteDir(String path) throws JSchException, SftpException { (new File(currentRemotePath).getParent())); currentRemoteFileList.add(fileListModel); } - Vector children = StorageManager.getInstance().getDirectoryListing(currentRemotePath.toString()); - for(ChannelSftp.LsEntry lsEntry : children){ - if(lsEntry.getFilename().equals(".") || lsEntry.getFilename().equals("..")) continue; - fileListModel = new FileListModel(lsEntry.getFilename(), lsEntry.getAttrs().isDir() == false - ? FileListModel.FileListModelType.FILE : FileListModel.FileListModelType.DIR, lsEntry.getAttrs().getSize(), - lsEntry.getAttrs().getATime() * 1000L, FileListModel.FileLocation.REMOTE, currentRemotePath.toString() - + "/" + lsEntry.getFilename()); - currentRemoteFileList.add(fileListModel); + + List resources = NextcloudStorageManager.getInstance().listDirectories(currentRemotePath.toString()); + if(resources!=null) { + for (DavResource res : resources) { + if (count != 0) { + ; + if (res.getName().equals(".") || res.getName().equals("..")) continue; + fileListModel = new FileListModel(res.getName(), res.isDirectory() == false + ? FileListModel.FileListModelType.FILE : FileListModel.FileListModelType.DIR, res.getContentLength().intValue(), + res.getModified().getTime(), FileListModel.FileLocation.REMOTE, currentRemotePath.toString() + + "/" + res.getName()); + currentRemoteFileList.add(fileListModel); + } + count++; + } } } } \ No newline at end of file diff --git a/src/main/java/org/seagrid/desktop/util/SEAGridConfig.java b/src/main/java/org/seagrid/desktop/util/SEAGridConfig.java index 68cb912..5417bcf 100644 --- a/src/main/java/org/seagrid/desktop/util/SEAGridConfig.java +++ b/src/main/java/org/seagrid/desktop/util/SEAGridConfig.java @@ -27,6 +27,7 @@ public class SEAGridConfig { private final static Logger logger = LoggerFactory.getLogger(SEAGridConfig.class); public static final boolean DEV = false; //true + public static final boolean IS_USE_HTTPS = false; public static final String USER_NAME = "user.name"; public static final String AUTHENTICATED = "authenticated"; @@ -51,4 +52,12 @@ public class SEAGridConfig { public static final java.lang.String DEV_REMOTE_DATA_DIR_ROOT = "dev.remote.data.dir.root"; public static final java.lang.String REMOTE_DATA_DIR_PREFIX = "remote.data.dir.prefix"; public static final java.lang.String DEV_REMOTE_DATA_DIR_PREFIX = "dev.remote.data.dir.prefix"; + + public static final java.lang.String NEXTCLOUD_SERVERNAME = "nextcloud.server"; + public static final java.lang.String NEXTCLOUD_WEBDAV_BASEPATH = "nextcloudwebdav.base.path"; + public static final java.lang.String CLIENT_ID = "nextcloud.client.id"; + public static final java.lang.String TOKEN_ENDPOINT = "nextcloud.token.endpoint"; + public static final java.lang.String TOKEN_GRANTYPE = "nextcloud.grantype"; + public static final java.lang.String TOKEN_VALIDATION_URL = "nextcloud.tokenvalidation.api"; + public static final java.lang.String BASE_DOWNLOAD_PATH = "nextcloud.file.basedownloadpath"; } \ No newline at end of file diff --git a/src/main/java/org/seagrid/desktop/util/SEAGridContext.java b/src/main/java/org/seagrid/desktop/util/SEAGridContext.java index c82597e..c8eb5ef 100644 --- a/src/main/java/org/seagrid/desktop/util/SEAGridContext.java +++ b/src/main/java/org/seagrid/desktop/util/SEAGridContext.java @@ -220,6 +220,38 @@ public void saveUserPrefs() { } + public String getNextcloudServername() { + return SEAGridConfig.NEXTCLOUD_SERVERNAME; + } + + public String getDavBasepath() { + return SEAGridConfig.NEXTCLOUD_WEBDAV_BASEPATH; + } + + public boolean isUseHttps() { + return SEAGridConfig.IS_USE_HTTPS; + } + + public String getClientID() { + return SEAGridConfig.CLIENT_ID; + } + + public String getTokenGenerateEndpoint() { + return SEAGridConfig.TOKEN_ENDPOINT; + } + + public String getGrantType() { + return SEAGridConfig.TOKEN_GRANTYPE; + } + + public String getValidationURL() { + return SEAGridConfig.TOKEN_VALIDATION_URL; + } + + public String getBaseDownloadPath() { + return SEAGridConfig.BASE_DOWNLOAD_PATH; + } + //FIXME - There is an issue in loading file from resources. This is a temporary hack public static final String logoBase64 = "iVBORw0KGgoAAAANSUhEUgAAAdkAAACTCAMAAAD1LZOOAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5" + "ccllPAAAAGxQTFRF1PPxWMW/j9jUx+zqLbqx8fr6LrevSsC6q+LfZc3H4/X0dM/KPLy11fHvgtTPuefkZsrFnd3as+vmnODc9Pz7YM7HNsC36f" + diff --git a/src/main/resources/seagrid.properties b/src/main/resources/seagrid.properties index c1e8ead..8096d72 100644 --- a/src/main/resources/seagrid.properties +++ b/src/main/resources/seagrid.properties @@ -47,4 +47,14 @@ remote.data.dir.prefix=file://pga@seagrid.org: dev.remote.data.dir.prefix=file://pga@dev.seagrid.org: #Default file download location. If not set will be downloaded to use.dir/SEAGrid/ExperimentData -default.file.download.path= \ No newline at end of file +default.file.download.path= + +#Nextcloud configurations, Jetstream instance is specified at present +nextcloud.server=149.165.157.194 +nextcloudwebdav.base.path = remote.php/dav/files +nextcloud.client.id = http://149.165.157.194/index.php/apps/user_saml/saml/metadata +nextcloud.client.secret= +nextcloud.token.endpoint=http://149.165.157.194:8180/auth/realms/realm-nextcloud/protocol/openid-connect/token +nextcloud.grantype=password +nextcloud.tokenvalidation.api=http://149.165.157.194/checkuser.php +nextcloud.file.basedownloadpath= \ No newline at end of file