From 1bd3975ff1acfa2338f7e31963320960684ca6e6 Mon Sep 17 00:00:00 2001 From: vagisha Date: Thu, 13 Jul 2023 07:18:41 -0700 Subject: [PATCH] Add filesystem audit events for Panorama Public symlinks (#361) - Added filesystem audit events when symlinks are created / updated, or replaced with the target file. - Added additional logging, and removed unused boolean parameter from moveAndSymLinkDirectory. --- .../PanoramaPublicFileImporter.java | 11 +- .../PanoramaPublicFileListener.java | 2 +- .../PanoramaPublicListener.java | 6 +- .../PanoramaPublicSymlinkHandler.java | 5 +- .../PanoramaPublicSymlinkManager.java | 117 +++++++++++++----- .../pipeline/CopyExperimentFinalTask.java | 12 +- .../view/publish/copyExperimentForm.jsp | 4 +- 7 files changed, 112 insertions(+), 45 deletions(-) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java index 15d37187..9bf70632 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java @@ -85,8 +85,15 @@ public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualF Files.createDirectories(targetFiles.toPath()); } - log.info("Moving files and creating sym links in folder " + ctx.getContainer().getPath()); - PanoramaPublicSymlinkManager.get().moveAndSymLinkDirectory(expJob, sourceFiles, targetFiles, false, log); + if (expJob.isMoveAndSymlink()) + { + log.info("Moving files to folder " + expJob.getContainer().getPath() + " and creating symlinks"); + } + else + { + log.info("Copying files to folder " + expJob.getContainer().getPath()); + } + PanoramaPublicSymlinkManager.get().moveAndSymLinkDirectory(expJob, sourceFiles, targetFiles, log); alignDataFileUrls(expJob.getUser(), ctx.getContainer(), log); } diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java index c334b425..4a63b650 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java @@ -33,7 +33,7 @@ public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Co public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) { // Update any symlinks targeting the file - PanoramaPublicSymlinkManager.get().fireSymlinkUpdate(src.toPath(), dest.toPath(), container); + PanoramaPublicSymlinkManager.get().fireSymlinkUpdate(src.toPath(), dest.toPath(), container, user); ExpData data = ExperimentService.get().getExpDataByURL(src, null); if (null != data) diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java index 4bcb576e..8dfca87c 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java @@ -75,13 +75,13 @@ public void containerDeleted(Container c, User user) { JournalManager.deleteProjectJournal(c, user); - PanoramaPublicSymlinkManager.get().beforeContainerDeleted(c); + PanoramaPublicSymlinkManager.get().beforeContainerDeleted(c, user); } @Override public void containerMoved(Container c, Container oldParent, User user) { - PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldParent, c); + PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldParent, c, user); } @Override @@ -109,7 +109,7 @@ public void propertyChange(PropertyChangeEvent evt) // ce.getOldValue() and ce.getNewValue() are just the names of the old and new containers. We need the full path. Path oldPath = parentPath.resolve((String) ce.getOldValue()); Path newPath = parentPath.resolve((String) ce.getNewValue()); - PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldPath.toString(), newPath.toString(), c); + PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldPath.toString(), newPath.toString(), c, ce.user); } } } diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java index 5607bf1e..8244cc31 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java @@ -1,9 +1,12 @@ package org.labkey.panoramapublic; +import org.labkey.api.data.Container; +import org.labkey.api.security.User; + import java.io.IOException; import java.nio.file.Path; public interface PanoramaPublicSymlinkHandler { - void handleSymlink(Path link, Path target) throws IOException; + void handleSymlink(Path link, Path target, Container container, User user) throws IOException; } diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java index 9e753672..004de3a6 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java @@ -4,11 +4,14 @@ import org.apache.commons.lang3.SystemUtils; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.provider.FileSystemAuditProvider; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ContainerService; import org.labkey.api.files.FileContentService; import org.labkey.api.module.ModuleLoader; +import org.labkey.api.security.User; import org.labkey.api.util.FileUtil; import org.labkey.api.util.logging.LogHelper; import org.labkey.panoramapublic.model.ExperimentAnnotations; @@ -54,13 +57,13 @@ public static PanoramaPublicSymlinkManager get() } - private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler handler) + private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler handler, Container container, User user) { for (File file : Objects.requireNonNull(source.listFiles())) { if (file.isDirectory()) { - handleContainerSymlinks(file, handler); + handleContainerSymlinks(file, handler, container, user); } else { @@ -69,7 +72,7 @@ private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler h { try { Path target = Files.readSymbolicLink(filePath); - handler.handleSymlink(filePath, target); + handler.handleSymlink(filePath, target, container, user); } catch (IOException x) { _log.error("Unable to resolve symlink target for symlink at " + filePath); } @@ -78,7 +81,7 @@ private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler h } } - public void handleContainerSymlinks(Container container, PanoramaPublicSymlinkHandler handler) + public void handleContainerSymlinks(Container container, User user, PanoramaPublicSymlinkHandler handler) { FileContentService fcs = FileContentService.get(); if (null != fcs) @@ -86,19 +89,19 @@ public void handleContainerSymlinks(Container container, PanoramaPublicSymlinkHa File root = fcs.getFileRoot(container); if (null != root) { - handleContainerSymlinks(root, handler); + handleContainerSymlinks(root, handler, container, user); } } } - private void handleAllSymlinks(Set containers, PanoramaPublicSymlinkHandler handler) + private void handleAllSymlinks(Set containers, User user, PanoramaPublicSymlinkHandler handler) { for (Container container : containers) { Set tree = ContainerManager.getAllChildren(container); for (Container node : tree) { - handleContainerSymlinks(node, handler); + handleContainerSymlinks(node, user, handler); } } } @@ -112,7 +115,7 @@ private String normalizeContainerPath(String path) return File.separator + path + File.separator; } - public void beforeContainerDeleted(Container container) + public void beforeContainerDeleted(Container container, User user) { if (PanoramaPublicManager.canBeSymlinkTarget(container)) // Fire the event only if the container being deleted is in the Panorama Public project. { @@ -121,7 +124,7 @@ public void beforeContainerDeleted(Container container) ExperimentAnnotations expAnnot = ExperimentAnnotationsManager.getExperimentIncludesContainer(container); if (null != expAnnot) { - fireSymlinkCopiedExperimentDelete(expAnnot, container); + fireSymlinkCopiedExperimentDelete(expAnnot, container, user); } } } @@ -132,7 +135,7 @@ public void beforeContainerDeleted(Container container) * @param container container being deleted. This could be a subfolder of the experiment container if the experiment * is configured to include subfolders. */ - private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, Container container) + private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, Container container, User user) { if (expAnnot.getDataVersion() != null && !ExperimentAnnotationsManager.isCurrentVersion(expAnnot)) { @@ -153,17 +156,19 @@ private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, C if (nextHighestVersion != null) { Container versionContainer = nextHighestVersion.getContainer(); - handleContainerSymlinks(versionContainer, (link, target) -> { + handleContainerSymlinks(versionContainer, user, (link, target, c, u) -> { if (!target.startsWith(deletedContainerPath)) { return; } Files.move(target, link, REPLACE_EXISTING); // Move the files back to the next highest version of the experiment + addReplaceSymlinkWithTargetAuditEvent(link, target, c, u); + _log.info("File moved from " + target + " to " + link); // This should update the symlinks in the submitted folder as well as // symlinks in versions older than this one to point to the files in the next highest version. - fireSymlinkUpdate(target, link, container); + fireSymlinkUpdate(target, link, container, user); }); } } @@ -180,12 +185,14 @@ private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, C } if (null != sourceContainer) { - handleContainerSymlinks(sourceContainer, (link, target) -> { + handleContainerSymlinks(sourceContainer, user, (link, target, c, u) -> { if (!target.startsWith(deletedContainerPath)) { return; } Files.move(target, link, REPLACE_EXISTING); + addReplaceSymlinkWithTargetAuditEvent(link, target, c, u); + _log.info("File moved from " + target + " to " + link); // Symlinks in the source container point to -> current version container on Panorama Public @@ -197,7 +204,7 @@ private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, C } } - public void fireSymlinkUpdateContainer(Container oldContainer, Container newContainer) + public void fireSymlinkUpdateContainer(Container oldContainer, Container newContainer, User user) { // Update symlinks to new target FileContentService fcs = FileContentService.get(); @@ -205,12 +212,12 @@ public void fireSymlinkUpdateContainer(Container oldContainer, Container newCont { if (fcs.getFileRoot(oldContainer) != null && fcs.getFileRoot(newContainer) != null) { - fireSymlinkUpdateContainer(fcs.getFileRoot(oldContainer).getPath(), fcs.getFileRoot(newContainer).getPath(), oldContainer); + fireSymlinkUpdateContainer(fcs.getFileRoot(oldContainer).getPath(), fcs.getFileRoot(newContainer).getPath(), oldContainer, user); } } } - public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, Container container) + public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, Container container, User user) { if (PanoramaPublicManager.canBeSymlinkTarget(container)) { @@ -218,7 +225,7 @@ public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, String newContainerPath = normalizeContainerPath(newContainer); Set containers = getSymlinkContainers(container); - handleAllSymlinks(containers, (link, target) -> { + handleAllSymlinks(containers, user, (link, target, c, u) -> { if (String.valueOf(target).contains(oldContainerPath)) { Path newTarget = Path.of(target.toString().replace(oldContainerPath, newContainerPath)); @@ -226,6 +233,7 @@ public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, { Files.delete(link); Files.createSymbolicLink(link, newTarget); + addLinkUpdatedAuditEvent(link, newTarget, c, u); } catch (IOException e) { @@ -236,13 +244,41 @@ public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, } } - public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container container) + private void addLinkUpdatedAuditEvent(Path link, Path newTarget, Container linkContainer, User user) + { + addFileAuditEvent(link, linkContainer, user, "Updated symlink target to " + newTarget); + } + + private void addReplaceSymlinkWithTargetAuditEvent(Path link, Path target, Container linkContainer, User user) + { + addFileAuditEvent(link, linkContainer, user, "Replaced symlink with target file " + target); + } + + private void addReplaceTargetWithSymlinkAuditEvent(Path link, Path target, Container container, User user) + { + addFileAuditEvent(link, container, user, "Replaced target file with symlink to " + target); + } + + private void addFileAuditEvent(Path link, Container container, User user, String comment) + { + FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent( + container != null ? container.getId() : null, comment); + event.setFile(link.toString()); + AuditLogService.get().addEvent(user, event); + } + + public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container container, User user) + { + fireSymlinkUpdate(oldTarget, newTarget, container, user, null); + } + + public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container container, User user, @Nullable Logger log) { if (PanoramaPublicManager.canBeSymlinkTarget(container)) // Only files in the Panorama Public project can be symlink targets. { Set containers = getSymlinkContainers(container); - handleAllSymlinks(containers, (link, target) -> { + handleAllSymlinks(containers, user, (link, target, c, u) -> { if (!target.equals(oldTarget)) return; @@ -250,6 +286,11 @@ public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container containe { Files.delete(link); Files.createSymbolicLink(link, newTarget); + addLinkUpdatedAuditEvent(link, newTarget, c, u); + if (log != null) + { + log.info("Target for symlink " + link + " updated to " + newTarget); + } } catch (IOException e) { @@ -259,7 +300,7 @@ public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container containe } } - public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, File target, boolean createSourceSymLinks, @Nullable Logger log) throws IOException + public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, File target, @Nullable Logger log) throws IOException { if (null == log) { @@ -283,7 +324,7 @@ public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, log.debug("Directory created: " + targetPath); } - moveAndSymLinkDirectory(job, file, targetPath.toFile(), createSourceSymLinks, log); + moveAndSymLinkDirectory(job, file, targetPath.toFile(), log); } else { @@ -313,6 +354,7 @@ public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, // Copy the file to panorama public Files.copy(filePath, targetPath, REPLACE_EXISTING); + log.debug("Copied file " + filePath + " to " + targetPath); fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer()); continue; @@ -321,35 +363,50 @@ public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, // Symbolic link should move the target file over. This would be for a re-copy to public. if (Files.isSymbolicLink(filePath)) { + log.debug("Source file is a symlink: " + filePath); + Path oldPath = Files.readSymbolicLink(filePath); Files.move(oldPath, targetPath, REPLACE_EXISTING); + log.debug("Moved symlink target " + oldPath + " to " + targetPath); fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer()); - fireSymlinkUpdate(oldPath, targetPath, job.getContainer()); // job container is the target container on Panorama Public - log.debug("File moved from " + oldPath + " to " + targetPath); + fireSymlinkUpdate(oldPath, targetPath, job.getContainer(), // job container is the target container on Panorama Public + job.getUser(), log); Path symlink = Files.createSymbolicLink(oldPath, targetPath); - log.debug("Symlink created: " + symlink); + Container oldTargetContainer = getContainerForFilePath(oldPath); + if (oldTargetContainer != null) + { + // The target of the symlink has been moved from a previous version on Panorama Public to the + // new version on Panorama Public, and a symink has been created in the previous version container. + // Add an audit event in the previous version container. + addReplaceTargetWithSymlinkAuditEvent(symlink, targetPath, oldTargetContainer, job.getUser()); + } + log.debug("Replaced old target with symlink: " + symlink); } else { Files.move(filePath, targetPath, REPLACE_EXISTING); + log.debug("Moved file " + filePath + " to " + targetPath); fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer()); Files.createSymbolicLink(filePath, targetPath); + log.debug("Created symlink " + filePath + " targeting " + targetPath); // We don't need to update any symlinks here since the source container should not have any symlink targets. } - - if (createSourceSymLinks) - { - Path symlink = Files.createSymbolicLink(filePath, targetPath); - log.debug("Symlink created: " + symlink); - } } } } } + // Get the container for the given file path. + // Example: if the file path is C:\Users\vsharma\WORK\LabKey\build\deploy\files\Panorama Public\TestProject V.1\@files\Study9S_Site52_v1.sky.zip + // this will return the container for "/Panorama Public/TestProject V.1" + private Container getContainerForFilePath(Path path) + { + return FileContentService.get().getContainersForFilePath(path).stream().findFirst().orElse(null); + } + private void verifyFileTreeSymlinks(File source, Map linkInvalidTarget, Map linkWithSymlinkTarget) throws IOException { for (File file : Objects.requireNonNull(source.listFiles())) diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java index 9257af61..adff1ca1 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java @@ -206,7 +206,7 @@ private void verifySymlinks(Container source, Container target, boolean matching if (null != targetRoot) { Path targetFileRoot = Path.of(targetRoot.toString(), File.separator); - PanoramaPublicSymlinkManager.get().handleContainerSymlinks(source, (sourceFile, targetFile) -> { + PanoramaPublicSymlinkManager.get().handleContainerSymlinks(source, null, (sourceFile, targetFile, c, u) -> { // valid path if (!FileUtil.isFileAndExists(targetFile)) @@ -241,7 +241,7 @@ private void verifySymlinks(Container source, Container target, boolean matching } } - private void cleanupExportDirectory(User user, File directory) throws IOException + private void cleanupExportDirectory(User user, File directory) { List datas = ExperimentService.get().getExpDatasUnderPath(directory.toPath(), null, true); for (ExpData data : datas) @@ -255,12 +255,12 @@ private void cleanupExportDirectory(User user, File directory) throws IOExceptio private void alignSymlinks(PipelineJob job, CopyExperimentJobSupport jobSupport) { if (jobSupport.getPreviousVersionName() != null) + { + FileContentService fcs = FileContentService.get(); + if (fcs != null) { - FileContentService fcs = FileContentService.get(); - if (fcs != null) - { PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(jobSupport.getPreviousVersionName(), - fcs.getFileRoot(job.getContainer()).getPath(), job.getContainer()); + fcs.getFileRoot(job.getContainer()).getPath(), job.getContainer(), job.getUser()); } } } diff --git a/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp b/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp index c5b3c5e7..7c0866cf 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp +++ b/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp @@ -164,7 +164,7 @@ autoScroll: true, title : '', border: true, - width: 450, + width: 650, height:150, listeners: { select: function(node, record, index, eOpts){ @@ -220,7 +220,7 @@ fieldLabel: "Reviewer Email Prefix", value: <%=q(form.getReviewerEmailPrefix())%>, name: 'reviewerEmailPrefix', - width: 450, + width: 650, afterBodyEl: 'A new LabKey user account email_prefix(unique numeric suffix)@proteinms.net will be created. ', msgTarget : 'under' },