diff --git a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java index 28672e99f..52f4b079d 100644 --- a/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java +++ b/server/src/main/java/org/eclipse/openvsx/ExtensionProcessor.java @@ -28,6 +28,18 @@ import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.ZipException; +import java.util.zip.ZipFile; /** * Processes uploaded extension files and extracts their metadata. diff --git a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java index 2d7934102..3f76ebd75 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/ExtractResourcesJobRequestHandler.java @@ -52,6 +52,5 @@ public void run(MigrationJobRequest jobRequest) throws Exception { } service.deleteWebResources(extVersion); - migrations.deleteFile(extensionFile); } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java index ccb798cbc..58af0e13d 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationRunner.java @@ -70,7 +70,7 @@ private void renameDownloadsMigration() { private void setContentTypeMigration() { var jobName = "SetContentTypeMigration"; var handler = SetContentTypeJobRequestHandler.class; - repositories.findNotMigratedContentTypes().forEach(item -> enqueueJob(jobName, handler, item)); + repositories.findNotMigratedContentTypes().forEach(item -> migrations.enqueueMigration(jobName, handler, item)); } private void extractVsixManifestMigration() { diff --git a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java index 2a43f0d50..9c6c8c109 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/MigrationService.java @@ -79,8 +79,8 @@ public TempFile getExtensionFile(Map.Entry entry) throws I var content = entry.getValue(); if(content == null) { - var storage = getStorage(resource); - var uri = storage.getLocation(resource); + var storage = getStorage(entry.getKey()); + var uri = storage.getLocation(entry.getKey()); backgroundRestTemplate.execute("{extensionLocation}", HttpMethod.GET, null, response -> { try(var out = Files.newOutputStream(extensionFile.getPath())) { response.getBody().transferTo(out); @@ -95,14 +95,6 @@ public TempFile getExtensionFile(Map.Entry entry) throws I return extensionFile; } - public void deleteFile(Path filePath) { - try { - Files.delete(filePath); - } catch (IOException e) { - throw new RuntimeException("Failed to delete file"); - } - } - @Retryable public void uploadFileResource(FileResource resource) { if(resource.getStorageType().equals(FileResource.STORAGE_DB)) { diff --git a/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobRequestHandler.java index abfab7a65..ee6f26427 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobRequestHandler.java @@ -10,7 +10,6 @@ package org.eclipse.openvsx.migration; import org.eclipse.openvsx.ExtensionProcessor; -import org.eclipse.openvsx.entities.ExtensionVersion; import org.eclipse.openvsx.entities.FileResource; import org.jobrunr.jobs.annotations.Job; import org.jobrunr.jobs.lambdas.JobRequestHandler; @@ -19,6 +18,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.io.IOException; import java.util.AbstractMap; import java.util.function.Consumer; @@ -36,28 +36,29 @@ public class SetContentTypeJobRequestHandler implements JobRequestHandler consumer = resource -> { - var existingResource = service.getExistingResource(extVersion, resource); - if(existingResource == null) { - return; - } - - var resourceFile = migrations.getFile(new AbstractMap.SimpleEntry<>(existingResource, resource.getContent())); - migrations.deleteResource(existingResource); - migrations.uploadResource(existingResource, resourceFile); - migrations.deleteFile(resourceFile); - }; - - processor.getFileResources(extVersion).forEach(consumer); - processor.processEachResource(extVersion, consumer); - } + try(var extensionFile = migrations.getExtensionFile(entry)) { + try (var processor = new ExtensionProcessor(extensionFile)) { + Consumer consumer = resource -> { + var existingResource = service.getExistingResource(extVersion, resource); + if (existingResource == null) { + return; + } + + try (var resourceFile = migrations.getExtensionFile(new AbstractMap.SimpleEntry<>(existingResource, resource.getContent()))) { + migrations.deleteFileResource(existingResource); + migrations.uploadFileResource(existingResource, resourceFile); + } catch (IOException e) { + throw new RuntimeException(e); + } + }; - migrations.deleteFile(extensionFile); + processor.getFileResources(extVersion).forEach(consumer); + processor.processEachResource(extVersion, consumer); + } + } } } diff --git a/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobService.java b/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobService.java index f063d2a4b..47cbed8a0 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobService.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/SetContentTypeJobService.java @@ -15,7 +15,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import javax.transaction.Transactional; +import jakarta.transaction.Transactional; @Component public class SetContentTypeJobService { diff --git a/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java b/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java index ee0169e77..30674559c 100644 --- a/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java +++ b/server/src/main/java/org/eclipse/openvsx/migration/SetPreReleaseJobRequestHandler.java @@ -38,7 +38,6 @@ public void run(MigrationJobRequest jobRequest) throws Exception { try (var extensionFile = migrations.getExtensionFile(entry)) { service.updatePreviewAndPreRelease(extVersion, extensionFile); } - migrations.deleteFile(extensionFile); } } } diff --git a/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java b/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java index 641b28ba4..c32613b4b 100644 --- a/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java +++ b/server/src/main/java/org/eclipse/openvsx/security/SecurityConfig.java @@ -18,10 +18,6 @@ import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.Http403ForbiddenEntryPoint; -import org.springframework.security.web.authentication.session.SessionAuthenticationStrategy; -import org.springframework.security.web.csrf.CookieCsrfTokenRepository; -import org.springframework.security.web.csrf.CsrfAuthenticationStrategy; -import org.springframework.security.web.csrf.HttpSessionCsrfTokenRepository; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.springframework.security.web.util.matcher.RequestMatcher; diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java index a0ad20451..f323eb64f 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java @@ -68,16 +68,16 @@ protected BlobContainerClient getContainerClient() { @Override public void uploadFile(FileResource resource) { var blobName = getBlobName(resource); - uploadFile(resource.getContent(), resource.getName(), blobName); + uploadFile(resource.getContent(), resource.getName(), blobName, ""); // TODO add contentType } @Override public void uploadNamespaceLogo(Namespace namespace) { var blobName = getBlobName(namespace); - uploadFile(namespace.getLogoBytes(), namespace.getLogoName(), blobName); + uploadFile(namespace.getLogoBytes(), namespace.getLogoName(), blobName, ""); // TODO add contentType } - - protected void uploadFile(byte[] content, String fileName, String blobName) { + + protected void uploadFile(byte[] content, String fileName, String blobName, String contentType) { if (StringUtils.isEmpty(serviceEndpoint)) { throw new IllegalStateException("Cannot upload file " + blobName + ": missing Azure blob service endpoint"); @@ -85,15 +85,14 @@ protected void uploadFile(byte[] content, String fileName, String blobName) { var blobClient = getContainerClient().getBlobClient(blobName); var headers = new BlobHttpHeaders(); - headers.setContentType(resource.getContentType()); - if (resource.getName().endsWith(".vsix")) { - headers.setContentDisposition("attachment; filename=\"" + resource.getName() + "\""); + headers.setContentType(contentType); + if (fileName.endsWith(".vsix")) { + headers.setContentDisposition("attachment; filename=\"" + fileName + "\""); } else { - var cacheControl = StorageUtil.getCacheControl(resource.getName()); + var cacheControl = StorageUtil.getCacheControl(fileName); headers.setCacheControl(cacheControl.getHeaderValue()); } - var content = resource.getContent(); try (var dataStream = new ByteArrayInputStream(content)) { blobClient.upload(dataStream, content.length, true); blobClient.setHttpHeaders(headers); @@ -103,12 +102,12 @@ protected void uploadFile(byte[] content, String fileName, String blobName) { } @Override - public void uploadFile(FileResource resource, TempFile file) { + public void uploadFile(FileResource resource, TempFile file, String contentType) { var blobName = getBlobName(resource); - uploadFile(file, resource.getName(), blobName); + uploadFile(file, resource.getName(), blobName, contentType); } - protected void uploadFile(TempFile file, String fileName, String blobName) { + protected void uploadFile(TempFile file, String fileName, String blobName, String contentType) { if (StringUtils.isEmpty(serviceEndpoint)) { throw new IllegalStateException("Cannot upload file " + blobName + ": missing Azure blob service endpoint"); diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java index feaad3a6d..cc0be954c 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java @@ -83,19 +83,19 @@ public void uploadNamespaceLogo(Namespace namespace) { + objectId + ": missing Google bucket id"); } - uploadFile(namespace.getLogoBytes(), namespace.getLogoName(), objectId); + uploadFile(namespace.getLogoBytes(), namespace.getLogoName(), objectId, ""); // TODO add contentType } - protected void uploadFile(byte[] content, String fileName, String objectId) { + protected void uploadFile(byte[] content, String fileName, String objectId, String contentType) { var blobInfoBuilder = BlobInfo.newBuilder(BlobId.of(bucketId, objectId)) - .setContentType(resource.getContentType()); - if (resource.getName().endsWith(".vsix")) { - blobInfoBuilder.setContentDisposition("attachment; filename=\"" + resource.getName() + "\""); + .setContentType(contentType); + if (fileName.endsWith(".vsix")) { + blobInfoBuilder.setContentDisposition("attachment; filename=\"" + fileName + "\""); } else { - var cacheControl = StorageUtil.getCacheControl(resource.getName()); + var cacheControl = StorageUtil.getCacheControl(fileName); blobInfoBuilder.setCacheControl(cacheControl.getHeaderValue()); } - getStorage().create(blobInfoBuilder.build(), resource.getContent()); + getStorage().create(blobInfoBuilder.build(), content); } @Override @@ -106,10 +106,10 @@ public void uploadFile(FileResource resource, TempFile file) { + objectId + ": missing Google bucket id"); } - uploadFile(file, resource.getName(), objectId); + uploadFile(file, resource.getName(), objectId, ""); // TODO add contentType } - protected void uploadFile(TempFile file, String fileName, String objectId) { + protected void uploadFile(TempFile file, String fileName, String objectId, String contentType) { var blobInfoBuilder = BlobInfo.newBuilder(BlobId.of(bucketId, objectId)) .setContentType(contentType); if (fileName.endsWith(".vsix")) {