Skip to content

Commit

Permalink
Fix: GCS integration (#25)
Browse files Browse the repository at this point in the history 
* fix: fixed bugs

* fix: fixed bugs

* fix: fixed bugs
  • Loading branch information
@YarikRevich
YarikRevich authored Dec 9, 2024
1 parent ca19f0a commit e6dc833
Show file tree
Hide file tree
Showing 18 changed files with 321 additions and 65 deletions.
6 changes: 5 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,4 +164,8 @@ option is selected as target database.
### Diagnostics dashboard
For **ObjectStorage API Server** configuration the following section should be modified:
For **ObjectStorage API Server** configuration the following section should be modified:
If GCS is selected, please make sure Cloud Resource Manager API is enabled.
Currently max object size is 1GB, will be changed in the future.
6 changes: 2 additions & 4 deletions ...rver/src/main/java/com/objectstorage/repository/common/RepositoryConfigurationHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@
import jakarta.enterprise.context.ApplicationScoped;
import org.apache.commons.lang3.RandomStringUtils;

import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.StringJoiner;

/**
Expand All @@ -24,7 +22,7 @@ public class RepositoryConfigurationHelper {
* @return packed external credentials signature.
*/
private String packExternalCredentials(String ...values) {
StringJoiner result = new StringJoiner(":");
StringJoiner result = new StringJoiner("|");

for (String value : values) {
result.add(value);
Expand All @@ -40,7 +38,7 @@ private String packExternalCredentials(String ...values) {
* @return unpacked external credentials signature.
*/
private List<String> unpackExternalCredentials(String credentials) {
return List.of(credentials.split(":"));
return List.of(credentials.split("\\|"));
}

/**
Expand Down
7 changes: 6 additions & 1 deletion api-server/src/main/java/com/objectstorage/resource/ContentResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,12 @@ public byte[] v1ContentBackupDownloadPost(String authorization, ContentBackupDow
ValidationSecretsApplication validationSecretsApplication =
resourceConfigurationHelper.getJwtDetails(authorization);

return processorService.downloadBackup(contentBackupDownload, validationSecretsApplication);
ValidationSecretsUnit validationSecretsUnit =
resourceConfigurationHelper.getConfiguredProvider(
contentBackupDownload.getProvider(), validationSecretsApplication);

return processorService.downloadBackup(
contentBackupDownload.getLocation(), validationSecretsUnit, validationSecretsApplication);
}

/**
Expand Down
4 changes: 3 additions & 1 deletion api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,9 @@ public void process() throws BackupPeriodRetrievalFailureException {
try {
workspaceFacade.addBackupFile(
workspaceUnitKey,
workspaceFacade.createFileUnitKey(properties.getWorkspaceContentBackupUnit()),
workspaceFacade.createBackupFileUnitKey(
repositoryContentApplicationUnit.getProvider().toString(),
properties.getWorkspaceContentBackupUnit()),
folderContentUnits);
} catch (FileCreationFailureException e) {
StateService.getBackupProcessorGuard().unlock();
Expand Down
148 changes: 134 additions & 14 deletions api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid
List<ContentRetrievalBackupUnit> backups;

try {
backups = workspaceFacade.getBackupUnits(workspaceUnitKey);
backups = workspaceFacade.getBackupUnits(
workspaceUnitKey, validationSecretsUnit.getProvider().toString());
} catch (FileUnitsRetrievalFailureException e) {
throw new ProcessorContentRetrievalFailureException(e.getMessage());
}
Expand Down Expand Up @@ -262,7 +263,7 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp
String workspaceUnitKey =
workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication);

String fileUnitKey = workspaceFacade.createFileUnitKey(location);
String fileUnitKey = workspaceFacade.createObjectFileUnitKey(location);

for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) {
try {
Expand All @@ -280,6 +281,50 @@ public void uploadObject(String location, InputStream file, ValidationSecretsApp

throw new ProcessorContentUploadFailureException(e1.getMessage());
}

RepositoryContentUnitDto repositoryContentLocationUnitDto;

try {
repositoryContentLocationUnitDto = repositoryFacade.retrieveContentApplication(validationSecretsUnit);
} catch (ContentApplicationRetrievalFailureException e1) {
try {
repositoryExecutor.rollbackTransaction();
} catch (TransactionRollbackFailureException e2) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentUploadFailureException(e2.getMessage());
}

StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentUploadFailureException(e1.getMessage());
}

try {
if (!vendorFacade.isBucketPresent(
validationSecretsUnit.getProvider(),
validationSecretsUnit.getCredentials().getExternal(),
VendorConfigurationHelper.createBucketName(
repositoryContentLocationUnitDto.getRoot()))) {
vendorFacade.createBucket(
validationSecretsUnit.getProvider(),
validationSecretsUnit.getCredentials().getExternal(),
VendorConfigurationHelper.createBucketName(
repositoryContentLocationUnitDto.getRoot()));
}
} catch (SecretsConversionException | VendorOperationFailureException e1) {
try {
repositoryExecutor.rollbackTransaction();
} catch (TransactionRollbackFailureException e2) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentUploadFailureException(e2.getMessage());
}

StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentUploadFailureException(e1.getMessage());
}
}

try {
Expand Down Expand Up @@ -389,23 +434,27 @@ public byte[] downloadObject(
}

/**
* Downloads given content backup with the help of the given content backup download application.
* Downloads given content backup with the help of the given content backup download location and
* configured provider.
*
* @param contentBackupDownload given content backup download application.
* @param location given content object location.
* @param validationSecretsUnit given content secrets unit.
* @param validationSecretsApplication given content secrets application.
* @return downloaded content backup.
* @throws ProcessorContentDownloadFailureException if content backup download operation fails.
*/
public byte[] downloadBackup(
ContentBackupDownload contentBackupDownload,
String location,
ValidationSecretsUnit validationSecretsUnit,
ValidationSecretsApplication validationSecretsApplication)
throws ProcessorContentDownloadFailureException {
logger.info(String.format("Downloading content backup for '%s' location", contentBackupDownload.getLocation()));
logger.info(String.format("Downloading content backup for '%s' location", location));

String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication);

try {
if (!workspaceFacade.isBackupFilePresent(workspaceUnitKey, contentBackupDownload.getLocation())) {
if (!workspaceFacade.isBackupFilePresent(
workspaceUnitKey, validationSecretsUnit.getProvider().toString(), location)) {
throw new ProcessorContentDownloadFailureException(
new WorkspaceObjectNotPresentException().getMessage());
}
Expand All @@ -414,7 +463,8 @@ public byte[] downloadBackup(
}

try {
return workspaceFacade.getBackupFile(workspaceUnitKey, contentBackupDownload.getLocation());
return workspaceFacade.getBackupFile(
workspaceUnitKey, validationSecretsUnit.getProvider().toString(), location);
} catch (FileUnitRetrievalFailureException e) {
throw new ProcessorContentDownloadFailureException(e.getMessage());
}
Expand Down Expand Up @@ -592,19 +642,49 @@ public void removeAll(ValidationSecretsApplication validationSecretsApplication)
throw new ProcessorContentRemovalFailureException(e.getMessage());
}

StateService.getTransactionProcessorGuard().lock();

try {
repositoryExecutor.beginTransaction();
} catch (TransactionInitializationFailureException e) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e.getMessage());
}

for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) {
try {
repositoryFacade.removeTemporateContentByProviderAndSecret(validationSecretsUnit);
} catch (TemporateContentRemovalFailureException e) {
throw new ProcessorContentRemovalFailureException(e.getMessage());
} catch (TemporateContentRemovalFailureException e1) {
try {
repositoryExecutor.rollbackTransaction();
} catch (TransactionRollbackFailureException e2) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e2.getMessage());
}

StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e1.getMessage());
}

RepositoryContentUnitDto repositoryContentLocationUnitDto;

try {
repositoryContentLocationUnitDto = repositoryFacade.retrieveContentApplication(validationSecretsUnit);
} catch (ContentApplicationRetrievalFailureException e) {
throw new ProcessorContentRemovalFailureException(e.getMessage());
} catch (ContentApplicationRetrievalFailureException e1) {
try {
repositoryExecutor.rollbackTransaction();
} catch (TransactionRollbackFailureException e2) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e2.getMessage());
}

StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e1.getMessage());
}

try {
Expand All @@ -614,9 +694,49 @@ public void removeAll(ValidationSecretsApplication validationSecretsApplication)
VendorConfigurationHelper.createBucketName(
repositoryContentLocationUnitDto.getRoot())
);
} catch (SecretsConversionException | VendorOperationFailureException e) {
throw new ProcessorContentRemovalFailureException(e.getMessage());
} catch (SecretsConversionException | VendorOperationFailureException e1) {
try {
repositoryExecutor.rollbackTransaction();
} catch (TransactionRollbackFailureException e2) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e2.getMessage());
}

StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e1.getMessage());
}

try {
vendorFacade.removeBucket(
validationSecretsUnit.getProvider(),
validationSecretsUnit.getCredentials().getExternal(),
VendorConfigurationHelper.createBucketName(
repositoryContentLocationUnitDto.getRoot()));
} catch (VendorOperationFailureException | SecretsConversionException e1) {
try {
repositoryExecutor.rollbackTransaction();
} catch (TransactionRollbackFailureException e2) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e2.getMessage());
}

StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e1.getMessage());
}
}

try {
repositoryExecutor.commitTransaction();
} catch (TransactionCommitFailureException e) {
StateService.getTransactionProcessorGuard().unlock();

throw new ProcessorContentRemovalFailureException(e.getMessage());
}

StateService.getTransactionProcessorGuard().unlock();
}
}
Loading

0 comments on commit e6dc833

Please sign in to comment.