Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename COLLABORATORY Strorage Profile to S3 in SCORe #480

Merged
merged 2 commits into from
Sep 11, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ KeyczarTest.java
HelperTest.java
DownloadPartRequest.java

# collaboratory
# s3
service.jks
application-prod.properties

Expand Down
2 changes: 1 addition & 1 deletion Dockerfile.server
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,6 @@ CMD mkdir -p $SCORE_HOME $SCORE_LOGS \
&& java -Dlog.path=$SCORE_LOGS \
-jar $JAR_FILE \
--spring.config.location=classpath:/application.yml \
--spring.profiles.active=amazon,collaboratory,prod,secure
--spring.profiles.active=amazon,prod,secure

#&& FOR_100_YEARS=$((100*365*24*60*60));while true;do sleep $FOR_100_YEARS;done
2 changes: 1 addition & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ services:
dockerfile: "$DOCKERFILE_NAME"
target: server
environment:
SPRING_PROFILES_ACTIVE: amazon,collaboratory,prod,secure
SPRING_PROFILES_ACTIVE: amazon,prod,secure
SERVER_PORT: 8080
OBJECT_SENTINEL: heliograph
BUCKET_NAME_OBJECT: score.data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class ProfileConfig {
private String endpoint;

@Autowired
@Value("${defaultProfile:collaboratory}")
@Value("${defaultProfile:s3}")
private String defaultProfile;

@Autowired
Expand Down
2 changes: 1 addition & 1 deletion score-client/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ token:


isTest: false
defaultProfile: collaboratory
defaultProfile: s3

---

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
public class TestProfileConfig {

@Autowired
@Value("${defaultProfile:collaboratory}")
@Value("${defaultProfile:s3}")
private String defaultProfile;

@Bean
Expand Down
2 changes: 1 addition & 1 deletion score-client/src/test/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ token:


isTest: true
defaultProfile: collaboratory
defaultProfile: s3

---
###############################################################################
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

public enum StorageProfiles {
AZURE("azure", "az"),
COLLABORATORY("collaboratory", "s3");
S3("s3", "s3");

private final String profileKey;
private final String profileValue;
Expand Down
2 changes: 1 addition & 1 deletion score-server/src/main/conf/application.properties
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@
# s3.secretKey=
# server.ssl.key-store-password=
# s3.masterEncryptionKeyId=
# collaboratory.bucket.name=
# s3.bucket.name=
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,10 @@
public class UploadCleanupService {

/** Configuration. */
@Value("${collaboratory.data.directory}")
@Value("${s3.data.directory}")
private String dataDir;

@Value("${collaboratory.upload.expiration}")
@Value("${s3.upload.expiration}")
private int expiration;

/** Dependencies. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public class AzureURLGenerator implements URLGenerator {
@Value("${bucket.policy.download}")
private String downloadPolicy;

@Value("${collaboratory.download.expiration}")
@Value("${s3.download.expiration}")
private int expiration;

@Autowired private CloudBlobContainer azureContainer;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@ public class S3DownloadService implements DownloadService {
private static final String PUBLISHED_ANALYSIS_STATE = "PUBLISHED";

/** Configuration. */
@Value("${collaboratory.data.directory}")
@Value("${s3.data.directory}")
private String dataDir;

@Value("${collaboratory.download.expiration}")
@Value("${s3.download.expiration}")
private int expiration;

@Value("${object.sentinel}")
Expand Down Expand Up @@ -210,7 +210,7 @@ public ObjectSpecification download(
private String getObjectMd5(ObjectMetadata metadata) {
val contentMd5 = metadata.getContentMD5();
if (contentMd5 != null) {
return MD5s.toHex(contentMd5);
return MD5s.toHex(contentMd5);
}
val userMetadataMd5 =
metadata.getUserMetaDataOf(s3config.getCustomMd5Property()); // get literal from config
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,9 @@ public class S3ListingService implements ListingService {
@Value("${bucket.name.object}")
private String bucketName;

@Value("${collaboratory.data.directory}")
@Value("${s3.data.directory}")
private String dataDir;

// @Value("${collaboratory.bucket.poolsize}")
// private int bucketPoolSize;
// @Value("${collaboratory.bucket.keysize}")
// private int bucketKeySize;

/** Dependencies. */
@Autowired private AmazonS3 s3;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ public class S3UploadService implements UploadService {
private static final String UNPUBLISHED_ANALYSIS_STATE = "UNPUBLISHED";

/** Configuration. */
@Value("${collaboratory.data.directory}")
@Value("${s3.data.directory}")
private String dataDir;

@Value("${collaboratory.upload.expiration}")
@Value("${s3.upload.expiration}")
private int expiration;

@Value("${metadata.useLegacyMode:false}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,10 @@ public class S3UploadStateStore implements UploadStateStore {
private static final ObjectMapper MAPPER = new ObjectMapper();

/** Configuration. */
@Value("${collaboratory.data.directory}")
@Value("${s3.data.directory}")
private String dataDir;

@Value("${collaboratory.upload.directory}")
@Value("${s3.upload.directory}")
private String uploadDir;

/** Dependencies. */
Expand Down
9 changes: 3 additions & 6 deletions score-server/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ server:
s3:
secured: true
sigV4Enabled: true
upload.directory: upload
upload.expiration: 6
data.directory: data

# custom meta property with md5 hash, unused when upload state files are available (default behaviour)
# customMd5Property: md5chksum
Expand All @@ -56,12 +59,6 @@ bucket:

object:
sentinel: heliograph

collaboratory:
upload.directory: upload
upload.expiration: 6
data.directory: data

# COL-131: Change pre-signed URLs TTL to 1 day max
download.expiration: 1

Expand Down