diff --git a/src/main/java/org/broad/igv/oauth/OAuthProvider.java b/src/main/java/org/broad/igv/oauth/OAuthProvider.java index 9e0734ee5..3bdda7f00 100644 --- a/src/main/java/org/broad/igv/oauth/OAuthProvider.java +++ b/src/main/java/org/broad/igv/oauth/OAuthProvider.java @@ -230,16 +230,6 @@ public void fetchAccessToken(String authorizationCode) throws IOException { fetchUserProfile(payload); } - if (authProvider != null && "Amazon".equals(authProvider)) { - // Get AWS credentials after getting relevant tokens - Credentials aws_credentials; - aws_credentials = AmazonUtils.GetCognitoAWSCredentials(); - - // Update S3 client with newly acquired token - AmazonUtils.updateS3Client(aws_credentials); - } - - // Notify UI that we are authz'd/authn'd if (isLoggedIn()) { IGVEventBus.getInstance().post(new AuthStateEvent(true, this.authProvider, this.getCurrentUserName())); @@ -247,7 +237,6 @@ public void fetchAccessToken(String authorizationCode) throws IOException { } catch (Exception e) { log.error(e); - e.printStackTrace(); } } @@ -256,7 +245,7 @@ public void setAccessToken(String accessToken) { } /** - * Fetch a new access token from a refresh token. + * Fetch a new access token from a refresh token. Unlike authorization, this is a synchronous operation * * @throws IOException */ @@ -293,18 +282,15 @@ private void refreshAccessToken() throws IOException { expirationTime = System.currentTimeMillis() + response.getAsJsonPrimitive("expires_in").getAsInt() * 1000; } else { // Refresh token has failed, reauthorize from scratch - reauthorize(); + logout(); + try { + openAuthorizationPage(); + } catch (URISyntaxException e) { + e.printStackTrace(); + } } } - private void reauthorize() throws IOException { - logout(); - try { - openAuthorizationPage(); - } catch (URISyntaxException e) { - e.printStackTrace(); - } - } /** * Extract user information from the claim information @@ -374,6 +360,15 @@ public void logout() { IGVEventBus.getInstance().post(new AuthStateEvent(false, this.authProvider, null)); } + public JsonObject getAuthorizationResponse() { + + if (response == null) { + // Go back to auth flow, not auth'd yet + checkLogin(); + response = getResponse(); + } + return response; + } /** * If not logged in, attempt to login @@ -390,10 +385,10 @@ public synchronized void checkLogin() { } } - // wait until authentication successful or 1 minute - + // wait until authentication successful or 2 minutes - // dwm08 int i = 0; - while (!isLoggedIn() && i < 600) { + while (!isLoggedIn() && i < 1200) { ++i; try { Thread.sleep(100); diff --git a/src/main/java/org/broad/igv/track/TrackLoader.java b/src/main/java/org/broad/igv/track/TrackLoader.java index ad4344006..3637a9997 100644 --- a/src/main/java/org/broad/igv/track/TrackLoader.java +++ b/src/main/java/org/broad/igv/track/TrackLoader.java @@ -120,11 +120,6 @@ public List load(ResourceLocator locator, Genome genome) throws DataLoadE final String path = locator.getPath().trim(); - // Check if the AWS credentials are still valid. If not, re-login and renew pre-signed urls - if (AmazonUtils.isAwsS3Path(path)) { - AmazonUtils.checkLogin(); - } - log.info("Loading resource: " + (locator.isDataURL() ? "" : path)); try { diff --git a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java index 12823f77a..d464afa99 100644 --- a/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java +++ b/src/main/java/org/broad/igv/ui/action/LoadFromURLMenuAction.java @@ -109,9 +109,7 @@ private void loadUrls(List inputs, List indexes, boolean isHtsGe } else if (inputs.size() == 1 && SessionReader.isSessionFile(inputs.getFirst())) { // Session URL String url = inputs.getFirst(); - if (url.startsWith("s3://")) { - checkAWSAccessbility(url); - } + try { LongRunningTask.submit(() -> this.igv.loadSession(url, null)); } catch (Exception ex) { @@ -186,29 +184,12 @@ private static boolean isHubURL(String input) { private static void checkURLs(List urls) { for (String url : urls) { - if (url.startsWith("s3://")) { - checkAWSAccessbility(url); - } else if (url.startsWith("ftp://")) { + if (url.startsWith("ftp://")) { MessageUtils.showMessage("FTP protocol is not supported"); } } } - private static void checkAWSAccessbility(String url) { - try { - // If AWS support is active, check if objects are in accessible tiers via Load URL menu... - if (AmazonUtils.isAwsS3Path(url)) { - String bucket = AmazonUtils.getBucketFromS3URL(url); - String key = AmazonUtils.getKeyFromS3URL(url); - AmazonUtils.s3ObjectAccessResult res = isObjectAccessible(bucket, key); - if (!res.isObjectAvailable()) { - MessageUtils.showErrorMessage(res.getErrorReason(), null); - } - } - } catch (NullPointerException npe) { - // User has not yet done Amazon->Login sequence - AmazonUtils.checkLogin(); - } - } + } diff --git a/src/main/java/org/broad/igv/util/AmazonUtils.java b/src/main/java/org/broad/igv/util/AmazonUtils.java index e832e5a67..c2caf9f4a 100644 --- a/src/main/java/org/broad/igv/util/AmazonUtils.java +++ b/src/main/java/org/broad/igv/util/AmazonUtils.java @@ -1,17 +1,15 @@ package org.broad.igv.util; import com.google.gson.JsonObject; -import org.broad.igv.Globals; +import org.broad.igv.DirectoryManager; import org.broad.igv.aws.IGVS3Object; import org.broad.igv.oauth.OAuthProvider; import org.broad.igv.oauth.OAuthUtils; import org.broad.igv.logging.LogManager; import org.broad.igv.logging.Logger; +import org.broad.igv.prefs.PreferencesManager; import org.broad.igv.ui.IGVMenuBar; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.auth.credentials.*; import software.amazon.awssdk.core.exception.SdkClientException; import software.amazon.awssdk.core.exception.SdkServiceException; import software.amazon.awssdk.regions.Region; @@ -23,19 +21,24 @@ import software.amazon.awssdk.services.cognitoidentity.model.GetOpenIdTokenRequest; import software.amazon.awssdk.services.cognitoidentity.model.GetOpenIdTokenResponse; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.*; import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; +import software.amazon.awssdk.services.s3.presigner.model.PresignedGetObjectRequest; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityRequest; import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityResponse; import software.amazon.awssdk.services.sts.model.Credentials; -import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.io.*; import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.Duration; import java.time.Instant; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -50,24 +53,15 @@ public class AmazonUtils { private static CognitoIdentityClient cognitoIdentityClient; private static Region AWSREGION; private static Boolean awsCredentialsPresent = null; - private static Credentials cognitoAWSCredentials = null; - private static int TOKEN_EXPIRE_GRACE_TIME = 1000 * 60; // 1 minute - - - - /** - * Maps s3:// URLs to presigned URLs - */ private static Map s3ToPresignedMap = new HashMap<>(); - /** - * Maps aws presigned URLs to s3://. This is needed in some cases (e.g. Tribble) to regenerate an expired URL - */ private static Map presignedToS3Map = new HashMap<>(); private static JsonObject CognitoConfig; + private static S3Presigner s3Presigner; + private static String endpointURL = "UNKNOWN"; public static void setCognitoConfig(JsonObject json) { CognitoConfig = json; @@ -97,7 +91,7 @@ public static boolean isAwsProviderPresent() { log.info("AWS configuration found. AWS support enabled."); awsCredentialsPresent = true; } else { - log.info("AWS configuration not found."); + log.info("Cognito configuration found but Amazon auth_provider not defined. Only Amazon provider is supported at this time."); awsCredentialsPresent = false; } } catch (NullPointerException np) { @@ -117,6 +111,11 @@ public static boolean isAwsProviderPresent() { return awsCredentialsPresent; } + /** + * Return the region for AWS credentials + * + * @return + */ private static Region getAWSREGION() { if (AWSREGION == null) { @@ -126,7 +125,12 @@ private static Region getAWSREGION() { // TODO -- find region in default place try { AWSREGION = (new DefaultAwsRegionProviderChain()).getRegion(); + if (AWSREGION == null) { + AWSREGION = Region.US_EAST_1; + log.info("Could not find AWS region setting. Assuming us-east-1"); + } } catch (Exception e) { + log.info("Unable to load region from any of the providers in the chain software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain. Assuming us-east-1"); AWSREGION = Region.US_EAST_1; } } @@ -135,7 +139,7 @@ private static Region getAWSREGION() { } /** - * Returns the AWS credentials + * Retrieve AWS credentials, which may trigger a login. * * @return returns the credentials based on the AWS STS access token returned from the AWS Cognito user pool. */ @@ -145,85 +149,73 @@ public static Credentials GetCognitoAWSCredentials() { log.debug("fetch credentials"); OAuthProvider provider = OAuthUtils.getInstance().getAWSProvider(); - JsonObject igv_oauth_conf = GetCognitoConfig(); - JsonObject response = provider.getResponse(); + JsonObject response = provider.getAuthorizationResponse(); - // Handle non-user initiated S3 auth (IGV early startup), i.e user-specified GenomesLoader - if (response == null) { - // Go back to auth flow, not auth'd yet - checkLogin(); - response = provider.getResponse(); - } - - JsonObject payload = JWTParser.getPayload(response.get("id_token").getAsString()); - - log.debug("JWT payload id token: " + payload); - - // Collect necessary information from federated IdP for Authentication purposes - String idTokenStr = response.get("id_token").getAsString(); - String idProvider = payload.get("iss").toString().replace("https://", "") - .replace("\"", ""); - String email = payload.get("email").getAsString(); - String federatedPoolId = igv_oauth_conf.get("aws_cognito_fed_pool_id").getAsString(); - String cognitoRoleARN = igv_oauth_conf.get("aws_cognito_role_arn").getAsString(); - - HashMap logins = new HashMap<>(); - logins.put(idProvider, idTokenStr); - - // Avoid "software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(" - // The use of the AnonymousCredentialsProvider essentially bypasses the provider chain's requirement to access ~/.aws/credentials. - // https://stackoverflow.com/questions/36604024/sts-saml-and-java-sdk-unable-to-load-aws-credentials-from-any-provider-in-the-c - AnonymousCredentialsProvider anoCredProv = AnonymousCredentialsProvider.create(); - - // https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cognitoidentity/CognitoIdentityClient.html - // Build the Cognito client - CognitoIdentityClientBuilder cognitoIdentityBuilder = CognitoIdentityClient.builder(); - - cognitoIdentityBuilder.region(getAWSREGION()).credentialsProvider(anoCredProv); - cognitoIdentityClient = cognitoIdentityBuilder.build(); - - - // https://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html - // Basic (Classic) Authflow - // Uses AssumeRoleWithWebIdentity and facilitates CloudTrail org.broad.igv.logging. Uses one more request but provides user traceability. - GetIdRequest.Builder idrequest = GetIdRequest.builder().identityPoolId(federatedPoolId) - .logins(logins); - GetIdResponse idResult = cognitoIdentityClient.getId(idrequest.build()); - - GetOpenIdTokenRequest.Builder openidrequest = GetOpenIdTokenRequest.builder().logins(logins).identityId(idResult.identityId()); - GetOpenIdTokenResponse openId = cognitoIdentityClient.getOpenIdToken(openidrequest.build()); - - - AssumeRoleWithWebIdentityRequest.Builder webidrequest = AssumeRoleWithWebIdentityRequest.builder().webIdentityToken(openId.token()) - .roleSessionName(email) - .roleArn(cognitoRoleARN); - - AssumeRoleWithWebIdentityResponse stsClientResponse = StsClient.builder().credentialsProvider(anoCredProv) - .region(getAWSREGION()) - .build() - .assumeRoleWithWebIdentity(webidrequest.build()); - -// // Enhanced (Simplified) Authflow -// // Major drawback: Does not store federated user information on CloudTrail only authenticated role name appears in logs. -// -// // "To provide end-user credentials, first make an unsigned call to GetId." -// GetIdRequest.Builder idrequest = GetIdRequest.builder().identityPoolId(federatedPoolId) -// .logins(logins); -// GetIdResponse idResult = cognitoIdentityClient.getId(idrequest.build()); -// -// // "Next, make an unsigned call to GetCredentialsForIdentity." -// GetCredentialsForIdentityRequest.Builder authedIds = GetCredentialsForIdentityRequest.builder(); -// authedIds.identityId(idResult.identityId()).logins(logins); -// -// GetCredentialsForIdentityResponse authedRes = cognitoIdentityClient.getCredentialsForIdentity(authedIds.build()); -// -// return authedRes.credentials() - - cognitoAWSCredentials = stsClientResponse.credentials(); + setCredentialsFromOauthResponse(response); } return cognitoAWSCredentials; } + private static void setCredentialsFromOauthResponse(JsonObject response) { + + JsonObject igv_oauth_conf = GetCognitoConfig(); + + JsonObject payload = JWTParser.getPayload(response.get("id_token").getAsString()); + + log.debug("JWT payload id token: " + payload); + + // Collect necessary information from federated IdP for Authentication purposes + String idTokenStr = response.get("id_token").getAsString(); + String idProvider = payload.get("iss").toString().replace("https://", "") + .replace("\"", ""); + String email = payload.get("email").getAsString(); + String federatedPoolId = igv_oauth_conf.get("aws_cognito_fed_pool_id").getAsString(); + String cognitoRoleARN = igv_oauth_conf.get("aws_cognito_role_arn").getAsString(); + + HashMap logins = new HashMap<>(); + logins.put(idProvider, idTokenStr); + + // Avoid "software.amazon.awssdk.core.exception.SdkClientException: Unable to load credentials from any of the providers in the chain AwsCredentialsProviderChain(" + // The use of the AnonymousCredentialsProvider essentially bypasses the provider chain's requirement to access ~/.aws/credentials. + // https://stackoverflow.com/questions/36604024/sts-saml-and-java-sdk-unable-to-load-aws-credentials-from-any-provider-in-the-c + AnonymousCredentialsProvider anoCredProv = AnonymousCredentialsProvider.create(); + + // https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/cognitoidentity/CognitoIdentityClient.html + // Build the Cognito client + CognitoIdentityClientBuilder cognitoIdentityBuilder = CognitoIdentityClient.builder(); + + cognitoIdentityBuilder.region(getAWSREGION()).credentialsProvider(anoCredProv); + cognitoIdentityClient = cognitoIdentityBuilder.build(); + + + // https://docs.aws.amazon.com/cognito/latest/developerguide/authentication-flow.html + // Basic (Classic) Authflow + // Uses AssumeRoleWithWebIdentity and facilitates CloudTrail org.broad.igv.logging. Uses one more request but provides user traceability. + GetIdRequest.Builder idrequest = GetIdRequest.builder() + .identityPoolId(federatedPoolId) + .logins(logins); + GetIdResponse idResult = cognitoIdentityClient.getId(idrequest.build()); + + GetOpenIdTokenRequest.Builder openidrequest = GetOpenIdTokenRequest.builder().logins(logins).identityId(idResult.identityId()); + GetOpenIdTokenResponse openId = cognitoIdentityClient.getOpenIdToken(openidrequest.build()); + + + AssumeRoleWithWebIdentityRequest.Builder webidrequest = AssumeRoleWithWebIdentityRequest.builder() + .webIdentityToken(openId.token()) + .roleSessionName(email) + .roleArn(cognitoRoleARN); + + AssumeRoleWithWebIdentityResponse stsClientResponse = StsClient.builder() + .credentialsProvider(anoCredProv) + .region(getAWSREGION()) + .build() + .assumeRoleWithWebIdentity(webidrequest.build()); + + cognitoAWSCredentials = stsClientResponse.credentials(); + + updateS3Client(cognitoAWSCredentials); + } + /** * @param cognitoAWSCredentials * @return true if credentials are due to expire within next 10 seconds* @@ -238,11 +230,35 @@ private static boolean isExpired(Credentials cognitoAWSCredentials) { * * @param credentials AWS credentials */ - public static void updateS3Client(Credentials credentials) { + private static void updateS3Client(Credentials credentials) { + final Region region = getAWSREGION(); if (credentials == null) { - s3Client = S3Client.builder().region(region).build(); + // .aws/credentials, environment variable, or other AWS supported credential store + String endpointURL = null; + try { + endpointURL = getEndpointURL(); + } catch (IOException e) { + log.error("Error searching for endpoint url", e); + } + + if (endpointURL == null) { + s3Client = S3Client.builder().region(region).build(); + } else { + // Custom endpoint + try { + s3Client = S3Client.builder() + .endpointOverride(new URI(endpointURL)) + .serviceConfiguration(srvcConf -> srvcConf.pathStyleAccessEnabled()) + .region(getAWSREGION()) // this is not used, but the AWS SDK requires it + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } else { + // Cognito AwsSessionCredentials creds = AwsSessionCredentials.create( credentials.accessKeyId(), credentials.secretAccessKey(), @@ -261,19 +277,16 @@ public static void updateS3Client(Credentials credentials) { * @return bucket list */ public static List ListBucketsForUser() { + if (bucketsFinalList.isEmpty()) { if (GetCognitoConfig() != null) { - OAuthUtils.getInstance().getAWSProvider().getAccessToken(); updateS3Client(GetCognitoAWSCredentials()); } else { updateS3Client(null); } - List bucketsList = new ArrayList<>(); - ListBucketsRequest listBucketsRequest = ListBucketsRequest.builder().build(); - ListBucketsResponse listBucketsResponse = s3Client.listBuckets(listBucketsRequest); - listBucketsResponse.buckets().stream().forEach(x -> bucketsList.add(x.name())); + List bucketsList = s3Client.listBuckets().buckets().stream().map(b -> b.name()).collect(Collectors.toList()); // Filter out buckets that the user does not have permissions for bucketsFinalList = getReadableBuckets(bucketsList); @@ -484,41 +497,158 @@ public static String getKeyFromS3URL(String s3URL) { return s3URI.getKey(); } - // Amazon S3 Presign URLs - // Also keeps an internal mapping between ResourceLocator and active/valid signed URLs. + /** + * Create a presigned URL for the s3:// + * + * @param s3Path + * @return + * @throws IOException + */ + private static String createPresignedURL(String s3Path) throws IOException { - // TODO: Ideally the presigned URL should be generated without any of the Cognito being involved first? - // Make sure access token are valid (refreshes token internally) - S3Presigner s3Presigner; + s3Presigner = getPresigner(); - if (GetCognitoConfig() != null) { - OAuthProvider provider = OAuthUtils.getInstance().getAWSProvider(); - provider.getAccessToken(); + AmazonS3URI s3URI = new AmazonS3URI(s3Path); + String bucket = s3URI.getBucket(); + String key = s3URI.getKey(); - Credentials credentials = GetCognitoAWSCredentials(); - AwsSessionCredentials creds = AwsSessionCredentials.create(credentials.accessKeyId(), - credentials.secretAccessKey(), - credentials.sessionToken()); - StaticCredentialsProvider awsCredsProvider = StaticCredentialsProvider.create(creds); + // URI presigned = s3Presigner.presignS3DownloadLink(bucket, key); + GetObjectRequest s3GetRequest = GetObjectRequest.builder().bucket(bucket).key(key).build(); + GetObjectPresignRequest getObjectPresignRequest = + GetObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(60)) + .getObjectRequest(s3GetRequest) + .build(); - s3Presigner = S3Presigner.builder() - .expiration(provider.getExpirationTime()) // Duration.ofSeconds(30) // <= for testing - .awsCredentials(awsCredsProvider) - .region(getAWSREGION()) - .build(); - } else { - s3Presigner = S3Presigner.builder().build(); - } + PresignedGetObjectRequest presignedGetObjectRequest = + s3Presigner.presignGetObject(getObjectPresignRequest); - String bucket = getBucketFromS3URL(s3Path); - String key = getKeyFromS3URL(s3Path); + URL presigned = presignedGetObjectRequest.url(); - URI presigned = s3Presigner.presignS3DownloadLink(bucket, key); log.debug("AWS presigned URL from translateAmazonCloudURL is: " + presigned); return presigned.toString(); } + private static S3Presigner getPresigner() throws IOException { + + if (s3Presigner == null) { + + if (GetCognitoConfig() != null) { + OAuthProvider provider = OAuthUtils.getInstance().getAWSProvider(); + provider.getAccessToken(); + + Credentials credentials = GetCognitoAWSCredentials(); + AwsSessionCredentials creds = AwsSessionCredentials.create(credentials.accessKeyId(), + credentials.secretAccessKey(), + credentials.sessionToken()); + StaticCredentialsProvider awsCredsProvider = StaticCredentialsProvider.create(creds); + + s3Presigner = S3Presigner.builder() + .credentialsProvider(awsCredsProvider) + .region(getAWSREGION()) + .build(); + } else { + final String endpointURL = getEndpointURL(); + + if (endpointURL == null) { + s3Presigner = S3Presigner.builder().build(); + } else { + // Override presigned url style -- some (all?) 3rd party S3 providers do not support virtual host style + S3Configuration configuration = S3Configuration.builder() + .pathStyleAccessEnabled(true).build(); + try { + s3Presigner = S3Presigner.builder() + .serviceConfiguration(configuration) + .endpointOverride(new URI(endpointURL)) + .region(getAWSREGION()) + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } + } + return s3Presigner; + } + + /** + * Return a custom endpoint URL, if defined. The endpointURL is searched for once per session and not updated. + * Search in the follow locations. + *

+ * (1) oauth config file + * (2) igv preference + * (3) environment variable AWS_ENDPOINT_URL + * (4) aws credentials file + * (5) aws config file + * + * @return Custom AWS endpoint url or null + * @throws IOException + */ + private static String getEndpointURL() throws IOException { + + if ("UNKNOWN".equals(endpointURL)) { + + // IGV preference + endpointURL = PreferencesManager.getPreferences().get("endpoint_url"); + if (endpointURL != null) { + return endpointURL; + } + + // environment variable + endpointURL = System.getenv("AWS_ENDPOINT_URL"); + if (endpointURL != null) { + return endpointURL; + } + + // oauth config + if (GetCognitoConfig() != null && GetCognitoConfig().has("endpoint_url")) { + endpointURL = GetCognitoConfig().get("endpoint_url").getAsString(); + if (endpointURL != null) { + return endpointURL; + } + } + + // Search aws directory + File awsDirectory = new File(DirectoryManager.getUserHome(), ".aws"); + if (awsDirectory.exists()) { + File credfile = new File(awsDirectory, "credentials"); + if (credfile.exists()) { + BufferedReader br = new BufferedReader(new FileReader(credfile)); + String nextLine; + while ((nextLine = br.readLine()) != null) { + String[] tokens = nextLine.split("="); + if (tokens.length == 2) { + String key = tokens[0].trim(); + if (key.equals("endpoint_url")) { + endpointURL = tokens[1].trim(); + return endpointURL; + } + } + } + } + credfile = new File(awsDirectory, "config"); + if (credfile.exists()) { + BufferedReader br = new BufferedReader(new FileReader(credfile)); + String nextLine; + while ((nextLine = br.readLine()) != null) { + String[] tokens = nextLine.split("="); + if (tokens.length == 2) { + String key = tokens[0].trim(); + if (key.equals("endpoint_url")) { + endpointURL = tokens[1].trim(); + return endpointURL; + } + } + } + } + } + } + + + return endpointURL; + } + /** * @param s3UrlString * @return @@ -540,29 +670,21 @@ public static Boolean isAwsS3Path(String path) { return (path.startsWith("s3://")); } - public static boolean isPresignedURL(String urlString) { + public static boolean isKnownPresignedURL(String urlString) { return presignedToS3Map.containsKey(urlString); } public static String updatePresignedURL(String urlString) throws IOException { String s3UrlString = presignedToS3Map.get(urlString); if (s3UrlString == null) { - throw new RuntimeException("Unrecognized presigned url: " + urlString); + // We haven't seen this url before. This shouldn't happen, but if it does we have to assume its valid + log.info("Unrecognized presigned url: " + urlString); + return urlString; } else { return translateAmazonCloudURL(s3UrlString); } } - /** - * If using Cognito, check that the use is logged in, and prompt for login if not. - */ - public static void checkLogin() { - if (GetCognitoConfig() != null && - !OAuthUtils.getInstance().getAWSProvider().isLoggedIn()) { - OAuthUtils.getInstance().getAWSProvider().checkLogin(); - } - } - /** * Checks whether a (pre)signed url is still accessible or it has expired, offline. * No extra request/head is required to the presigned object since we have all information @@ -575,34 +697,30 @@ public static void checkLogin() { **/ private static boolean isPresignedURLValid(URL url) { - boolean isValidSignedUrl; + boolean isValidSignedUrl; try { - long presignedTime = signedURLValidity(url); + Map params = StringUtils.splitQuery(url); + String amzDateStr = params.get("X-Amz-Date"); + long amzExpires = Long.parseLong(params.get("X-Amz-Expires")); + + SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); + formatter.setTimeZone(TimeZone.getTimeZone("UTC")); // Z(ulu) -> UTC + Date amzDate = formatter.parse(amzDateStr); + + long presignedTime = amzDate.getTime() + amzExpires * 1000; + log.debug("The date of expiration is " + amzDate + ", expires after " + amzExpires + " seconds for url: " + url); + isValidSignedUrl = presignedTime - System.currentTimeMillis() - TOKEN_EXPIRE_GRACE_TIME > 0; // Duration in milliseconds } catch (ParseException e) { log.error("The AWS signed URL date parameter X-Amz-Date has incorrect formatting"); isValidSignedUrl = false; } catch (UnsupportedEncodingException e) { - e.printStackTrace(); + log.error("Error decoding signed url", e); isValidSignedUrl = false; } return isValidSignedUrl; } - private static long signedURLValidity(URL url) throws ParseException, UnsupportedEncodingException { - Map params = StringUtils.splitQuery(url); - String amzDateStr = params.get("X-Amz-Date"); - long amzExpires = Long.parseLong(params.get("X-Amz-Expires")); - - SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'"); - formatter.setTimeZone(TimeZone.getTimeZone("UTC")); // Z(ulu) -> UTC - Date amzDate = formatter.parse(amzDateStr); - - long timeOfExpirationMillis = amzDate.getTime() + amzExpires * 1000; - - log.debug("The date of expiration is " + amzDate + ", expires after " + amzExpires + " seconds for url: " + url); - return timeOfExpirationMillis; - } } diff --git a/src/main/java/org/broad/igv/util/HttpUtils.java b/src/main/java/org/broad/igv/util/HttpUtils.java index 086d035ce..38734e461 100644 --- a/src/main/java/org/broad/igv/util/HttpUtils.java +++ b/src/main/java/org/broad/igv/util/HttpUtils.java @@ -715,11 +715,10 @@ private HttpURLConnection openConnection( } // If a presigned URL, check its validity and update if needed - if (AmazonUtils.isPresignedURL(url.toExternalForm())) { + if (AmazonUtils.isKnownPresignedURL(url.toExternalForm())) { url = new URL(AmazonUtils.updatePresignedURL(url.toExternalForm())); } - // If an S3 url, obtain a signed https url if (AmazonUtils.isAwsS3Path(url.toExternalForm())) { url = new URL(AmazonUtils.translateAmazonCloudURL(url.toExternalForm())); diff --git a/src/main/java/org/broad/igv/util/S3Presigner.java b/src/main/java/org/broad/igv/util/S3Presigner.java deleted file mode 100644 index cd50fe489..000000000 --- a/src/main/java/org/broad/igv/util/S3Presigner.java +++ /dev/null @@ -1,195 +0,0 @@ -package org.broad.igv.util; - -/* -* -* This is a transitional class until the official java-aws-sdk-v2 includes a S3 URL presigners class, see: -* -* https://github.com/aws/aws-sdk-java-v2/issues/849#issuecomment-468892839 -* https://github.com/aws/aws-sdk-java-v2/issues/203 -* -*/ - -import java.io.ByteArrayInputStream; -import java.net.URI; -import java.time.Duration; -import java.time.Instant; - -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; -import software.amazon.awssdk.auth.signer.AwsS3V4Signer; -import software.amazon.awssdk.auth.signer.AwsSignerExecutionAttribute; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; -import software.amazon.awssdk.core.exception.SdkClientException; -import software.amazon.awssdk.core.interceptor.Context.BeforeTransmission; -import software.amazon.awssdk.core.interceptor.ExecutionAttributes; -import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; -import software.amazon.awssdk.core.sync.RequestBody; -import software.amazon.awssdk.http.AbortableInputStream; -import software.amazon.awssdk.http.ExecutableHttpRequest; -import software.amazon.awssdk.http.HttpExecuteRequest; -import software.amazon.awssdk.http.HttpExecuteResponse; -import software.amazon.awssdk.http.SdkHttpClient; -import software.amazon.awssdk.http.SdkHttpFullRequest; -import software.amazon.awssdk.http.SdkHttpResponse; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; - -public class S3Presigner { - - private Region region; - private AwsCredentialsProvider awsCredentialsProvider; - private Duration expirationTime; - private Integer timeOffset; - private S3PresignExecutionInterceptor presignInterceptor; - private S3Client s3Client; - - private S3Presigner() { - presignInterceptor = new S3PresignExecutionInterceptor(); - } - - public static Builder builder() { - return new Builder(); - } - - public URI presignS3DownloadLink(String bucketName, String fileName) throws SdkClientException { - try { - - GetObjectRequest s3GetRequest = GetObjectRequest.builder().bucket(bucketName).key(fileName).build(); - ResponseInputStream response = s3Client.getObject(s3GetRequest); - response.close(); - - return presignInterceptor.getSignedURI(); - } catch (Throwable t) { - if (t instanceof SdkClientException) { - throw (SdkClientException) t; - } - throw SdkClientException.builder().cause(t).build(); - } - } - - public URI presignS3UploadLink(String bucketName, String fileName) throws SdkClientException { - try { - - PutObjectRequest s3PutRequest = PutObjectRequest.builder().bucket(bucketName).key(fileName).build(); - PutObjectResponse response = s3Client.putObject(s3PutRequest, RequestBody.empty()); - - return presignInterceptor.getSignedURI(); - } catch (Throwable t) { - if (t instanceof SdkClientException) { - throw (SdkClientException) t; - } - throw SdkClientException.builder().cause(t).build(); - } - } - - public static class Builder { - S3Presigner presigner = new S3Presigner(); - - public S3Presigner build() { - if (presigner.awsCredentialsProvider == null) { - DefaultCredentialsProvider provider = DefaultCredentialsProvider.create(); - presigner.awsCredentialsProvider = provider; - } - - if (presigner.region == null) { - presigner.region = new DefaultAwsRegionProviderChain().getRegion(); - } - - if (presigner.expirationTime == null) { - presigner.expirationTime = Duration.ofDays(4); - } - - if (presigner.timeOffset == null) { - presigner.timeOffset = 2; - } - - S3ClientBuilder s3Builder = S3Client.builder().region(presigner.region).credentialsProvider(presigner.awsCredentialsProvider); - s3Builder.overrideConfiguration(ClientOverrideConfiguration.builder().addExecutionInterceptor(presigner.presignInterceptor).build()); - s3Builder.httpClient(new NullSdkHttpClient()); - presigner.s3Client = s3Builder.build(); - - return presigner; - } - - public Builder awsCredentials(AwsCredentialsProvider awsCredentialsProvider) { - presigner.awsCredentialsProvider = awsCredentialsProvider; - return this; - } - - public Builder region(Region region) { - presigner.region = region; - return this; - } - - public Builder expiration(Duration expirationTime) { - presigner.expirationTime = expirationTime; - return this; - } - - public Builder timeOffset(Integer timeOffset) { - presigner.timeOffset = timeOffset; - return this; - } - - } - - public static class NullSdkHttpClient implements SdkHttpClient { - - @Override - public void close() { - - } - - @Override - public ExecutableHttpRequest prepareRequest(HttpExecuteRequest request) { - return new ExecutableHttpRequest() { - @Override - public HttpExecuteResponse call() { - return HttpExecuteResponse.builder().response(SdkHttpResponse.builder().statusCode(200).build()).responseBody(AbortableInputStream.create(new ByteArrayInputStream(new byte[0]))).build(); - } - - @Override - public void abort() { - } - }; - } - } - - public class S3PresignExecutionInterceptor implements ExecutionInterceptor { - - final private AwsS3V4Signer signer; - private URI signedURI; - - public S3PresignExecutionInterceptor() { - this.signer = AwsS3V4Signer.create(); - } - - @Override - public void beforeTransmission(BeforeTransmission context, ExecutionAttributes executionAttributes) { - // remove all headers because a Browser that downloads the shared URL will not send the exact values. X-Amz-SignedHeaders should only contain the host header. - SdkHttpFullRequest modifiedSdkRequest = (SdkHttpFullRequest) context.httpRequest().toBuilder().clearHeaders().build(); - - - executionAttributes.putAttribute(AwsSignerExecutionAttribute.AWS_CREDENTIALS, awsCredentialsProvider.resolveCredentials()); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.PRESIGNER_EXPIRATION, Instant.ofEpochSecond(System.currentTimeMillis()/1000).plus(expirationTime)); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.SERVICE_SIGNING_NAME, "s3"); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.SIGNING_REGION, region); - executionAttributes.putAttribute(AwsSignerExecutionAttribute.TIME_OFFSET, timeOffset); - SdkHttpFullRequest signedRequest = signer.presign(modifiedSdkRequest, executionAttributes);// sign(getRequest, new ExecutionAttributes()); - signedURI = signedRequest.getUri(); - } - - public URI getSignedURI() { - return signedURI; - } - - } - -}