diff --git a/gradle.properties b/gradle.properties index 3f863afb5..9c211faa4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=0.91.8 +version=0.91.9 springVersion=4.3.3.RELEASE springBootVersion=1.4.1.RELEASE jerseyVersion=2.24 diff --git a/micro-s3/build.gradle b/micro-s3/build.gradle index 5095e1305..4cf6e735a 100644 --- a/micro-s3/build.gradle +++ b/micro-s3/build.gradle @@ -5,7 +5,7 @@ dependencies { compile 'com.amazonaws:aws-java-sdk:' + s3Version compile project(':micro-core') compile project(':micro-manifest-comparator') - testCompile group: 'org.springframework', name: 'spring-test', version: '4.0.5.RELEASE' + testCompile group: 'org.springframework', name: 'spring-test', version: springVersion testCompile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0' testCompile group: 'org.coursera', name: 'dropwizard-metrics-datadog', version: '1.1.6' testCompile group: 'com.github.cbismuth', name: 'junit-repeat-rule', version: '1.1.0' diff --git a/micro-s3/readme.md b/micro-s3/readme.md index cf9bf891a..0dbf77b5d 100644 --- a/micro-s3/readme.md +++ b/micro-s3/readme.md @@ -181,8 +181,14 @@ Then inject in either DistributedMap or it's concrete implementation S3Distribu ## Usage AmazonS3Client -This plugin also provides an AmazonS3Client implementation bean. You should define properties for -s3.accessKey, s3.secretKey and optionall s3.sessionToken (optionally - only for short term keys) +This plugin also provides an AmazonS3Client implementation bean. + +You may configure AWS credentials in one of two ways. +The first is to use the AWS default chain for configuration, by setting `s3.useDefaultChain` to `true`. +This allows you to rely on instance roles if your client is hosted on AWS, or to specify your credentials as environment variables if not. +For full details on the default AWS chain, see the [official AWS docs](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) + +The second option is to define properties for `s3.accessKey`, `s3.secretKey` and optionally `s3.sessionToken` (optionally - only for short term keys) [AmazonS3Client](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Client.html) Configure the S3 Bucket for Manifest comparision via this property diff --git a/micro-s3/src/main/java/com/aol/micro/server/s3/S3Configuration.java b/micro-s3/src/main/java/com/aol/micro/server/s3/S3Configuration.java index 871a5d521..076830cce 100644 --- a/micro-s3/src/main/java/com/aol/micro/server/s3/S3Configuration.java +++ b/micro-s3/src/main/java/com/aol/micro/server/s3/S3Configuration.java @@ -5,6 +5,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.Objects; + @Component @Getter public class S3Configuration { @@ -13,21 +15,30 @@ public class S3Configuration { private final String secretKey; private final String sessionToken; private final String region; + private final boolean defaultChainEnabled; private final int uploadThreads; private final String uploadThreadNamePrefix; private final int maxConnections; @Autowired - public S3Configuration(@Value("${s3.accessKey}") String accessKey, - @Value("${s3.secretKey}") String secretKey, + public S3Configuration(@Value("${s3.accessKey:#{null}}") String accessKey, + @Value("${s3.secretKey:#{null}}") String secretKey, + @Value("${s3.useDefaultChain:false}") boolean defaultChainEnabled, @Value("${s3.sessionToken:#{null}}") String sessionToken, @Value("${s3.region:#{null}}") String region, @Value("${s3.upload.threads:5}") int uploadThreads, @Value("${s3.upload.thread.name.prefix:s3-transfer-manager-worker-}") String uploadThreadNamePrefix, @Value("${s3.client.maxConnections:100}") int maxConnections) { + + if((Objects.isNull(accessKey) || Objects.isNull(secretKey)) && !defaultChainEnabled) { + throw new RuntimeException("No S3 authorization method provided. " + + "Please either enable the aws default credentials chain with s3.useDefaultChain, " + + "or provide access keys via s3.accessKey and s3.secretKey (and optionally s3.sessionToken)"); + } this.accessKey = accessKey; this.secretKey = secretKey; this.sessionToken = sessionToken; + this.defaultChainEnabled = defaultChainEnabled; this.region = region; this.uploadThreads = uploadThreads; this.uploadThreadNamePrefix = uploadThreadNamePrefix; diff --git a/micro-s3/src/main/java/com/aol/micro/server/s3/plugin/S3ClientProvider.java b/micro-s3/src/main/java/com/aol/micro/server/s3/plugin/S3ClientProvider.java index 1705ef3d4..20cecf96c 100644 --- a/micro-s3/src/main/java/com/aol/micro/server/s3/plugin/S3ClientProvider.java +++ b/micro-s3/src/main/java/com/aol/micro/server/s3/plugin/S3ClientProvider.java @@ -1,6 +1,7 @@ package com.aol.micro.server.s3.plugin; import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; @@ -24,14 +25,7 @@ public class S3ClientProvider { @Bean public AmazonS3Client getClient() { - AWSCredentials credentials; - - if (s3Configuration.getSessionToken() == null) { - credentials = new BasicAWSCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey()); - } else { - credentials = new BasicSessionCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey(), - s3Configuration.getSessionToken()); - } + AWSCredentials credentials = getAwsCredentials(); AmazonS3Client amazonS3Client = new AmazonS3Client(credentials, getClientConfiguration()); @@ -43,6 +37,20 @@ public AmazonS3Client getClient() { return amazonS3Client; } + AWSCredentials getAwsCredentials() { + AWSCredentials credentials; + + if(s3Configuration.isDefaultChainEnabled()) { + credentials = new DefaultAWSCredentialsProviderChain().getCredentials(); + } else if (s3Configuration.getSessionToken() == null) { + credentials = new BasicAWSCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey()); + } else { + credentials = new BasicSessionCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey(), + s3Configuration.getSessionToken()); + } + return credentials; + } + private ClientConfiguration getClientConfiguration() { return new ClientConfiguration().withMaxConnections(s3Configuration.getMaxConnections()); } diff --git a/micro-s3/src/test/java/com/aol/micro/server/s3/plugin/S3ClientProviderTest.java b/micro-s3/src/test/java/com/aol/micro/server/s3/plugin/S3ClientProviderTest.java index 88a765452..1548d8e5b 100644 --- a/micro-s3/src/test/java/com/aol/micro/server/s3/plugin/S3ClientProviderTest.java +++ b/micro-s3/src/test/java/com/aol/micro/server/s3/plugin/S3ClientProviderTest.java @@ -1,19 +1,29 @@ package com.aol.micro.server.s3.plugin; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicSessionCredentials; import com.amazonaws.services.s3.AmazonS3Client; +import com.aol.micro.server.s3.S3Configuration; import lombok.SneakyThrows; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; import org.springframework.test.context.support.AnnotationConfigContextLoader; +import org.springframework.test.util.ReflectionTestUtils; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(classes=S3ClientProvider.class, loader = AnnotationConfigContextLoader.class) +@TestPropertySource(properties = { + "s3.accessKey=", + "s3.secretKey=" +}) public class S3ClientProviderTest { @Autowired @@ -25,4 +35,60 @@ public void getClient() { AmazonS3Client client = ctx.getBean(AmazonS3Client.class); assertNotNull(client); } + + @Test + public void defaultChain() { + S3ClientProvider provider = new S3ClientProvider(); + S3Configuration s3Configuration = new S3Configuration(null, + null, + true, + null, + null, + 5, + "s3-transfer-manager-worker-", + 100); + ReflectionTestUtils.setField(provider, "s3Configuration", s3Configuration); + // system properties used here as they're the easiest to test + // but it can also pull them from the metadata service on ec2 + System.setProperty("aws.accessKeyId", "fakeKeyId"); + System.setProperty("aws.secretKey", "fakeSecretKey"); + AWSCredentials credentials = provider.getAwsCredentials(); + assertEquals("fakeKeyId", credentials.getAWSAccessKeyId()); + assertEquals("fakeSecretKey", credentials.getAWSSecretKey()); + } + + @Test + public void accessKey() { + S3ClientProvider provider = new S3ClientProvider(); + S3Configuration s3Configuration = new S3Configuration("fakeProvidedKeyId", + "fakeProvidedSecretKey", + false, + null, + null, + 5, + "s3-transfer-manager-worker-", + 100); + ReflectionTestUtils.setField(provider, "s3Configuration", s3Configuration); + AWSCredentials credentials = provider.getAwsCredentials(); + assertEquals("fakeProvidedKeyId", credentials.getAWSAccessKeyId()); + assertEquals("fakeProvidedSecretKey", credentials.getAWSSecretKey()); + } + + @Test + public void sessionToken() { + S3ClientProvider provider = new S3ClientProvider(); + S3Configuration s3Configuration = new S3Configuration("fakeProvidedKeyId", + "fakeProvidedSecretKey", + false, + "fakeProvidedSessionToken", + null, + 5, + "s3-transfer-manager-worker-", + 100); + ReflectionTestUtils.setField(provider, "s3Configuration", s3Configuration); + BasicSessionCredentials credentials = (BasicSessionCredentials) provider.getAwsCredentials(); + assertEquals("fakeProvidedKeyId", credentials.getAWSAccessKeyId()); + assertEquals("fakeProvidedSecretKey", credentials.getAWSSecretKey()); + assertEquals("fakeProvidedSessionToken", credentials.getSessionToken()); + } } \ No newline at end of file