Merge pull request #366 from aol/enable_s3_default_chain

Enable support for AWS role based auth. Fixes #306.

gradle.properties

@@ -1,4 +1,4 @@
-version=0.91.8
+version=0.91.9
 springVersion=4.3.3.RELEASE
 springBootVersion=1.4.1.RELEASE
 jerseyVersion=2.24

micro-s3/build.gradle

@@ -5,7 +5,7 @@ dependencies {
 	compile 'com.amazonaws:aws-java-sdk:' + s3Version
 	compile project(':micro-core')
 	compile project(':micro-manifest-comparator')
-    testCompile group: 'org.springframework', name: 'spring-test', version: '4.0.5.RELEASE'
+    testCompile group: 'org.springframework', name: 'spring-test', version: springVersion
     testCompile group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.0'
     testCompile group: 'org.coursera', name: 'dropwizard-metrics-datadog', version: '1.1.6'
     testCompile group: 'com.github.cbismuth', name: 'junit-repeat-rule', version: '1.1.0'

micro-s3/readme.md

@@ -181,8 +181,14 @@ Then inject in either  DistributedMap or it's concrete implementation S3Distribu
 
 ## Usage AmazonS3Client
 
-This plugin also provides an AmazonS3Client implementation bean. You should define properties for
-s3.accessKey, s3.secretKey and optionall s3.sessionToken (optionally - only for short term keys)
+This plugin also provides an AmazonS3Client implementation bean. 
+
+You may configure AWS credentials in one of two ways. 
+The first is to use the AWS default chain for configuration, by setting `s3.useDefaultChain` to `true`.
+This allows you to rely on instance roles if your client is hosted on AWS, or to specify your credentials as environment variables if not.
+For full details on the default AWS chain, see the [official AWS docs](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html)
+
+The second option is to define properties for `s3.accessKey`, `s3.secretKey` and optionally `s3.sessionToken` (optionally - only for short term keys)
 [AmazonS3Client](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Client.html)
 
 Configure the S3 Bucket for Manifest comparision via this property

micro-s3/src/main/java/com/aol/micro/server/s3/S3Configuration.java

@@ -5,6 +5,8 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
+import java.util.Objects;
+
 @Component
 @Getter
 public class S3Configuration {
@@ -13,21 +15,30 @@ public class S3Configuration {
     private final String secretKey;
     private final String sessionToken;
     private final String region;
+    private final boolean defaultChainEnabled;
     private final int uploadThreads;
     private final String uploadThreadNamePrefix;
     private final int maxConnections;
 
     @Autowired
-    public S3Configuration(@Value("${s3.accessKey}") String accessKey,
-                           @Value("${s3.secretKey}") String secretKey,
+    public S3Configuration(@Value("${s3.accessKey:#{null}}") String accessKey,
+                           @Value("${s3.secretKey:#{null}}") String secretKey,
+                           @Value("${s3.useDefaultChain:false}") boolean defaultChainEnabled,
                            @Value("${s3.sessionToken:#{null}}") String sessionToken,
                            @Value("${s3.region:#{null}}") String region,
                            @Value("${s3.upload.threads:5}") int uploadThreads,
                            @Value("${s3.upload.thread.name.prefix:s3-transfer-manager-worker-}") String uploadThreadNamePrefix,
                            @Value("${s3.client.maxConnections:100}") int maxConnections) {
+
+        if((Objects.isNull(accessKey) || Objects.isNull(secretKey)) && !defaultChainEnabled) {
+            throw new RuntimeException("No S3 authorization method provided. "
+                                       + "Please either enable the aws default credentials chain with s3.useDefaultChain, "
+                                       + "or provide access keys via s3.accessKey and s3.secretKey (and optionally s3.sessionToken)");
+        }
         this.accessKey = accessKey;
         this.secretKey = secretKey;
         this.sessionToken = sessionToken;
+        this.defaultChainEnabled = defaultChainEnabled;
         this.region = region;
         this.uploadThreads = uploadThreads;
         this.uploadThreadNamePrefix = uploadThreadNamePrefix;

micro-s3/src/main/java/com/aol/micro/server/s3/plugin/S3ClientProvider.java

@@ -1,6 +1,7 @@
 package com.aol.micro.server.s3.plugin;
 
 import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
@@ -24,14 +25,7 @@ public class S3ClientProvider {
     @Bean
     public AmazonS3Client getClient() {
 
-        AWSCredentials credentials;
-
-        if (s3Configuration.getSessionToken() == null) {
-            credentials = new BasicAWSCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey());
-        } else {
-            credentials = new BasicSessionCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey(),
-                                                      s3Configuration.getSessionToken());
-        }
+        AWSCredentials credentials = getAwsCredentials();
 
         AmazonS3Client amazonS3Client = new AmazonS3Client(credentials, getClientConfiguration());
 
@@ -43,6 +37,20 @@ public AmazonS3Client getClient() {
         return amazonS3Client;
     }
 
+    AWSCredentials getAwsCredentials() {
+        AWSCredentials credentials;
+
+        if(s3Configuration.isDefaultChainEnabled()) {
+            credentials = new DefaultAWSCredentialsProviderChain().getCredentials();
+        } else if (s3Configuration.getSessionToken() == null) {
+            credentials = new BasicAWSCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey());
+        } else {
+            credentials = new BasicSessionCredentials(s3Configuration.getAccessKey(), s3Configuration.getSecretKey(),
+                                                      s3Configuration.getSessionToken());
+        }
+        return credentials;
+    }
+
     private ClientConfiguration getClientConfiguration() {
         return new ClientConfiguration().withMaxConnections(s3Configuration.getMaxConnections());
     }

micro-s3/src/test/java/com/aol/micro/server/s3/plugin/S3ClientProviderTest.java

@@ -1,19 +1,29 @@
 package com.aol.micro.server.s3.plugin;
 
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.aol.micro.server.s3.S3Configuration;
 import lombok.SneakyThrows;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.ApplicationContext;
 import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 import org.springframework.test.context.support.AnnotationConfigContextLoader;
+import org.springframework.test.util.ReflectionTestUtils;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(classes=S3ClientProvider.class, loader = AnnotationConfigContextLoader.class)
+@TestPropertySource(properties = {
+        "s3.accessKey=",
+        "s3.secretKey="
+})
 public class S3ClientProviderTest {
 
     @Autowired
@@ -25,4 +35,60 @@ public void getClient() {
         AmazonS3Client client = ctx.getBean(AmazonS3Client.class);
         assertNotNull(client);
     }
+
+    @Test
+    public void defaultChain() {
+        S3ClientProvider provider = new S3ClientProvider();
+        S3Configuration s3Configuration = new S3Configuration(null,
+                                                              null,
+                                                              true,
+                                                              null,
+                                                              null,
+                                                              5,
+                                                              "s3-transfer-manager-worker-",
+                                                              100);
+        ReflectionTestUtils.setField(provider, "s3Configuration", s3Configuration);
+        // system properties used here as they're the easiest to test
+        // but it can also pull them from the metadata service on ec2
+        System.setProperty("aws.accessKeyId", "fakeKeyId");
+        System.setProperty("aws.secretKey", "fakeSecretKey");
+        AWSCredentials credentials = provider.getAwsCredentials();
+        assertEquals("fakeKeyId", credentials.getAWSAccessKeyId());
+        assertEquals("fakeSecretKey", credentials.getAWSSecretKey());
+    }
+
+    @Test
+    public void accessKey() {
+        S3ClientProvider provider = new S3ClientProvider();
+        S3Configuration s3Configuration = new S3Configuration("fakeProvidedKeyId",
+                                                              "fakeProvidedSecretKey",
+                                                              false,
+                                                              null,
+                                                              null,
+                                                              5,
+                                                              "s3-transfer-manager-worker-",
+                                                              100);
+        ReflectionTestUtils.setField(provider, "s3Configuration", s3Configuration);
+        AWSCredentials credentials = provider.getAwsCredentials();
+        assertEquals("fakeProvidedKeyId", credentials.getAWSAccessKeyId());
+        assertEquals("fakeProvidedSecretKey", credentials.getAWSSecretKey());
+    }
+
+    @Test
+    public void sessionToken() {
+        S3ClientProvider provider = new S3ClientProvider();
+        S3Configuration s3Configuration = new S3Configuration("fakeProvidedKeyId",
+                                                              "fakeProvidedSecretKey",
+                                                              false,
+                                                              "fakeProvidedSessionToken",
+                                                              null,
+                                                              5,
+                                                              "s3-transfer-manager-worker-",
+                                                              100);
+        ReflectionTestUtils.setField(provider, "s3Configuration", s3Configuration);
+        BasicSessionCredentials credentials = (BasicSessionCredentials) provider.getAwsCredentials();
+        assertEquals("fakeProvidedKeyId", credentials.getAWSAccessKeyId());
+        assertEquals("fakeProvidedSecretKey", credentials.getAWSSecretKey());
+        assertEquals("fakeProvidedSessionToken", credentials.getSessionToken());
+    }
 }