Skip to content

Commit f823450

Browse files
narqonarqo
authored
add Client.SetS3EnableDualstack (#1945)
Closes #1766 This PR adds a new configuration method, that enables users to switch off the use of dual-stack endpoints, when they use the client against Amazon S3. I wanted to keep the name of the config aligned with how AWS SDKs and CLI call it, even though the default value is reversed. That is, AWS SDKs call it like `UseDualStack bool` or `withDualstackEnabled(bool)`.
1 parent 4a223cc commit f823450

File tree

3 files changed

+172
-46
lines changed

3 files changed

+172
-46
lines changed

‎api.go

+18-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
3-
* Copyright 2015-2023 MinIO, Inc.
3+
* Copyright 2015-2024 MinIO, Inc.
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
66
* you may not use this file except in compliance with the License.
@@ -80,6 +80,8 @@ type Client struct {
8080

8181
// S3 specific accelerated endpoint.
8282
s3AccelerateEndpoint string
83+
// S3 dual-stack endpoints are enabled by default.
84+
s3DualstackEnabled bool
8385

8486
// Region endpoint
8587
region string
@@ -158,9 +160,12 @@ func New(endpoint string, opts *Options) (*Client, error) {
158160
if err != nil {
159161
return nil, err
160162
}
161-
// If Amazon S3 set to signature v4.
162163
if s3utils.IsAmazonEndpoint(*clnt.endpointURL) {
164+
// If Amazon S3 set to signature v4.
163165
clnt.overrideSignerType = credentials.SignatureV4
166+
// Amazon S3 endpoints are resolved into dual-stack endpoints by default
167+
// for backwards compatibility.
168+
clnt.s3DualstackEnabled = true
164169
}
165170

166171
return clnt, nil
@@ -330,6 +335,16 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) {
330335
}
331336
}
332337

338+
// SetS3EnableDualstack turns s3 dual-stack endpoints on or off for all requests.
339+
// The feature is only specific to S3 and is on by default. To read more about
340+
// Amazon S3 dual-stack endpoints visit -
341+
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/dual-stack-endpoints.html
342+
func (c *Client) SetS3EnableDualstack(enabled bool) {
343+
if s3utils.IsAmazonEndpoint(*c.endpointURL) {
344+
c.s3DualstackEnabled = enabled
345+
}
346+
}
347+
333348
// Hash materials provides relevant initialized hash algo writers
334349
// based on the expected signature type.
335350
//
@@ -926,7 +941,7 @@ func (c *Client) makeTargetURL(bucketName, objectName, bucketLocation string, is
926941
// Do not change the host if the endpoint URL is a FIPS S3 endpoint or a S3 PrivateLink interface endpoint
927942
if !s3utils.IsAmazonFIPSEndpoint(*c.endpointURL) && !s3utils.IsAmazonPrivateLinkEndpoint(*c.endpointURL) {
928943
// Fetch new host based on the bucket location.
929-
host = getS3Endpoint(bucketLocation)
944+
host = getS3Endpoint(bucketLocation, c.s3DualstackEnabled)
930945
}
931946
}
932947
}

‎api_unit_test.go

+9-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
3-
* Copyright 2015-2017 MinIO, Inc.
3+
* Copyright 2015-2024 MinIO, Inc.
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
66
* you may not use this file except in compliance with the License.
@@ -29,14 +29,18 @@ import (
2929
func TestValidBucketLocation(t *testing.T) {
3030
s3Hosts := []struct {
3131
bucketLocation string
32+
useDualstack bool
3233
endpoint string
3334
}{
34-
{"us-east-1", "s3.dualstack.us-east-1.amazonaws.com"},
35-
{"unknown", "s3.dualstack.us-east-1.amazonaws.com"},
36-
{"ap-southeast-1", "s3.dualstack.ap-southeast-1.amazonaws.com"},
35+
{"us-east-1", true, "s3.dualstack.us-east-1.amazonaws.com"},
36+
{"us-east-1", false, "s3.us-east-1.amazonaws.com"},
37+
{"unknown", true, "s3.dualstack.us-east-1.amazonaws.com"},
38+
{"unknown", false, "s3.us-east-1.amazonaws.com"},
39+
{"ap-southeast-1", true, "s3.dualstack.ap-southeast-1.amazonaws.com"},
40+
{"ap-southeast-1", false, "s3.ap-southeast-1.amazonaws.com"},
3741
}
3842
for _, s3Host := range s3Hosts {
39-
endpoint := getS3Endpoint(s3Host.bucketLocation)
43+
endpoint := getS3Endpoint(s3Host.bucketLocation, s3Host.useDualstack)
4044
if endpoint != s3Host.endpoint {
4145
t.Fatal("Error: invalid bucket location", endpoint)
4246
}

‎s3-endpoints.go

+145-38
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/*
22
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
3-
* Copyright 2015-2017 MinIO, Inc.
3+
* Copyright 2015-2024 MinIO, Inc.
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
66
* you may not use this file except in compliance with the License.
@@ -17,48 +17,155 @@
1717

1818
package minio
1919

20+
type awsS3Endpoint struct {
21+
endpoint string
22+
dualstackEndpoint string
23+
}
24+
2025
// awsS3EndpointMap Amazon S3 endpoint map.
21-
var awsS3EndpointMap = map[string]string{
22-
"us-east-1": "s3.dualstack.us-east-1.amazonaws.com",
23-
"us-east-2": "s3.dualstack.us-east-2.amazonaws.com",
24-
"us-west-2": "s3.dualstack.us-west-2.amazonaws.com",
25-
"us-west-1": "s3.dualstack.us-west-1.amazonaws.com",
26-
"ca-central-1": "s3.dualstack.ca-central-1.amazonaws.com",
27-
"eu-west-1": "s3.dualstack.eu-west-1.amazonaws.com",
28-
"eu-west-2": "s3.dualstack.eu-west-2.amazonaws.com",
29-
"eu-west-3": "s3.dualstack.eu-west-3.amazonaws.com",
30-
"eu-central-1": "s3.dualstack.eu-central-1.amazonaws.com",
31-
"eu-central-2": "s3.dualstack.eu-central-2.amazonaws.com",
32-
"eu-north-1": "s3.dualstack.eu-north-1.amazonaws.com",
33-
"eu-south-1": "s3.dualstack.eu-south-1.amazonaws.com",
34-
"eu-south-2": "s3.dualstack.eu-south-2.amazonaws.com",
35-
"ap-east-1": "s3.dualstack.ap-east-1.amazonaws.com",
36-
"ap-south-1": "s3.dualstack.ap-south-1.amazonaws.com",
37-
"ap-south-2": "s3.dualstack.ap-south-2.amazonaws.com",
38-
"ap-southeast-1": "s3.dualstack.ap-southeast-1.amazonaws.com",
39-
"ap-southeast-2": "s3.dualstack.ap-southeast-2.amazonaws.com",
40-
"ap-northeast-1": "s3.dualstack.ap-northeast-1.amazonaws.com",
41-
"ap-northeast-2": "s3.dualstack.ap-northeast-2.amazonaws.com",
42-
"ap-northeast-3": "s3.dualstack.ap-northeast-3.amazonaws.com",
43-
"af-south-1": "s3.dualstack.af-south-1.amazonaws.com",
44-
"me-central-1": "s3.dualstack.me-central-1.amazonaws.com",
45-
"me-south-1": "s3.dualstack.me-south-1.amazonaws.com",
46-
"sa-east-1": "s3.dualstack.sa-east-1.amazonaws.com",
47-
"us-gov-west-1": "s3.dualstack.us-gov-west-1.amazonaws.com",
48-
"us-gov-east-1": "s3.dualstack.us-gov-east-1.amazonaws.com",
49-
"cn-north-1": "s3.dualstack.cn-north-1.amazonaws.com.cn",
50-
"cn-northwest-1": "s3.dualstack.cn-northwest-1.amazonaws.com.cn",
51-
"ap-southeast-3": "s3.dualstack.ap-southeast-3.amazonaws.com",
52-
"ap-southeast-4": "s3.dualstack.ap-southeast-4.amazonaws.com",
53-
"il-central-1": "s3.dualstack.il-central-1.amazonaws.com",
26+
var awsS3EndpointMap = map[string]awsS3Endpoint{
27+
"us-east-1": {
28+
"s3.us-east-1.amazonaws.com",
29+
"s3.dualstack.us-east-1.amazonaws.com",
30+
},
31+
"us-east-2": {
32+
"s3.us-east-2.amazonaws.com",
33+
"s3.dualstack.us-east-2.amazonaws.com",
34+
},
35+
"us-west-2": {
36+
"s3.us-west-2.amazonaws.com",
37+
"s3.dualstack.us-west-2.amazonaws.com",
38+
},
39+
"us-west-1": {
40+
"s3.us-west-1.amazonaws.com",
41+
"s3.dualstack.us-west-1.amazonaws.com",
42+
},
43+
"ca-central-1": {
44+
"s3.ca-central-1.amazonaws.com",
45+
"s3.dualstack.ca-central-1.amazonaws.com",
46+
},
47+
"eu-west-1": {
48+
"s3.eu-west-1.amazonaws.com",
49+
"s3.dualstack.eu-west-1.amazonaws.com",
50+
},
51+
"eu-west-2": {
52+
"s3.eu-west-2.amazonaws.com",
53+
"s3.dualstack.eu-west-2.amazonaws.com",
54+
},
55+
"eu-west-3": {
56+
"s3.eu-west-3.amazonaws.com",
57+
"s3.dualstack.eu-west-3.amazonaws.com",
58+
},
59+
"eu-central-1": {
60+
"s3.eu-central-1.amazonaws.com",
61+
"s3.dualstack.eu-central-1.amazonaws.com",
62+
},
63+
"eu-central-2": {
64+
"s3.eu-central-2.amazonaws.com",
65+
"s3.dualstack.eu-central-2.amazonaws.com",
66+
},
67+
"eu-north-1": {
68+
"s3.eu-north-1.amazonaws.com",
69+
"s3.dualstack.eu-north-1.amazonaws.com",
70+
},
71+
"eu-south-1": {
72+
"s3.eu-south-1.amazonaws.com",
73+
"s3.dualstack.eu-south-1.amazonaws.com",
74+
},
75+
"eu-south-2": {
76+
"s3.eu-south-2.amazonaws.com",
77+
"s3.dualstack.eu-south-2.amazonaws.com",
78+
},
79+
"ap-east-1": {
80+
"s3.ap-east-1.amazonaws.com",
81+
"s3.dualstack.ap-east-1.amazonaws.com",
82+
},
83+
"ap-south-1": {
84+
"s3.ap-south-1.amazonaws.com",
85+
"s3.dualstack.ap-south-1.amazonaws.com",
86+
},
87+
"ap-south-2": {
88+
"s3.ap-south-2.amazonaws.com",
89+
"s3.dualstack.ap-south-2.amazonaws.com",
90+
},
91+
"ap-southeast-1": {
92+
"s3.ap-southeast-1.amazonaws.com",
93+
"s3.dualstack.ap-southeast-1.amazonaws.com",
94+
},
95+
"ap-southeast-2": {
96+
"s3.ap-southeast-2.amazonaws.com",
97+
"s3.dualstack.ap-southeast-2.amazonaws.com",
98+
},
99+
"ap-southeast-3": {
100+
"s3.ap-southeast-3.amazonaws.com",
101+
"s3.dualstack.ap-southeast-3.amazonaws.com",
102+
},
103+
"ap-southeast-4": {
104+
"s3.ap-southeast-4.amazonaws.com",
105+
"s3.dualstack.ap-southeast-4.amazonaws.com",
106+
},
107+
"ap-northeast-1": {
108+
"s3.ap-northeast-1.amazonaws.com",
109+
"s3.dualstack.ap-northeast-1.amazonaws.com",
110+
},
111+
"ap-northeast-2": {
112+
"s3.ap-northeast-2.amazonaws.com",
113+
"s3.dualstack.ap-northeast-2.amazonaws.com",
114+
},
115+
"ap-northeast-3": {
116+
"s3.ap-northeast-3.amazonaws.com",
117+
"s3.dualstack.ap-northeast-3.amazonaws.com",
118+
},
119+
"af-south-1": {
120+
"s3.af-south-1.amazonaws.com",
121+
"s3.dualstack.af-south-1.amazonaws.com",
122+
},
123+
"me-central-1": {
124+
"s3.me-central-1.amazonaws.com",
125+
"s3.dualstack.me-central-1.amazonaws.com",
126+
},
127+
"me-south-1": {
128+
"s3.me-south-1.amazonaws.com",
129+
"s3.dualstack.me-south-1.amazonaws.com",
130+
},
131+
"sa-east-1": {
132+
"s3.sa-east-1.amazonaws.com",
133+
"s3.dualstack.sa-east-1.amazonaws.com",
134+
},
135+
"us-gov-west-1": {
136+
"s3.us-gov-west-1.amazonaws.com",
137+
"s3.dualstack.us-gov-west-1.amazonaws.com",
138+
},
139+
"us-gov-east-1": {
140+
"s3.us-gov-east-1.amazonaws.com",
141+
"s3.dualstack.us-gov-east-1.amazonaws.com",
142+
},
143+
"cn-north-1": {
144+
"s3.cn-north-1.amazonaws.com.cn",
145+
"s3.dualstack.cn-north-1.amazonaws.com.cn",
146+
},
147+
"cn-northwest-1": {
148+
"s3.cn-northwest-1.amazonaws.com.cn",
149+
"s3.dualstack.cn-northwest-1.amazonaws.com.cn",
150+
},
151+
"il-central-1": {
152+
"s3.il-central-1.amazonaws.com",
153+
"s3.dualstack.il-central-1.amazonaws.com",
154+
},
54155
}
55156

56157
// getS3Endpoint get Amazon S3 endpoint based on the bucket location.
57-
func getS3Endpoint(bucketLocation string) (s3Endpoint string) {
158+
func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) {
58159
s3Endpoint, ok := awsS3EndpointMap[bucketLocation]
59160
if !ok {
60-
// Default to 's3.dualstack.us-east-1.amazonaws.com' endpoint.
61-
s3Endpoint = "s3.dualstack.us-east-1.amazonaws.com"
161+
// Default to 's3.us-east-1.amazonaws.com' endpoint.
162+
if useDualstack {
163+
return "s3.dualstack.us-east-1.amazonaws.com"
164+
}
165+
return "s3.us-east-1.amazonaws.com"
166+
}
167+
if useDualstack {
168+
return s3Endpoint.dualstackEndpoint
62169
}
63-
return s3Endpoint
170+
return s3Endpoint.endpoint
64171
}

0 commit comments

Comments
 (0)