Skip to content

Commit

Permalink
Update to latest arbiter.
Browse files Browse the repository at this point in the history
  • Loading branch information
@connormanning
connormanning committed Jun 5, 2024
1 parent ee3d882 commit 0c3c0d1
Show file tree
Hide file tree
Showing 2 changed files with 96 additions and 87 deletions.
133 changes: 90 additions & 43 deletions entwine/third/arbiter/arbiter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,16 +353,13 @@ std::shared_ptr<Driver> Arbiter::getDriver(const std::string path) const
{
const auto type(getProtocol(path));

{
std::lock_guard<std::mutex> lock(m_mutex);
auto it = m_drivers.find(type);
if (it != m_drivers.end()) return it->second;
}
std::lock_guard<std::mutex> lock(m_mutex);
auto it = m_drivers.find(type);
if (it != m_drivers.end()) return it->second;

const json config = getConfig(m_config);
if (auto driver = Driver::create(*m_pool, type, config.dump()))
{
std::lock_guard<std::mutex> lock(m_mutex);
m_drivers[type] = driver;
return driver;
}
Expand Down Expand Up @@ -1695,6 +1692,8 @@ namespace
const std::string ec2CredBase(
ec2CredIp + "/latest/meta-data/iam/security-credentials");

const std::string defaultDnsSuffix = "amazonaws.com";

// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html
const std::string fargateCredIp("169.254.170.2");

Expand Down Expand Up @@ -1751,6 +1750,11 @@ namespace
else if (auto e = env("ARBITER_VERBOSE")) verbose = *e;
return (!verbose.empty()) && !!std::stol(verbose);
}

bool doSignRequests()
{
return !env("AWS_NO_SIGN_REQUEST");
}
}

namespace drivers
Expand Down Expand Up @@ -1780,9 +1784,7 @@ std::unique_ptr<S3> S3::create(
if (auto p = env("AWS_PROFILE")) profile = *p;
}

auto auth(Auth::create(s, profile));
if (!auth) return std::unique_ptr<S3>();

auto auth(doSignRequests() ? Auth::create(s, profile) : nullptr);
auto config = makeUnique<Config>(s, profile);
return makeUnique<S3>(pool, profile, std::move(auth), std::move(config));
}
Expand Down Expand Up @@ -1953,11 +1955,6 @@ S3::Config::Config(const std::string s, const std::string profile)
m_baseHeaders[p.key()] = p.value().get<std::string>();
}
}
else
{
std::cout << "s3.headers expected to be object - skipping" <<
std::endl;
}
}
}

Expand Down Expand Up @@ -2007,6 +2004,12 @@ std::string S3::Config::extractBaseUrl(
const std::string s,
const std::string region)
{
if (auto p = env("AWS_ENDPOINT_URL"))
{
const std::string path = *p;
return path.back() == '/' ? path : path + '/';
}

const json c(s.size() ? json::parse(s) : json());

if (!c.is_null() &&
Expand All @@ -2024,41 +2027,49 @@ std::string S3::Config::extractBaseUrl(
endpointsPath = *e;
}

std::string dnsSuffix("amazonaws.com");

drivers::Fs fsDriver;
if (std::unique_ptr<std::string> e = fsDriver.tryGet(endpointsPath))
{
const json ep(json::parse(*e));

for (const auto& partition : ep["partitions"])
{
if (partition.count("dnsSuffix"))
if (
!partition.count("regions") ||
!partition.at("regions").count(region))
{
dnsSuffix = partition["dnsSuffix"].get<std::string>();
continue;
}

const auto& endpoints(
partition.at("services").at("s3").at("endpoints"));

for (const auto& r : endpoints.items())
// Look for an explicit hostname for this region/service.
if (
partition.count("services") &&
partition["services"].count("s3") &&
partition["services"]["s3"].count("endpoints"))
{
if (r.key() == region &&
endpoints.value("region", json::object())
.count("hostname"))
const auto& endpoints(partition["services"]["s3"]["endpoints"]);

for (const auto& r : endpoints.items())
{
return endpoints["region"]["hostname"].get<std::string>() +
'/';
if (r.key() == region &&
endpoints.value("region", json::object())
.count("hostname"))
{
return endpoints["region"]["hostname"].get<std::string>() +
'/';
}
}
}

// No explicit hostname found, so build it from our region/DNS suffix.
std::string dnsSuffix = partition.value("dnsSuffix", defaultDnsSuffix);
return "s3." + region + "." + dnsSuffix + "/";
}
}

if (dnsSuffix.size() && dnsSuffix.back() != '/') dnsSuffix += '/';

// https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
if (region == "us-east-1") return "s3." + dnsSuffix;
else return "s3-" + region + "." + dnsSuffix;
if (region == "us-east-1") return "s3." + defaultDnsSuffix + "/";
else return "s3-" + region + "." + defaultDnsSuffix + "/";
}

S3::AuthFields S3::Auth::fields() const
Expand Down Expand Up @@ -2142,7 +2153,7 @@ std::unique_ptr<std::size_t> S3::tryGetSize(
"HEAD",
m_config->region(),
resource,
m_auth->fields(),
authFields(),
query,
headers,
empty);
Expand Down Expand Up @@ -2178,7 +2189,7 @@ bool S3::get(
"GET",
m_config->region(),
resource,
m_auth->fields(),
authFields(),
query,
headers,
empty);
Expand All @@ -2196,11 +2207,10 @@ bool S3::get(
data = res.data();
return true;
}
else
{
std::cout << res.code() << ": " << res.str() << std::endl;
return false;
}

if (isVerbose()) std::cout << res.code() << ": " << res.str() << std::endl;

return false;
}

std::vector<char> S3::put(
Expand All @@ -2223,7 +2233,7 @@ std::vector<char> S3::put(
"PUT",
m_config->region(),
resource,
m_auth->fields(),
authFields(),
query,
headers,
data);
Expand Down Expand Up @@ -2373,6 +2383,11 @@ std::vector<std::string> S3::glob(std::string path, bool verbose) const
return results;
}

S3::AuthFields S3::authFields() const
{
return m_auth ? m_auth->fields() : S3::AuthFields();
}

S3::ApiV4::ApiV4(
const std::string verb,
const std::string& region,
Expand Down Expand Up @@ -2407,6 +2422,8 @@ S3::ApiV4::ApiV4(
m_headers.erase("Expect");
}

if (!m_authFields) return;

const Headers normalizedHeaders(
std::accumulate(
m_headers.begin(),
Expand Down Expand Up @@ -2900,7 +2917,7 @@ std::unique_ptr<std::size_t> AZ::tryGetSize(
if (m_config->hasSasToken())
{
Query q = m_config->sasToken();
q.insert(std::cbegin(query),std::cend(query));
q.insert(std::begin(query), std::end(query));
res.reset(new Response(http.internalHead(resource.url(), headers, q)));
}
else
Expand Down Expand Up @@ -4445,7 +4462,11 @@ int Curl::perform()

if (code != CURLE_OK)
{
std::cerr << "Curl failure: " << curl_easy_strerror(code) << std::endl;
if (m_verbose)
{
std::cout << "Curl failure: " << curl_easy_strerror(code) <<
std::endl;
}
httpCode = 550;
}

Expand Down Expand Up @@ -4965,9 +4986,25 @@ Contents parse(const std::string& s)

#ifndef ARBITER_IS_AMALGAMATION
#include <arbiter/util/md5.hpp>
#include <arbiter/util/macros.hpp>
#endif

// Various crypto utilities.
#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b))))
#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b))))
#define F(x,y,z) ((x & y) | (~x & z))
#define G(x,y,z) ((x & z) | (y & ~z))
#define H(x,y,z) (x ^ y ^ z)
#define I(x,y,z) (y ^ (x | ~z))

#define FF(a,b,c,d,m,s,t) { a += F(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }
#define GG(a,b,c,d,m,s,t) { a += G(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }
#define HH(a,b,c,d,m,s,t) { a += H(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }
#define II(a,b,c,d,m,s,t) { a += I(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }

#ifdef ARBITER_CUSTOM_NAMESPACE
namespace ARBITER_CUSTOM_NAMESPACE
{
Expand Down Expand Up @@ -5188,9 +5225,19 @@ std::string md5(const std::string& data)

#ifndef ARBITER_IS_AMALGAMATION
#include <arbiter/util/sha256.hpp>
#include <arbiter/util/macros.hpp>
#endif


// Various crypto utilities.
#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b))))
#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b))))
#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z)))
#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22))
#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25))
#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3))
#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10))

#ifdef ARBITER_CUSTOM_NAMESPACE
namespace ARBITER_CUSTOM_NAMESPACE
{
Expand Down
50 changes: 6 additions & 44 deletions entwine/third/arbiter/arbiter.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/// Arbiter amalgamated header (https://github.com/connormanning/arbiter).
/// It is intended to be used with #include "arbiter.hpp"

// Git SHA: 3e8919a297f4dc357e1ef2473bb295d8d1eac226
// Git SHA: 5c3f36e86e7a74aadb8a98edb14ad207158aa785

// //////////////////////////////////////////////////////////////////////
// Beginning of content of file: LICENSE
Expand Down Expand Up @@ -3276,48 +3276,6 @@ class ARBITER_DLL Time



// //////////////////////////////////////////////////////////////////////
// Beginning of content of file: arbiter/util/macros.hpp
// //////////////////////////////////////////////////////////////////////

#pragma once

#define ROTLEFT(a,b) (((a) << (b)) | ((a) >> (32-(b))))
#define ROTRIGHT(a,b) (((a) >> (b)) | ((a) << (32-(b))))

// SHA256.
#define CH(x,y,z) (((x) & (y)) ^ (~(x) & (z)))
#define MAJ(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z)))
#define EP0(x) (ROTRIGHT(x,2) ^ ROTRIGHT(x,13) ^ ROTRIGHT(x,22))
#define EP1(x) (ROTRIGHT(x,6) ^ ROTRIGHT(x,11) ^ ROTRIGHT(x,25))
#define SIG0(x) (ROTRIGHT(x,7) ^ ROTRIGHT(x,18) ^ ((x) >> 3))
#define SIG1(x) (ROTRIGHT(x,17) ^ ROTRIGHT(x,19) ^ ((x) >> 10))

// MD5.
#define F(x,y,z) ((x & y) | (~x & z))
#define G(x,y,z) ((x & z) | (y & ~z))
#define H(x,y,z) (x ^ y ^ z)
#define I(x,y,z) (y ^ (x | ~z))

#define FF(a,b,c,d,m,s,t) { a += F(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }
#define GG(a,b,c,d,m,s,t) { a += G(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }
#define HH(a,b,c,d,m,s,t) { a += H(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }
#define II(a,b,c,d,m,s,t) { a += I(b,c,d) + m + t; \
a = b + ROTLEFT(a,s); }


// //////////////////////////////////////////////////////////////////////
// End of content of file: arbiter/util/macros.hpp
// //////////////////////////////////////////////////////////////////////






// //////////////////////////////////////////////////////////////////////
// Beginning of content of file: arbiter/util/md5.hpp
// //////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -4320,6 +4278,8 @@ class S3 : public Http
std::string path,
bool verbose) const override;

AuthFields authFields() const;

class ApiV4;
class Resource;

Expand All @@ -4330,14 +4290,16 @@ class S3 : public Http
class S3::AuthFields
{
public:
AuthFields(std::string access, std::string hidden, std::string token = "")
AuthFields(std::string access = "", std::string hidden = "", std::string token = "")
: m_access(access), m_hidden(hidden), m_token(token)
{ }

const std::string& access() const { return m_access; }
const std::string& hidden() const { return m_hidden; }
const std::string& token() const { return m_token; }

explicit operator bool() const { return m_access.size() || m_hidden.size() || m_token.size(); }

private:
std::string m_access;
std::string m_hidden;
Expand Down

0 comments on commit 0c3c0d1

Please sign in to comment.