Properly handle missing regions in URLS

NOTE: it is important that this fix gets into 4.9.3 re: Issue #2798 ## Modifications * This PR includes PR #2813 * Support the following AWS environment variables in the internal S3 library (they are already supported by aws-sdk-cpp). - AWS_REGION - AWS_DEFAULT_REGION - AWS_ACCESS_KEY_ID - AWS_CONFIG_FILE - AWS_PROFILE - AWS_SECRET_ACCESS_KEY - (source https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html). * Support an empty region when specifying s3.amazonaws.com as the host. * Move some S3/AWS related functions to ds3util.c * Add a test case to test empty region and AWS_[DEFAULT]_REGION.
Unidata · Dec 3, 2023 · 27f615b · 27f615b
1 parent 0c6fd78
commit 27f615b
Show file tree

Hide file tree

Showing 31 changed files with 809 additions and 631 deletions.
diff --git a/include/nc4internal.h b/include/nc4internal.h
@@ -332,6 +332,13 @@ typedef struct NCglobalstate {
     struct GlobalZarr { /* Zarr specific parameters */
 	char dimension_separator;
     } zarr;
+    struct GlobalAWS { /* AWS S3 specific parameters/defaults */
+	char* default_region;
+	char* config_file;
+	char* profile;
+	char* access_key_id;
+	char* secret_access_key;
+    } aws;
     struct Alignment { /* H5Pset_alignment parameters */
         int defined; /* 1 => threshold and alignment explicitly set */
 	int threshold;

diff --git a/include/ncpathmgr.h b/include/ncpathmgr.h
@@ -14,6 +14,9 @@
 #ifdef HAVE_DIRENT_H
 #include <dirent.h>
 #endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif

diff --git a/include/ncrc.h b/include/ncrc.h
@@ -33,16 +33,6 @@ typedef struct NCRCentry {
         char* value;
 } NCRCentry;
 
-struct AWSentry {
-    char* key;
-    char* value;
-};
-
-struct AWSprofile {
-    char* name;
-    NClist* entries; /* NClist<struct AWSentry*> */
-};
-
 /* collect all the relevant info around the rc file and AWS */
 typedef struct NCRCinfo {
 	int ignore; /* if 1, then do not use any rc file */
@@ -88,16 +78,10 @@ EXTERNL char* NC_mktmp(const char* base);
 EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp);
 EXTERNL int NC_testmode(NCURI* uri, const char* tag);
 EXTERNL int NC_testpathmode(const char* path, const char* tag);
+EXTERNL int NC_addmodetag(NCURI* uri, const char* tag);
 EXTERNL int NC_split_delim(const char* path, char delim, NClist* segments);
 EXTERNL int NC_join(struct NClist* segments, char** pathp);
-
-/* From ds3util.c */
-/* S3 profiles */
-EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
-EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
-EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
-EXTERNL int NC_iss3(NCURI* uri);
-EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);
+EXTERNL int NC_joinwith(NClist* segments, const char* sep, const char* prefix, const char* suffix, char** pathp);
 
 #if defined(__cplusplus)
 }

diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h
@@ -6,6 +6,12 @@
 #ifndef NCS3SDK_H
 #define NCS3SDK_H 1
 
+#define AWSHOST ".amazonaws.com"
+#define GOOGLEHOST "storage.googleapis.com"
+
+/* Define the "global" default region to be used if no other region is specified */
+#define AWS_GLOBAL_DEFAULT_REGION "us-east-1"
+
 /* Track the server type, if known */
 typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
                  NCS3=1,    /* s3.amazon.aws */
@@ -21,6 +27,20 @@ typedef struct NCS3INFO {
     NCS3SVC svc;
 } NCS3INFO;
 
+struct AWSentry {
+    char* key;
+    char* value;
+};
+
+struct AWSprofile {
+    char* name;
+    struct NClist* entries; /* NClist<struct AWSentry*> */
+};
+
+/* Opaque Types */
+struct NClist;
+struct NCglobalstate;
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -38,13 +58,23 @@ EXTERNL int NC_s3sdkclose(void* s3client0, NCS3INFO* info, int deleteit, char**
 EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefix, size_t* nkeysp, char*** keysp, char** errmsgp);
 EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp);
 EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
-EXTERNL const char* NC_s3dumps3info(NCS3INFO* info);
 
 /* From ds3util.c */
+EXTERNL int NC_s3sdkinitialize(void);
+EXTERNL int NC_s3sdkfinalize(void);
+
 EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp);
 EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp);
 EXTERNL int NC_s3clear(NCS3INFO* s3);
 EXTERNL int NC_s3clone(NCS3INFO* s3, NCS3INFO** news3p);
+EXTERNL const char* NC_s3dumps3info(NCS3INFO* info);
+EXTERNL void NC_s3freeprofilelist(struct NClist* profiles);
+EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
+EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
+EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
+EXTERNL int NC_iss3(NCURI* uri, enum NCS3SVC*);
+EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);
+EXTERNL int NC_aws_load_credentials(struct NCglobalstate* gstate);
 
 #ifdef __cplusplus
 }

diff --git a/libdispatch/dauth.c b/libdispatch/dauth.c
@@ -17,9 +17,10 @@ See COPYRIGHT for license information.
 #include "netcdf.h"
 #include "ncbytes.h"
 #include "ncuri.h"
-#include "ncauth.h"
 #include "nclog.h"
 #include "ncpathmgr.h"
+#include "ncs3sdk.h"
+#include "ncauth.h"
 
 #ifdef _MSC_VER
 #include <windows.h>

diff --git a/libdispatch/dcopy.c b/libdispatch/dcopy.c
@@ -694,7 +694,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2)
     int gid;
     uintptr_t id;
 
-    id = grp;
+    id = (uintptr_t)grp;
     nclistpush(queue,(void*)id); /* prime the queue */
     while(nclistlength(queue) > 0) {
         id = (uintptr_t)nclistremove(queue,0);
@@ -712,7 +712,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2)
             goto done;
 	/* push onto the end of the queue */
         for(i=0;i<nids;i++) {
-	    id = ids[i];
+	    id = (uintptr_t)ids[i];
 	    nclistpush(queue,(void*)id);
 	}
 	free(ids); ids = NULL;

diff --git a/libdispatch/ddispatch.c b/libdispatch/ddispatch.c
@@ -134,12 +134,12 @@ int
 NCDISPATCH_finalize(void)
 {
     int status = NC_NOERR;
-    NC_freeglobalstate();
 #if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4)
     curl_global_cleanup();
 #endif
 #if defined(ENABLE_DAP4)
    ncxml_finalize();
 #endif
+    NC_freeglobalstate(); /* should be one of the last things done */
     return status;
 }
diff --git a/libdispatch/derror.c b/libdispatch/derror.c
@@ -267,9 +267,9 @@ const char *nc_strerror(int ncerr1)
       case NC_EMPI: return "NetCDF: MPI operation failed.";
       case NC_ERCFILE:
 	return "NetCDF: RC File Failure.";
-     case NC_ENULLPAD:
+      case NC_ENULLPAD:
        return "NetCDF: File fails strict Null-Byte Header check.";
-     case NC_EINMEMORY:
+      case NC_EINMEMORY:
        return "NetCDF: In-memory File operation failed.";
       case NC_ENCZARR:
 	 return "NetCDF: NCZarr error";

diff --git a/libdispatch/dhttp.c b/libdispatch/dhttp.c
@@ -99,7 +99,11 @@ nc_http_open_verbose(const char* path, int verbose, NC_HTTP_STATE** statep)
         {stat = NCTHROW(NC_ENOMEM); goto done;}
     state->path = strdup(path);
     state->url = uri; uri = NULL;    
-    state->format = (NC_iss3(state->url)?HTTPS3:HTTPCURL);
+#ifdef ENABLE_S3
+    state->format = (NC_iss3(state->url,NULL)?HTTPS3:HTTPCURL);
+#else
+    state->format = HTTPCURL;
+#endif
 
     switch (state->format) {
     case HTTPCURL: {

diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c
@@ -898,13 +898,16 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
         ncurisetfragments(uri,sfrag);
         nullfree(sfrag); sfrag = NULL;
 
+#ifdef ENABLE_S3
 	/* If s3, then rebuild the url */
-	if(NC_iss3(uri)) {
+	if(NC_iss3(uri,NULL)) {
 	    NCURI* newuri = NULL;
 	    if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done;
 	    ncurifree(uri);
 	    uri = newuri;
-	} else if(strcmp(uri->protocol,"file")==0) {
+	} else
+#endif
+	if(strcmp(uri->protocol,"file")==0) {
             /* convert path to absolute */
 	    char* canon = NULL;
 	    abspath = NCpathabsolute(uri->path);