Fix duplicate definition when using aws-sdk-cpp.

re: Issue #2927

The NC_s3sdkinitialize NC_s3sdkfinalize functions were
misplaced. They should have been moved from ds3util.c to
ncs3sdk_h5.c.  When using ncs3sdl_aws.cpp, this resulted in a
duplicate definition.

Also, found and fixed a memory leak in the NCZarr S3 code.

include/ncs3sdk.h

@@ -45,6 +45,7 @@ struct NCglobalstate;
 extern "C" {
 #endif
 
+/* API for ncs3sdk_XXX.[c|cpp] */
 EXTERNL int NC_s3sdkinitialize(void);
 EXTERNL int NC_s3sdkfinalize(void);
 EXTERNL void* NC_s3sdkcreateclient(NCS3INFO* context);
@@ -60,8 +61,7 @@ EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* pref
 EXTERNL int NC_s3sdkdeletekey(void* client0, const char* bucket, const char* pathkey, char** errmsgp);
 
 /* From ds3util.c */
-EXTERNL int NC_s3sdkinitialize(void);
-EXTERNL int NC_s3sdkfinalize(void);
+EXTERNL void NC_s3sdkenvironment(void);
 
 EXTERNL int NC_getdefaults3region(NCURI* uri, const char** regionp);
 EXTERNL int NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp);

libdispatch/ds3util.c

@@ -43,9 +43,6 @@ enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4};
 static const char* awsconfigfiles[] = {".aws/config",".aws/credentials",NULL};
 #define NCONFIGFILES (sizeof(awsconfigfiles)/sizeof(char*))
 
-static int ncs3_initialized = 0;
-static int ncs3_finalized = 0;
-
 /**************************************************/
 /* Forward */
 
@@ -56,38 +53,21 @@ static int awsparse(const char* text, NClist* profiles);
 /**************************************************/
 /* Capture environmental Info */
 
-EXTERNL int
-NC_s3sdkinitialize(void)
-{
-    if(!ncs3_initialized) {
-	ncs3_initialized = 1;
-	ncs3_finalized = 0;
-    }
-    {
-        /* Get various environment variables as defined by the AWS sdk */
-	NCglobalstate* gs = NC_getglobalstate();
-	if(getenv("AWS_REGION")!=NULL)
-	    gs->aws.default_region = nulldup(getenv("AWS_REGION"));
-	else if(getenv("AWS_DEFAULT_REGION")!=NULL)
-	    gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION"));
-	else if(gs->aws.default_region == NULL)
-	    gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION);
-	gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID"));
-	gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE"));
-	gs->aws.profile = nulldup(getenv("AWS_PROFILE"));
-	gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY"));
-    }
-    return NC_NOERR;
-}
-
-EXTERNL int
-NC_s3sdkfinalize(void)
+EXTERNL void
+NC_s3sdkenvironment(void)
 {
-    if(!ncs3_finalized) {
-	ncs3_initialized = 0;
-	ncs3_finalized = 1;
-    }
-    return NC_NOERR;
+    /* Get various environment variables as defined by the AWS sdk */
+    NCglobalstate* gs = NC_getglobalstate();
+    if(getenv("AWS_REGION")!=NULL)
+        gs->aws.default_region = nulldup(getenv("AWS_REGION"));
+    else if(getenv("AWS_DEFAULT_REGION")!=NULL)
+        gs->aws.default_region = nulldup(getenv("AWS_DEFAULT_REGION"));
+    else if(gs->aws.default_region == NULL)
+        gs->aws.default_region = nulldup(AWS_GLOBAL_DEFAULT_REGION);
+    gs->aws.access_key_id = nulldup(getenv("AWS_ACCESS_KEY_ID"));
+    gs->aws.config_file = nulldup(getenv("AWS_CONFIG_FILE"));
+    gs->aws.profile = nulldup(getenv("AWS_PROFILE"));
+    gs->aws.secret_access_key = nulldup(getenv("AWS_SECRET_ACCESS_KEY"));
 }
 
 /**************************************************/

libdispatch/ncs3sdk_aws.cpp

@@ -133,17 +133,19 @@ NC_s3sdkinitialize(void)
     if(!ncs3_initialized) {
     	ncs3_initialized = 1;
 	    ncs3_finalized = 0;
-
 
 #ifdef DEBUG
-	    //ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
+        //ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;
         ncs3options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace;
         ncs3options.httpOptions.installSigPipeHandler = true; 
         ncs3options.loggingOptions.logger_create_fn = [] { return std::make_shared<Aws::Utils::Logging::ConsoleLogSystem>(Aws::Utils::Logging::LogLevel::Trace); };
 
 #endif
         Aws::InitAPI(ncs3options);
 
+	/* Get environment information */
+        NC_s3sdkenvironment();
+
     }
     return NCUNTRACE(NC_NOERR);
 }
@@ -500,7 +502,6 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
     int stat = NC_NOERR;
     const char* key = NULL;
 
-    const char* mcontent = (char*)content;
     NCTRACE(11,"bucket=%s pathkey=%s count=%lld content=%p",bucket,pathkey,count,content);
 
     AWSS3CLIENT s3client = (AWSS3CLIENT)s3client0;
@@ -535,7 +536,7 @@ NC_s3sdkwriteobject(void* s3client0, const char* bucket, const char* pathkey,  s
     put_request.SetContentLength((long long)count);
 
     std::shared_ptr<Aws::IOStream> data = std::shared_ptr<Aws::IOStream>(new Aws::StringStream());
-    data->rdbuf()->pubsetbuf((char*)content,count);
+    data->rdbuf()->pubsetbuf((char*)content,(std::streamsize)count);
     put_request.SetBody(data);
     auto put_result = AWSS3GET(s3client)->PutObject(put_request);
     if(!put_result.IsSuccess()) {

libdispatch/ncs3sdk_h5.c

@@ -108,6 +108,37 @@ static int queryinsert(NClist* list, char* ekey, char* evalue);
 
 #define NT(x) ((x)==NULL?"null":x)
 
+/**************************************************/
+
+static int ncs3_initialized = 0;
+static int ncs3_finalized = 0;
+
+EXTERNL int
+NC_s3sdkinitialize(void)
+{
+    if(!ncs3_initialized) {
+	ncs3_initialized = 1;
+	ncs3_finalized = 0;
+    }
+
+    /* Get environment information */
+    NC_s3sdkenvironment(void);
+
+    return NC_NOERR;
+}
+
+EXTERNL int
+NC_s3sdkfinalize(void)
+{
+    if(!ncs3_finalized) {
+	ncs3_initialized = 0;
+	ncs3_finalized = 1;
+    }
+    return NC_NOERR;
+}
+
+/**************************************************/
+
 #if 0
 static void
 dumps3info(NCS3INFO* s3info, const char* tag)

libnczarr/zmap_s3sdk.c

@@ -499,20 +499,21 @@ s3clear(void* s3client, const char* bucket, const char* rootkey)
 {
     int stat = NC_NOERR;
     char** list = NULL;
-    char** p;
     size_t nkeys = 0;
 
     if(s3client && bucket && rootkey) {
         if((stat = NC_s3sdksearch(s3client, bucket, rootkey, &nkeys, &list, NULL)))
             goto done;
         if(list != NULL) {
-            for(p=list;*p;p++) {
+	    size_t i;
+	    for(i=0;i<nkeys;i++) {
+                char* p = list[i];
 	        /* If the key is the rootkey, skip it */
-	        if(strcmp(rootkey,*p)==0) continue;
+	        if(strcmp(rootkey,p)==0) continue;
 #ifdef S3DEBUG
-fprintf(stderr,"s3clear: %s\n",*p);
+fprintf(stderr,"s3clear: %s\n",p);
 #endif
-                if((stat = NC_s3sdkdeletekey(s3client, bucket, *p, NULL)))	
+                if((stat = NC_s3sdkdeletekey(s3client, bucket, p, NULL)))	
 	            goto done;
 	    }
         }

nczarr_test/run_corrupt.sh

@@ -32,4 +32,7 @@ testnoshape2() {
 }
 
 testnoshape1
-if test "x$FEATURE_S3TESTS" = xyes ; then testnoshape2; fi
+if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_S3_INTERNAL" = xyes ; then
+    # The aws-sdk-cpp driver does not support google storage
+    testnoshape2
+fi