Significantly Improve Amazon S3 Cloud Storage Support

## S3 Related Fixes

* Add comprehensive support for specifying AWS profiles to provide access credentials.
* Parse the files "~/.aws/config" and "~/.aws/credentials to provide credentials for the HDF5 ROS3 driver and to locate default region.
* Add a function to obtain the currently active S3 credentials. The search rules are defined in docs/nczarr.md.
* Provide documentation for the new features.
* Modify the struct NCauth (in include/ncauth.h) to replace specific S3 credentials with a profile name.
* Add a unit test to test the operation of profile and credentials management.
* Add support for URLS of the form "s3://<bucket>/<key>"; this requires obtaining a default region.
* Allows the specification of profile and/or region in a URL of the form "#mode=nczarr,...&aws.region=...&aws.profile=..."

## Misc. Fixes

* Move the ezxml code to libdispatch so that it can be used both by DAP4 and nczarr.
* Modify nclist to provide a deep clone operation.
* Modify ncuri to provide a deep clone operation.
* Modify the .rc file format to allow the specification of a path to be tested when looking for an entry in the .rc file.
* Ensure that the NC_rcload function is called.
* Modify nchttp to support setting request headers.

.github/workflows/run_tests.yml

@@ -4,7 +4,7 @@
 
 name: Run netCDF Tests
 
-on: [pull_request]
+on: [pull_request,push]
 
 jobs:
 

CMakeLists.txt

@@ -1136,7 +1136,7 @@ ENDIF(ENABLE_NCZARR_S3)
 
 IF(NOT ENABLE_S3_SDK)
   IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
-    message(FATAL_ERROR "AWS S3 libraries not found; please specify option DENABLE_NCZARR_S3=NO")
+    message(FATAL_ERROR "S3 support library not found; please specify option DENABLE_NCZARR_S3=NO")
     SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
     SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
   ENDIF()

Makefile.am

@@ -113,7 +113,8 @@ endif
 # and run. ncgen must come before ncdump, because their tests
 # depend on it.
 SUBDIRS = include $(H5_TEST_DIR) libdispatch libsrc $(LIBSRC4_DIR)	\
-$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} ${NCPOCO} ${ZARR} liblib \
+$(LIBSRCP) $(LIBHDF4) $(LIBHDF5) $(OCLIB) $(DAP2) ${DAP4} \
+${NCPOCO} ${ZARR} liblib \
 $(NCGEN3) $(NCGEN) $(NCDUMP) ${PLUGIN_DIR} $(TESTDIRS) docs		\
 $(EXAMPLES)
 

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.2 - TBD
 
+* [Enhancement] Support Amazon S3 access for NCZarr. Also support use of the existing Amazon SDK credentials system. See [Github #2???](https://github.com/Unidata/netcdf-c/pull/2???)
 * [Bug Fix] Ensure that internal Fortran APIs are always defined. See [Github #2098](https://github.com/Unidata/netcdf-c/pull/2098).
 * [Enhancement] Support filters for NCZarr. See [Github #2101](https://github.com/Unidata/netcdf-c/pull/2101)
 * [Bug Fix] Make PR 2075 long file name be idempotent. See [Github #2094](https://github.com/Unidata/netcdf-c/pull/2094).

configure.ac

@@ -642,25 +642,27 @@ AC_MSG_RESULT($enable_nczarr_s3)
 # Note we check for the library after checking for enable_nczarr_s3
 # because for some reason this screws up if we unconditionally test for sdk
 # and it is not available. Fix someday
-have_aws=no
+enable_s3_sdk=no
 if test "x$enable_nczarr_s3" = xyes ; then
 # See if we have the s3 aws library
 # Check for the AWS S3 SDK library
 AC_LANG_PUSH([C++])
-AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
+AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [enable_s3_sdk=yes],[enable_s3_sdk=no])
 AC_LANG_POP
 fi
 
 AC_MSG_CHECKING([whether AWS S3 SDK library is available])
-AC_MSG_RESULT([$have_aws])
+AC_MSG_RESULT([$enable_s3_sdk])
 
-if test "x$have_aws" = xno ; then
-AC_MSG_WARN([AWS SDK not found; disabling S3 support])
+if test "x$enable_s3_sdk" = xno ; then
+AC_MSG_WARN([No S3 library available; disabling S3 support])
 enable_nczarr_s3=no
-else
+fi
+
+if test "x$enable_s3_sdk" = xyes ; then
 AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
 fi
-AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
+AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
 
 # Check for enabling S3 testing
 AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
@@ -1265,10 +1267,10 @@ if test "x$enable_hdf5" = xyes; then
 
    # Check for the main hdf5 and hdf5_hl library.
 
-   AC_SEARCH_LIBS([H5Fflush], [hdf5dll hdf5], [],
+   AC_SEARCH_LIBS([H5Fflush], [hdf5 hdf5.dll], [],
+   [AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-hdf5, or see config.log for errors.])])
+   AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hl hdf5_hl.dll], [],
    [AC_MSG_ERROR([Can't find or link to the hdf5 library. Use --disable-hdf5, or see config.log for errors.])])
-   AC_SEARCH_LIBS([H5DSis_scale], [hdf5_hldll hdf5_hl], [],
-   [AC_MSG_ERROR([Can't find or link to the hdf5 high-level. Use --disable-hdf5, or see config.log for errors.])])
 
    AC_CHECK_HEADERS([hdf5.h], [], [AC_MSG_ERROR([Compiling a test with HDF5 failed.  Either hdf5.h cannot be found, or config.log should be checked for other reason.])])
 
@@ -1600,7 +1602,7 @@ AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes])
 AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
 AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
 AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
-AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes])
+AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes])
 AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
 AM_CONDITIONAL(ENABLE_BLOSC, [test "x$enable_blosc" = xyes])
 AM_CONDITIONAL(ENABLE_SZIP, [test "x$enable_szip" = xyes])

docs/CMakeLists.txt

@@ -92,6 +92,6 @@ obsolete/fan_utils.html bestpractices.md filters.md indexing.md
 inmemory.md DAP2.dox attribute_conventions.md FAQ.md
 file_format_specifications.md known_problems.md
 COPYRIGHT.dox user_defined_formats.md DAP4.md DAP4.dox
-testserver.dox byterange.dox filters.md nczarr.md)
+testserver.dox byterange.dox filters.md nczarr.md auth.md)
 
 ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

docs/Doxyfile.in

@@ -742,8 +742,9 @@ INPUT = \
     @abs_top_srcdir@/docs/windows-binaries.md \
     @abs_top_srcdir@/docs/attribute_conventions.md \
     @abs_top_srcdir@/docs/file_format_specifications.md \
-    @abs_top_srcdir@/docs/byterange.dox \
     @abs_top_srcdir@/docs/inmemory.md \
+    @abs_top_srcdir@/docs/filters.md \
+    @abs_top_srcdir@/docs/byterange.dox \
     @abs_top_srcdir@/docs/auth.md \
     @abs_top_srcdir@/docs/nczarr.md \
     @abs_top_srcdir@/docs/notes.md \

docs/Doxyfile.user

@@ -751,6 +751,7 @@ INPUT = \
     ./docs/byterange.dox \
     ./docs/inmemory.md \
     ./docs/auth.md \
+    ./docs/filters.md \
     ./docs/notes.md \
     ./docs/all-error-codes.md \
     ./docs/building-with-cmake.md \

docs/Makefile.am

@@ -11,7 +11,7 @@ EXTRA_DIST = netcdf.m4 DoxygenLayout.xml Doxyfile.in footer.html	\
 mainpage.dox tutorial.dox cdl.dox			\
 architecture.dox internal.dox windows-binaries.md			\
 building-with-cmake.md CMakeLists.txt groups.dox notes.md	\
-install-fortran.md all-error-codes.md credits.md auth.md		\
+install-fortran.md all-error-codes.md credits.md auth.md filters.md \
 obsolete/fan_utils.html indexing.dox	\
 inmemory.md attribute_conventions.md FAQ.md	                \
 file_format_specifications.md known_problems.md COPYRIGHT.md		\