Additional Fixes to NCZarr

Primary Fixes:
* Add a whole variable optimization -- used in the rare case that nc_get/put_vara covers the whole of a variable and the variable has a single chunk.
* Fix chunking error when stride causes whole chunks to be skipped.
* Fix some memory leaks
* Add test cases
* Add one performance test to nczarr_test/. This uses the timer utils from unit_test: timer_utils.[ch].
* Move ncdumpchunks utility from ncdump to nczarr_test

Misc. Other Changes:
* Make check for aws libraries conditional on --enable-nczarr-s3
* Remove all but one bm tests from nczarr_test until they are working.
* Remove another dependency on HDF5 from supposedly non-HDF5 specific code; specifically hdf5_log_hdf5.
* Make the BAIL2 macro be hdf5 specific and replace elsewhere with an HDF5 independent equivalent.
* Move hdf5cache.c to libsrc4/nc4cache.c because it is used by nczarr.
* Modify unit_tests so that some of them are run even if using Windows.
* Misc. small bug fixes and refactors and memory leaks.
* Rename some conflicting tests for cmake.
* Attempted to make nc_perf work with cmake and failed.

CMakeLists.txt

@@ -1896,6 +1896,10 @@ ENDIF()
 
 IF(ENABLE_NCZARR)
   ADD_SUBDIRECTORY(libnczarr)
+  FILE(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.h
+       DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
+  FILE(COPY ${netCDF_SOURCE_DIR}/unit_test/timer_utils.c
+       DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
 ENDIF()
 
 add_subdirectory(liblib)
@@ -1927,9 +1931,6 @@ IF(ENABLE_TESTS)
   IF(USE_HDF5)
     INCLUDE_DIRECTORIES(h5_test)
     ADD_SUBDIRECTORY(nc_test4)
-    IF(BUILD_BENCHMARKS)
-      ADD_SUBDIRECTORY(nc_perf)
-    ENDIF()
     ADD_SUBDIRECTORY(h5_test)
   ENDIF()
   IF(ENABLE_DAP2)

configure.ac

@@ -594,6 +594,10 @@ if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; t
     enable_nczarr_s3_tests=no
 fi
 
+# Set default
+have_aws=no
+
+if test "x$enable_nczarr_s3" = xyes ; then
 # See if we have the s3 aws library
 # Check for the AWS S3 SDK library
 AC_LANG_PUSH([C++])
@@ -605,6 +609,7 @@ AC_MSG_RESULT([$have_aws])
 if test "x$have_aws" = xyes ; then
 AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
 fi
+fi
 AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
 
 if test "x$have_aws" = xno ; then
@@ -1679,6 +1684,8 @@ AM_CONDITIONAL([AX_IGNORE], [test xno = xyes])
 
 AC_MSG_NOTICE([generating header files and makefiles])
 AC_CONFIG_FILES(test_common.sh:test_common.in)
+AC_CONFIG_FILES(nczarr_test/timer_utils.h:unit_test/timer_utils.h)
+AC_CONFIG_FILES(nczarr_test/timer_utils.c:unit_test/timer_utils.c)
 AC_CONFIG_FILES(nc_test4/findplugin.sh:nc_test4/findplugin.in)
 AC_CONFIG_FILES(examples/C/findplugin.sh:nc_test4/findplugin.in)
 AC_CONFIG_FILES(ncdap_test/findtestserver.c:ncdap_test/findtestserver.c.in)

dap4_test/Makefile.am

@@ -1,52 +1,29 @@
 ## This is a automake file, part of Unidata's netCDF package.
-# Copyright 2018, see the COPYRIGHT file for more information.
+# Copyright 2018v, see the COPYRIGHT file for more information.
 
-# This file builds and runs DAP4 tests.
+# This file builds and runs DAP tests.
 
 # Put together AM_CPPFLAGS and AM_LDFLAGS.
 include $(top_srcdir)/lib_flags.am
 
-#TEST_EXTENSIONS = .sh
-
+# Un comment to use a more verbose test driver
 #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
-#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
 #LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
-#TESTS_ENVIRONMENT = export SETX=1;
+#TEST_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
+
+# Note which tests depend on other tests. Necessary for make -j check.
+TEST_EXTENSIONS = .sh
 
 LDADD = ${top_builddir}/liblib/libnetcdf.la
-AM_CPPFLAGS += -I$(top_srcdir)/libdap4
+AM_CPPFLAGS += -I$(top_srcdir)/liblib
+AM_CPPFLAGS += -DTOPSRCDIR=${abs_top_srcdir}
+AM_CPPFLAGS += -DTOPBINDIR=${abs_top_bindir}
 
 # Set up the tests; do the .sh first, then .c
 check_PROGRAMS =
 TESTS =
 
-if ENABLE_DAP4
-
-# WARNING: these are unit tests, so they will not
-# appear in CMakeLists.txt
-check_PROGRAMS += test_parse test_meta test_data
-test_parse_SOURCES = test_parse.c test_common.h
-test_meta_SOURCES = test_meta.c test_common.h
-test_data_SOURCES = test_data.c test_common.h
-
-TESTS += test_parse.sh
-
-if BUILD_UTILITIES
-# These rely on ncdump
-TESTS += test_parse.sh
-TESTS += test_raw.sh
-TESTS += test_meta.sh
-TESTS += test_data.sh
-TESTS += test_fillmismatch.sh
-endif
-
-# Note which tests depend on other tests. Necessary for make -j check.
-test_raw.log: test_parse.log
-test_meta.log: test_raw.log
-test_data.log: test_meta.log
-
-noinst_PROGRAMS = dump
-dump_SOURCES = dump.c
+noinst_PROGRAMS =
 
 # Disable Dap4 Remote Tests until the test server is working
 if AX_IGNORE
@@ -56,17 +33,12 @@ if ENABLE_DAP_REMOTE_TESTS
 noinst_PROGRAMS += findtestserver4 pingurl4
 findtestserver4_SOURCES = findtestserver4.c
 pingurl4_SOURCES = pingurl4.c
-endif
-
-if ENABLE_DAP_REMOTE_TESTS
 if BUILD_UTILITIES
   # relies on ncdump
   TESTS += test_remote.sh
 endif
 endif
-endif
-
-endif #ENABLE_DAP4
+endif #AX_IGNORE
 
 EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \
              test_raw.sh test_remote.sh test_hyrax.sh test_fillmismatch.sh \

debug/cf

@@ -1,27 +1,25 @@
 #!/bin/bash
 
 #NB=1
-DB=1
+#DB=1
 #X=-x
 
 #ANSI=1
-#MEM=1
+MEM=1
 #NOTUIL=1
 FAST=1
 #PROF=1
-#BENCH=1
 
 NCZARR=1
 HDF5=1
-DAP=1
-#S3=1
-#S3TEST=1
-SZIP=1
+#DAP=1
+S3=1
+S3TEST=1
+#SZIP=1
 #HDF4=1
 #PNETCDF=1
 #PAR4=1
-
-AWSSDK_DIR="/cygdrive/c/tools/aws-cpp-sdk-all"
+#NOCDF5=1
 
 #TESTSERVERS="149.165.169.123:8080,remotetest.unidata.ucar.edu"
 #TESTSERVERS="localhost:8080,149.165.169.123:8080"
@@ -88,12 +86,19 @@ CFLAGS="-g -O0 $CFLAGS -Wno-undefined"
 MAKE=make
 IGNORE="test 0 = 1"
 
+FLAGS="$FLAGS --enable-logging"
+FLAGS="$FLAGS --enable-byterange"
+#FLAGS="$FLAGS --enable-extra-tests"
+FLAGS="$FLAGS --enable-mmap"
+FLAGS="$FLAGS --enable-examples"
+FLAGS="$FLAGS --enable-unit-tests"
+FLAGS="$FLAGS --enable-benchmarks"
+
 if test "x$HDF5" != x ; then
 CPPFLAGS="-I${stddir}/include $CPPFLAGS"
 LDFLAGS="-lhdf5_hl -lhdf5 -lz $LDFLAGS"
 else
-FLAGS="$FLAGS --disable-netcdf-4"
-#FLAGS="$FLAGS --disable-hdf5"
+FLAGS="$FLAGS --disable-hdf5"
 fi
 
 if test "x$HDF4" = x1 ; then
@@ -117,39 +122,31 @@ if test "x$MPIO" = x1 ; then
   FLAGS="$FLAGS --enable-parallel-tests"
 fi
 
+if test "x$NOCDF5" = "x1" ; then
+FLAGS="$FLAGS --disable-cdf5"
+fi
+
 if test "x$PNETCDF" = x1 ; then
   FLAGS="$FLAGS --enable-pnetcdf"
 fi
 
 if test "x$NCZARR" != "x" ; then
 FLAGS="$FLAGS --enable-nczarr"
-fi
-
 if test "x$S3" != x ; then
-FLAGS="$FLAGS --enable-nczarr-s3"
-else
-FLAGS="$FLAGS --disable-nczarr-s3"
+    FLAGS="$FLAGS --enable-nczarr-s3"
+    LDFLAGS="$LDFLAGS -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-c-common"
+    if test "x$S3TEST" != x ; then
+        FLAGS="$FLAGS --enable-nczarr-s3-tests"
+    fi  
 fi
-if test "x$S3TEST" != x ; then
-FLAGS="$FLAGS --enable-nczarr-s3-tests"
-export AWSSDK_DIR="${AWSSDK_DIR}"
-LDFLAGS="$LDFLAGS $AWSSDK_DIR/bin/aws-cpp-sdk-s3.dll"
-CPPFLAGS="$CPPFLAGS -I${AWSSDK_DIR}/include"
 fi
 
 if test "x${PROF}" = x1 ; then
 CFLAGS="${CFLAGS} -pg"
 LDFLAGS="${LDFLAGS} -pg"
+FLAGS="$FLAGS --enable-static --disable-shared"
 fi
 
-if test "x${BENCH}" = x1 ; then
-FLAGS="$FLAGS --enable-benchmarks"
-fi
-
-FLAGS="$FLAGS --enable-byterange"
-FLAGS="$FLAGS --disable-filter-testing"
-#FLAGS="$FLAGS --enable-erange-fill"
-
 export PATH
 export CC
 export CPPFLAGS