Cleanup DAP4 testing

NOTE: This PR should not be included in 4.9.1 since additional
DAP4 related PRs will be forthcoming.

This PR makes major changes to libdap4 and dap4_test driven by changes to TDS.

* Enable DAP4
* Clean up the test input files and the test baseline comparison files. This entails:
    * Remove a multitude of unused test input and baseline data files; among them are dap4_test/: daptestfiles, dmrtestfiles, nctestfiles, and misctestfiles.
    * Define a canonical set of test input files and record in dap4_test/cdltestfiles.
    * Use the cdltestfiles to generate the .nc test inputs. This set of .nc files is then moved to the d4ts (DAP4 test server) war file in the tds repository. This set then becomes the canonical set of DAP4 test sources.
    * Scrape d4ts to obtain copies of the raw streams of DAP4 encoded data. The .dmr and .dap streams are then stored in dap4_test/rawtestfiles.
    * Disable some remote server tests until those servers are fixed.
* Add an option to ncdump (-XF) that forces the type of the _FillValue attribute; this is primarily to simplify testing of fill mismatch.
* Minor bug fixes to ncgen.
* Changes to libdap4:
    * Replace old checksum hack with the dap4.checksum flag.
    * Support the dap4.XXX controls.
    * Cleanup _FillValue handling, especially var-attribute type mismatches.
    * Fix enum handling based on changes to netcdf-java.
* Changes to dap4_test:
    * Add getopt support to various test support programs.
    * Remove unneeded shell scripts.
    * Add new scripts: test_curlopt.sh

CMakeLists.txt

@@ -1062,9 +1062,9 @@ ENDIF()
 IF(ENABLE_DAP)
   SET(USE_DAP ON CACHE BOOL "")
   SET(ENABLE_DAP2 ON CACHE BOOL "")
-  SET(ENABLE_DAP4 OFF CACHE BOOL "")
-
-  IF(NOT ENABLE_HDF5)
+  IF(ENABLE_HDF5)
+    SET(ENABLE_DAP4 ON CACHE BOOL "")
+  ELSE(ENABLE_HDF5)
     SET(ENABLE_DAP4 OFF CACHE BOOL "")
   ENDIF(ENABLE_HDF5)
 
@@ -1262,6 +1262,8 @@ IF(MSVC)
          DESTINATION ${netCDF_BINARY_DIR}/nczarr_test/)
     FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c
          DESTINATION ${netCDF_BINARY_DIR}/ncdap_test/)
+    FILE(COPY ${netCDF_SOURCE_DIR}/libsrc/XGetopt.c
+         DESTINATION ${netCDF_BINARY_DIR}/dap4_test/)
   ENDIF()
 ENDIF()
 

RELEASE_NOTES.md

@@ -16,6 +16,7 @@ This file contains a high-level description of this package's evolution. Release
 
 #### Changes
 
+* [Bug Fix] Make major changes to libdap4 and dap4_test to update the non-remote DAP4 tests. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
 * [Bug Fix] Fix some race conditions due to use of a common file in multiple shell scripts . See [Github #2552](https://github.com/Unidata/netcdf-c/pull/2552).
 * [Enhancement][Documentation] Add Plugins Quick Start Guide.  See [GitHub #2524](https://github.com/Unidata/netcdf-c/pull/2524) for more information.
 * [Enhancement] Add new entries in `netcdf_meta.h`, `NC_HAS_BLOSC` and `NC_HAS_BZ2`. See [Github #2511](https://github.com/Unidata/netcdf-c/issues/2511) and [Github #2512](https://github.com/Unidata/netcdf-c/issues/2512) for more information.

configure.ac

@@ -663,13 +663,15 @@ AC_ARG_WITH([testservers],
               [AS_HELP_STRING([--with-testservers=<host:port>,<host:port>...],
                               [Specify the testserver(s) to try for remote tests.])],
             [REMOTETESTSERVERS=$with_testservers], [REMOTETESTSERVERS=no])
-msg="$REMOTETESTSERVERS"
 if test "x$REMOTETESTSERVERS" = xno ; then
-  svclist="remotetest.unidata.ucar.edu"
-  REMOTETESTSERVERS="$svclist"
+  dfaltsvc="remotetest.unidata.ucar.edu"
+  REMOTETESTSERVERS="${dfaltsvc}"
 fi
 AC_MSG_RESULT([$svclist])
 AC_DEFINE_UNQUOTED([REMOTETESTSERVERS], ["$REMOTETESTSERVERS"], [the testservers for remote tests.])
+if test "x$dfaltsvc" != x ; then
+  AC_MSG_NOTICE([Using default: --with-testservers=${dfaltsvc}])
+fi
 
 # Set the config.h flags
 if test "x$enable_dap" = xyes; then

dap4_test/CMakeLists.txt

@@ -15,10 +15,7 @@ ADD_SUBDIRECTORY(baselineremote)
 ADD_SUBDIRECTORY(baselinehyrax)
 ADD_SUBDIRECTORY(baselinethredds)
 ADD_SUBDIRECTORY(cdltestfiles)
-ADD_SUBDIRECTORY(daptestfiles)
-ADD_SUBDIRECTORY(dmrtestfiles)
-ADD_SUBDIRECTORY(nctestfiles)
-ADD_SUBDIRECTORY(misctestfiles)
+ADD_SUBDIRECTORY(rawtestfiles)
 
 FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh)
 
@@ -39,24 +36,24 @@ IF(ENABLE_TESTS)
   # keep it from being too bad.
 
   IF(BUILD_UTILITIES)
-    add_sh_test(dap4_test test_fillmismatch)
-
-    # Following tests are Unit tests, so not tested under visual studio
-    IF(NOT MSVC)
       INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/libdap4)
       build_bin_test(test_parse)
       build_bin_test(test_meta)
+IF(USE_X_GETOPT)
+      build_bin_test(test_data XGetopt.c)
+ELSE()
       build_bin_test(test_data)
+ENDIF()
       add_sh_test(dap4_test test_parse)
       add_sh_test(dap4_test test_raw)
       add_sh_test(dap4_test test_meta)
       add_sh_test(dap4_test test_data)
-    ENDIF()
   ENDIF(BUILD_UTILITIES)
 
   IF(ENABLE_DAP_REMOTE_TESTS)
-      add_sh_test(dap4_test test_hyrax)
-      add_sh_test(dap4_test test_thredds)
+#      add_sh_test(dap4_test test_remote)
+#      add_sh_test(dap4_test test_hyrax)
+#      add_sh_test(dap4_test test_thredds)
   ENDIF(ENABLE_DAP_REMOTE_TESTS)
 
 ENDIF(ENABLE_TESTS)

dap4_test/Makefile.am

@@ -30,9 +30,9 @@ check_PROGRAMS += test_parse test_meta test_data
 
 noinst_PROGRAMS =
 
-TESTS += test_parse.sh test_meta.sh test_data.sh test_fillmismatch.sh test_raw.sh 
+TESTS += test_parse.sh test_meta.sh test_data.sh test_raw.sh 
 
-# Note tst_curlopt.sh is intended to be run manually; see comments in file.
+# Note test_curlopt.sh is intended to be run manually; see comments in file.
 
 # Note: This program name was changed to findtestserver4
 # to avoid cmake complaint about duplicate targets.
@@ -41,21 +41,19 @@ findtestserver4_SOURCES = findtestserver4.c
 pingurl4_SOURCES = pingurl4.c
 
 # Disable Dap4 Remote Tests until the test server is working
-if ENABLE_DAP_REMOTE_TESTS
 if BUILD_UTILITIES
-  # relies on ncdump
-  TESTS += test_hyrax.sh test_thredds.sh
-if AX_IGNORE
-  TESTS += test_remote.sh
-endif
+if ENABLE_DAP_REMOTE_TESTS
+# Test require help at the server end
+#  TESTS += test_remote.sh test_hyrax.sh test_thredds.sh
 endif
 endif
 
-EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \
-             test_raw.sh test_remote.sh test_hyrax.sh test_thredds.sh test_fillmismatch.sh \
-             tst_curlopt.sh d4test_common.sh \
-	     daptestfiles dmrtestfiles cdltestfiles nctestfiles misctestfiles \
-	     baseline baselineraw baselineremote baselinehyrax baselinethredds CMakeLists.txt test_common.h
+EXTRA_DIST = CMakeLists.txt test_common.h \
+	d4test_common.sh \
+	test_curlopt.sh test_data.sh test_hyrax.sh test_meta.sh \
+	test_parse.sh test_raw.sh test_remote.sh test_thredds.sh \
+	cdltestfiles rawtestfiles nctestfiles \
+	baseline baselineraw baselineremote baselinehyrax baselinethredds
 
 CLEANFILES = *.exe
 # This should only be left behind if using parallel io
@@ -80,7 +78,7 @@ clean-local-check:
 
 # The shell file maketests.sh is used to build the testdata
 # for dap4 testing. It creates and fills the directories
-# daptestfiles, dmrtestfiles, and cdltestfiles
+# daptestfiles, dmrtestfiles, and testsrc
 # As a rule, this will only be invoked if there was a change
 # in the GenerateRaw.java program in the thredds tree.
 maketests::