Primary change: add dap4 support

Specific changes:
1. Add dap4 code: libdap4 and dap4_test.
   Note that until the d4ts server problem is solved, dap4 is turned off.
2. Modify various files to support dap4 flags:
	configure.ac, Makefile.am, CMakeLists.txt, etc.
3. Add nc_test/test_common.sh. This centralizes
   the handling of the locations of various
   things in the build tree: e.g. where is
   ncgen.exe located. See nc_test/test_common.sh
   for details.
4. Modify .sh files to use test_common.sh
5. Obsolete separate oc2 by moving it to be part of
   netcdf-c. This means replacing code with netcdf-c
   equivalents.
5. Add --with-testserver to configure.ac to allow
   override of the servers to be used for --enable-dap-remote-tests.
6. There were multiple versions of nctypealignment code. Try to
   centralize in libdispatch/doffset.c and include/ncoffsets.h
7. Add a unit test for the ncuri code because of its complexity.
8. Move the findserver code out of libdispatch and into
   a separate, self contained program in ncdap_test and dap4_test.
9. Move the dispatch header files (nc{3,4}dispatch.h) to
   .../include because they are now shared by modules.
10. Revamp the handling of TOPSRCDIR and TOPBUILDDIR for shell scripts.
11. Make use of MREMAP if available
12. Misc. minor changes e.g.
	- #include <config.h> -> #include "config.h"
	- Add some no-install headers to /include
	- extern -> EXTERNL and vice versa as needed
	- misc header cleanup
	- clean up checking for misc. unix vs microsoft functions
13. Change copyright decls in some files to point to LICENSE file.
14. Add notes to RELEASENOTES.md

CMakeLists.txt

@@ -751,10 +751,19 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4)
 ENDIF(USE_HDF5 OR ENABLE_NETCDF_4)
 
 
-# Option to Build DAP Client
-OPTION(ENABLE_DAP "Enable DAP Client." ON)
+# Option to Build DAP2+DAP4 Clients
+OPTION(ENABLE_DAP "Enable DAP2 and DAP4 Client." ON)
+OPTION(ENABLE_DAP4 "Enable DAP4 Client." OFF) # Temporary
 IF(ENABLE_DAP)
   SET(USE_DAP ON)
+  SET(ENABLE_DAP2 ON)
+  IF(ENABLE_NETCDF_4)
+    IF(ENABLE_DAP4)
+      SET(ENABLE_DAP4 ON)
+    ENDIF()
+  ELSE()
+    SET(ENABLE_DAP4 OFF)
+  ENDIF()
   FIND_PACKAGE(CURL)
   IF(NOT CURL_LIBRARY)
     MESSAGE(FATAL_ERROR "DAP Support specified, CURL libraries are not found.")
@@ -793,8 +802,9 @@ IF(ENABLE_DAP)
   #include <curl/curl.h>
   int main() {int x = CURLOPT_CHUNK_BGN_FUNCTION;}" HAVE_CURLOPT_CHUNK_BGN_FUNCTION)
 
-
-
+ELSE()
+  SET(ENABLE_DAP2 OFF)
+  SET(ENABLE_DAP4 OFF)
 ENDIF()
 
 # Check to see if libtool supports
@@ -811,14 +821,10 @@ ENDIF()
 # Option to Enable DAP long tests, remote tests.
 OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
 OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
-
-IF(ENABLE_DAP_REMOTE_TESTS)
-  OPTION(ENABLE_DAP_AUTH_TESTS "Enable DAP remote authorization tests." OFF)
-ENDIF()
-
+SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu,jetstream.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 
 # If netCDF4 and DAP, Option for DAP groups.
-IF(ENABLE_NETCDF_4 AND USE_DAP)
+IF(ENABLE_NETCDF_4 AND ENABLE_DAP2)
   OPTION(ENABLE_DAP_GROUPS "Whether netcdf4 group names should be enabled." ON)
 ELSE()
   SET(ENABLE_DAP_GROUPS OFF CACHE BOOL "Whether netcdf4 group names should be enabled.")
@@ -1196,6 +1202,8 @@ SET(HAVE_ST_BLKSIZE ${HAVE_STRUCT_STAT_ST_BLKSIZE})
 
 # Type checks
 CHECK_TYPE_SIZE("void*"     SIZEOF_VOIDSTAR)
+# Alias for automake consistency
+SET(SIZEOF_VOIDP ${SIZEOF_VOIDSTAR})
 CHECK_TYPE_SIZE("char"      SIZEOF_CHAR)
 CHECK_TYPE_SIZE("double"    SIZEOF_DOUBLE)
 CHECK_TYPE_SIZE("float"     SIZEOF_FLOAT)
@@ -1313,6 +1321,7 @@ ENDIF(ENABLE_MMAP)
 ################################
 
 # Macro to append files to the EXTRA_DIST files.
+# Note: can only be used in subdirectories because of the use of PARENT_SCOPE
 SET(EXTRA_DIST "")
 MACRO(ADD_EXTRA_DIST files)
   FOREACH(F ${files})
@@ -1442,7 +1451,9 @@ MACRO(add_bin_test prefix F)
       PROPERTIES LINK_FLAGS_DEBUG " /NODEFAULTLIB:MSVCRT"
       )
   ENDIF()
-  ADD_TEST(${prefix}_${F} ${EXECUTABLE_OUTPUT_PATH}/${prefix}_${F})
+  ADD_TEST(${prefix}_${F}
+           ${EXECUTABLE_OUTPUT_PATH}/${prefix}_${F}
+           )
   IF(MSVC)
     SET_PROPERTY(TEST ${prefix}_${F} PROPERTY FOLDER "tests/")
     SET_TARGET_PROPERTIES(${prefix}_${F} PROPERTIES RUNTIME_OUTPUT_DIRECTORY
@@ -1462,7 +1473,8 @@ MACRO(print_conf_summary)
   MESSAGE("")
   MESSAGE(STATUS "Building Shared Libraries:     ${BUILD_SHARED_LIBS}")
   MESSAGE(STATUS "Building netCDF-4:             ${ENABLE_NETCDF_4}")
-  MESSAGE(STATUS "Building DAP Support:          ${ENABLE_DAP}")
+  MESSAGE(STATUS "Building DAP2 Support:         ${ENABLE_DAP2}")
+  MESSAGE(STATUS "Building DAP4 Support:         ${ENABLE_DAP4}")
   MESSAGE(STATUS "Building Utilities:            ${BUILD_UTILITIES}")
   IF(CMAKE_PREFIX_PATH)
     MESSAGE(STATUS "CMake Prefix Path:             ${CMAKE_PREFIX_PATH}")
@@ -1479,7 +1491,6 @@ MACRO(print_conf_summary)
   MESSAGE("Tests Enabled:              ${ENABLE_TESTS}")
   IF(ENABLE_TESTS)
     MESSAGE(STATUS "DAP Remote Tests:  ${ENABLE_DAP_REMOTE_TESTS}")
-    MESSAGE(STATUS "DAP AUTH Tests:    ${ENABLE_DAP_AUTH_TESTS}")
     MESSAGE(STATUS "Extra Tests:       ${ENABLE_EXTRA_TESTS}")
     MESSAGE(STATUS "Coverage Tests:    ${ENABLE_COVERAGE_TESTS}")
     MESSAGE(STATUS "Parallel Tests:    ${ENABLE_PARALLEL_TESTS}")
@@ -1615,11 +1626,15 @@ IF(USE_HDF5)
   add_subdirectory(libsrc4)
 ENDIF(USE_HDF5)
 
-IF(USE_DAP)
+IF(ENABLE_DAP2)
   ADD_SUBDIRECTORY(oc2)
   ADD_SUBDIRECTORY(libdap2)
 ENDIF()
 
+IF(ENABLE_DAP4)
+  ADD_SUBDIRECTORY(libdap4)
+ENDIF()
+
 add_subdirectory(liblib)
 
 # For tests and utilities, we are no longer
@@ -1647,10 +1662,12 @@ IF(ENABLE_TESTS)
     ADD_SUBDIRECTORY(nc_test4)
     ADD_SUBDIRECTORY(h5_test)
   ENDIF()
-  IF(USE_DAP AND ENABLE_DAP_REMOTE_TESTS)
+  IF(ENABLE_DAP2)
     ADD_SUBDIRECTORY(ncdap_test)
   ENDIF()
-
+  IF(ENABLE_DAP4)
+    ADD_SUBDIRECTORY(dap4_test)
+  ENDIF()
   IF(ENABLE_EXAMPLES)
     ADD_SUBDIRECTORY(examples)
   ENDIF()
@@ -1742,7 +1759,8 @@ configure_file(
 
 FILE(MAKE_DIRECTORY ${netCDF_BINARY_DIR}/tmp)
 configure_file("${netCDF_SOURCE_DIR}/nc-config.cmake.in"
-  "${netCDF_BINARY_DIR}/tmp/nc-config" @ONLY)
+  "${netCDF_BINARY_DIR}/tmp/nc-config" @ONLY
+  NEWLINE_STYLE LF)
 FILE(COPY "${netCDF_BINARY_DIR}/tmp/nc-config"
   DESTINATION ${netCDF_BINARY_DIR}/
   FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
@@ -1802,7 +1820,8 @@ is_enabled(USE_SZIP HAS_SZIP)
 is_enabled(STATUS_PNETCDF HAS_PNETCDF)
 is_enabled(STATUS_PARALLEL HAS_PARALLEL)
 is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4)
-is_enabled(USE_DAP HAS_DAP)
+is_enabled(ENABLE_DAP HAS_DAP)
+is_enabled(ENABLE_DAP4 HAS_DAP4)
 is_enabled(USE_DISKLESS HAS_DISKLESS)
 is_enabled(USE_MMAP HAS_MMAP)
 is_enabled(JNA HAS_JNA)
@@ -1839,6 +1858,14 @@ configure_file(
 FILE(COPY ${netCDF_SOURCE_DIR}/include/netcdf_meta.h
   DESTINATION ${netCDF_BINARY_DIR}/include/)
 
+#####
+# Build test_common.sh
+#####
+SET(EXTRA_DIST ${EXTRA_DIST} ${CMAKE_SOURCE_DIR}/test_common.in)
+SET(TOPSRCDIR "${CMAKE_SOURCE_DIR}")
+SET(TOPBUILDDIR "${CMAKE_BINARY_DIR}")
+configure_file(${CMAKE_SOURCE_DIR}/test_common.in ${CMAKE_BINARY_DIR}/test_common.sh @ONLY NEWLINE_STYLE LF)
+
 ####
 # Export files
 ####

COPYRIGHT

@@ -2,6 +2,7 @@ The NetCDF Copyright.
 
 Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
+2015, 2016
 University Corporation for Atmospheric Research/Unidata.
 
 Portions of this software were developed by the Unidata Program at the 

Make0

@@ -1,6 +1,7 @@
-T=t
+T=t_dap3a
 
 NCLIB=./liblib/.libs/libnetcdf.a
+D4LIB=./libdap4/.libs/libdap4.a
 
 LFLAG=-L/usr/local/lib
 
@@ -10,7 +11,7 @@ LFLAG=-L/usr/local/lib
 
 OTHERLIBS=-lm
 
-LIBS=${NCLIB} ${LFLAG} ${CURLLIB} ${HDF5LIB} ${ZLIB} ${OTHERLIBS}
+LIBS=${D4LIB} ${NCLIB} ${LFLAG} ${CURLLIB} ${HDF5LIB} ${ZLIB} ${OTHERLIBS}
 INCL=-I. -I./include
 RPATH=-Wl,-rpath,${LFLAG}
 
@@ -29,3 +30,11 @@ all::
 
 clean:
 	rm -f ${T}.o ${T}.exe
+
+
+EZXML=ezxml-0.8.6.tar.gz
+ezxml::
+	rm -fr ./ezxml ./ezxml.[ch]
+	tar -zxf ./${EZXML}
+	sed -e 's|//\(.*\)|/*\1*/|' <ezxml/ezxml.c >./ezxml.c
+	sed -e 's|//\(.*\)|/*\1*/|' <ezxml/ezxml.h >./ezxml.h

Makefile.am

@@ -15,7 +15,7 @@ EXTRA_DIST = README.md COPYRIGHT INSTALL.md test_prog.c \
 	FixBundle.cmake.in \
 	nc-config.cmake.in RELEASE_NOTES.md CTestCustom.cmake \
 	CTestConfig.cmake.in libnetcdf.settings.in netCDFConfig.cmake.in \
-	CMakeInstallation.cmake
+	CMakeInstallation.cmake test-driver-verbose test_common.in
 
 # Doxygen doesn't build nicely in vpath builds.
 # Don't do this; it wipes out any exported values
@@ -53,15 +53,18 @@ LIBSRC4_DIR = libsrc4
 NC_TEST4 = nc_test4
 endif
 
-# Build the opendap client?
-if BUILD_DAP
+# Build the dap2 client
+if ENABLE_DAP
 OCLIB = oc2
 DAP2 = libdap2
-#if BUILD_UTILITIES
-NCDAPTESTDIR = ncdap_test
-#endif # BUILD_UTILITIES
+NCDAP2TESTDIR = ncdap_test
 endif
 
+if ENABLE_DAP4
+DAP4 = libdap4
+NCDAP4TESTDIR = dap4_test
+endif #DAP4
+
 # Build pnetcdf
 if USE_PNETCDF
 LIBSRCP = libsrcp
@@ -79,21 +82,21 @@ endif
 
 # Define Test directories
 if BUILD_TESTSETS
-TESTDIRS = $(V2_TEST) nc_test $(NC_TEST4) $(NCDAPTESTDIR)
+TESTDIRS = $(V2_TEST) nc_test $(NC_TEST4) $(NCDAP2TESTDIR) $(NCDAP4TESTDIR)
 endif
 
 # This is the list of subdirs for which Makefiles will be constructed
 # and run. ncgen must come before ncdump, because their tests
 # depend on it.
-SUBDIRS = include $(OCLIB) $(H5_TEST_DIR) libdispatch libsrc		\
-$(LIBSRC4_DIR) $(DAP2) $(LIBCDMR) $(LIBSRCP) liblib  \
+SUBDIRS = include $(H5_TEST_DIR) libdispatch libsrc		\
+$(LIBSRC4_DIR) $(LIBSRCP) $(OCLIB) $(DAP2) ${DAP4} liblib  \
 $(NCGEN3) $(NCGEN) $(NCDUMP) \
 $(TESTDIRS) \
 docs $(EXAMPLES) \
 $(UDUNITS) $(LIBCF)
 
 # Remove these generated files, for a distclean.
-DISTCLEANFILES = VERSION comps.txt test_prog libnetcdf.settings
+DISTCLEANFILES = VERSION comps.txt test_prog libnetcdf.settings test_common.sh
 
 # The nc-config script helps the user build programs with netCDF.
 bin_SCRIPTS = nc-config
@@ -182,6 +185,14 @@ check_nc_config:
 	$(CC) `./nc-config --cflags` test_prog.c -o test_prog `./nc-config --libs`
 	./test_prog
 
+# Build nc_test/test_common.sh
+# Complicated by need to create during distcheck
+# which uses a read-only source tree
+#all-local:
+#	rm -f ${top_builddir}/nc_test/test_common.sh
+#	cat /dev/null > ${top_builddir}/nc_test/test_common.sh
+#	chmod a+x ${top_builddir}/nc_test/test_common.sh
+#	sed -e "s|@TOPSRCDIR@|${abs_top_srcdir}|" -e "s|@TOPBUILDDIR@|${abs_top_builddir}|" <${top_srcdir}/test_common.in >>${top_builddir}/test_common.sh
 
 install-data-hook:
 

RELEASE_NOTES.md

@@ -11,6 +11,14 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.4.2 - TBD
 
+* [Upgrade] The bash based test scripts have been upgraded to use a common test_common.sh include file that isolates build specific information.
+* [Enhancement] DAP4 is now included. Since dap2 is the default for urls, dap4 must be specified by
+(1) using "dap4:" as the url protocol, or
+(2) appending "#protocol=dap4" to the end of the url, or
+(3) appending "#dap4" to the end of the url 
+Note that dap4 is disabled by default until the testserver situation is resolved.
+* [Enhancement] The remote testing server can now be specified with the '--with-testserver" option to ./configure.
+* [Refactor] the oc2 library is no longer independent of the main netcdf-c library. For example, it now uses ncuri, nclist, and ncbytes instead of its homegrown equivalents.
 * [Upgrade] Update utf8proc.[ch] to use the version now maintained by the
 Julia Language project (https://github.com/JuliaLang/utf8proc/blob/master/LICENSE.md).
 * [Bug] Addressed conversion problem with Windows sscanf.  This primarily affected some OPeNDAP URLs on Windows.  See [GitHub #365](https://github.com/Unidata/netcdf-c/issues/365) and [GitHub #366](https://github.com/Unidata/netcdf-c/issues/366) for more information.

cf

@@ -2,9 +2,9 @@
 #NB=1
 DB=1
 #X=-x
-#FAST=1
+FAST=1
 
-#HDF5=1
+HDF5=1
 DAP=1
 #HDF4=1
 #PNETCDF=1
@@ -56,23 +56,6 @@ LDFLAGS=""
 
 CFLAGS="-g -O0 $CFLAGS"
 
-case "$HOST" in
-  mort)
-	CFLAGS="-std=c99 $CFLAGS"
-	;;
-  yakov)
-	CFLAGS="-std=c99 $CFLAGS"
-	;;
-  spock)
-  	  CFLAGS="-Wdeclaration-after-statement -Wall $CFLAGS"
-	;;
-  spike)
-	CFLAGS="-Wall $CFLAGS"
-	;;
-  *)
-	;;
-esac
-
 MAKE=make
 IGNORE="test 0 = 1"
 
@@ -86,7 +69,8 @@ if test "x$HDF4" = "x1" ; then
 LDFLAGS="$LDFLAGS -ljpeg"
 fi
 
-if test "x$DAP" = "x1" -o "x$CDMR" = "x1" -o "x$RPC" = "x1" ; then
+export PKG_CONFIG_PATH=/usr/lib/pkgconfig
+if test "x$DAP" = "x1" ; then
 if curl-config --version >/dev/null ; then
 TMP=`curl-config --cflags`
 CPPFLAGS="$TMP $CPPFLAGS"
@@ -95,8 +79,7 @@ LDFLAGS="$TMP $LDFLAGS"
 TMP=`curl-config --prefix`
 LD_LIBRARY_PATH="$TMP/lib:$LD_LIBRARY_PATH"
 else
-  echo "Cannot find curl-config"
-  exit 1
+  LDFLAGS="${LDFLAGS} -lcurl"  
 fi
 fi
 
@@ -111,11 +94,12 @@ FLAGS="$FLAGS --disable-examples"
 #FLAGS="$FLAGS --enable-dap-long-tests"
 #FLAGS="$FLAGS --enable-ffio"
 #FLAGS="$FLAGS --enable-benchmarks"
-#FLAGS="$FLAGS --enable-extra-tests"
+FLAGS="$FLAGS --enable-extreme-numbers"
+FLAGS="$FLAGS --enable-extra-tests"
 #FLAGS="$FLAGS --enable-large-file-tests"
 #FLAGS="$FLAGS --disable-testsets"
 #FLAGS="$FLAGS --disable-dap-remote-tests"
-FLAGS="$FLAGS --enable-dap-auth-tests"
+#FLAGS="$FLAGS --enable-dap-auth-tests" -- requires a new remotetest server
 #FLAGS="$FLAGS --enable-doxygen"
 FLAGS="$FLAGS --enable-logging"
 #FLAGS="$FLAGS --disable-diskless"
@@ -127,6 +111,8 @@ FLAGS="$FLAGS --enable-logging"
 #FLAGS="$FLAGS --enable-jna"
 #FLAGS="$FLAGS --disable-properties-attribute"
 #FLAGS="$FLAGS --disable-silent-rules"
+#FLAGS="$FLAGS --enable-dap4"
+#FLAGS="$FLAGS --with-testservers=remotestserver.localhost:8083"
 
 if test "x$PAR4" != x1 ; then
 FLAGS="$FLAGS --disable-parallel4"