Add option to control accessing external servers

Add options "--enable-external-server-tests" (automake) "-DENABLE_EXTERNAL_SERVER_TESTS" (cmake) to enable/disable tests that access external URLS, where an external URL refers to some server not directly under the control of netcdf-c. This option defaults to off. The reason for this PR is that external servers can be up or down or change content without any warning to us, thus causing some of our tests to fail. Additionally. and recently, the thredds-test server was shut down and some of our tests used it. So these tests are disabled until a suitable replacement server is available. ## Misc. Fixes * Bump the depth of trace support in nclog.c to avoid overflow
Unidata · Aug 27, 2022 · 8774625 · 8774625
1 parent 9cc8831
commit 8774625
Show file tree

Hide file tree

Showing 13 changed files with 86 additions and 34 deletions.
diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml
@@ -42,7 +42,7 @@ jobs:
           wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2
           tar -jxf hdf-4.2.15.tar.bz2
           pushd hdf-4.2.15
-          ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib 
+          ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib
           make -j
           make install -j
           popd
@@ -164,7 +164,7 @@ jobs:
 
       - name: Configure
         shell: bash -l {0}
-        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests  --enable-doxygen
+        run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests  --enable-doxygen  --enable-external-server-tests
         if: ${{ success() }}
 
       - name: Look at config.log if error

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1092,9 +1092,12 @@ ENDIF()
 
 # Option to Enable DAP long tests, remote tests.
 OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON)
+OPTION(ENABLE_EXTERNAL_SERVER_TESTS "Enable external Server remote tests." OFF)
 OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF)
 SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
 
+SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")
+
 # See if we have zlib
 FIND_PACKAGE(ZLIB)
 
@@ -1731,7 +1734,7 @@ ENDIF()
 
 # Set some of the options as advanced.
 MARK_AS_ADVANCED(ENABLE_INTERNAL_DOCS VALGRIND_TESTS ENABLE_COVERAGE_TESTS )
-MARK_AS_ADVANCED(ENABLE_DAP_REMOTE_TESTS ENABLE_DAP_LONG_TESTS USE_REMOTE_CDASH)
+MARK_AS_ADVANCED(ENABLE_DAP_REMOTE_TESTS ENABLE_DAP_LONG_TESTS USE_REMOTE_CDASH ENABLE_EXTERNAL_SERVER_TESTS)
 MARK_AS_ADVANCED(ENABLE_DOXYGEN_BUILD_RELEASE_DOCS DOXYGEN_ENABLE_TASKS ENABLE_DOXYGEN_SERVER_SIDE_SEARCH)
 MARK_AS_ADVANCED(ENABLE_SHARED_LIBRARY_VERSION)
 

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -7,12 +7,14 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.9.1 - T.B.D.
 
-* [Bug Fix] Fix support for reading arrays of HDF5 fixed size strings. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
+* [Bug Fix] Split the remote tests into two parts: one for the remotetest server and one for all other external servers. Also add a configure option to enable the latter set. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????).
+* [Bug Fix] Fix support for reading arrays of HDF5 fixed size strings. See [Github #2462](https://github.com/Unidata/netcdf-c/pull/2466).
 * [Bug Fix] Provide a default enum const when fill value does not match any enum constant for the value zero. See [Github #2462](https://github.com/Unidata/netcdf-c/pull/2462).
 * [Bug Fix] Fix the json submodule symbol conflicts between libnetcdf and the plugin specific netcdf_json.h. See [Github #2448](https://github.com/Unidata/netcdf-c/pull/2448).
 * [Bug Fix] Fix quantize with CLASSIC_MODEL files. See [Github #2405](https://github.com/Unidata/netcdf-c/pull/2445).
 * [Enhancement] Add `--disable-quantize` option to `configure`.
 * [Bug Fix] Fix CMakeLists.txt to handle all acceptable boolean values for -DPLUGIN_INSTALL_DIR. See [Github #2430](https://github.com/Unidata/netcdf-c/pull/2430).
+>>>>>>> master
 * [Enhancement] Provide a simple API to allow user access to the internal .rc file table: supports get/set/overwrite of entries of the form "key=value". See [Github #2408](https://github.com/Unidata/netcdf-c/pull/2408).
 * [Bug Fix] Use env variable USERPROFILE instead of HOME for windows and mingw. See [Github #2405](https://github.com/Unidata/netcdf-c/pull/2405).
 * [Bug Fix] Fix the nc_def_var_fletcher32 code in hdf5 to properly test value of the fletcher32 argument. See [Github #2403](https://github.com/Unidata/netcdf-c/pull/2403).
@@ -27,8 +29,12 @@ This file contains a high-level description of this package's evolution. Release
 * [Enhancement] Allow the read/write of JSON-valued Zarr attributes to allow
 for domain specific info such as used by GDAL/Zarr. See [Github #2278](https://github.com/Unidata/netcdf-c/pull/2278).
 * [Enhancement] Turn on the XArray convention for NCZarr files by default. WARNING, this means that the mode should explicitly specify "nczarr" or "zarr" even if "xarray" or "noxarray" is specified. See [Github #2257](https://github.com/Unidata/netcdf-c/pull/2257).
+<<<<<<< HEAD
+* [Enhancement] Update the documentation to match the current filter capabilities  See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249).
+=======
 
 * [Enhancement] Update the documentation to match the current filter capabilities. See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249).
+>>>>>>> master
 * [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318).
 * [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245).
 * [Enhancement] Switch to tinyxml2 as the default xml parser implementation. See [Github #2170](https://github.com/Unidata/netcdf-c/pull/2170).

diff --git a/configure.ac b/configure.ac
@@ -604,31 +604,51 @@ AM_CONDITIONAL(ENABLE_QUANTIZE, [test x$enable_quantize = xyes])
 
 # --enable-dap => enable-dap4
 enable_dap4=$enable_dap
-AC_MSG_CHECKING([whether dap remote testing should be enabled])
+AC_MSG_CHECKING([whether dap use of remotetest server should be enabled])
 AC_ARG_ENABLE([dap-remote-tests],
-              [AS_HELP_STRING([--enable-dap-remote-tests],
-                                 [enable dap remote tests])])
-test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
+              [AS_HELP_STRING([--disable-dap-remote-tests],
+                                 [disable dap remote tests])])
+test "x$enable_dap_remote_tests" = xyes || enable_dap_remote_tests=no
 if test "x$enable_dap" = "xno" ; then
   enable_dap_remote_tests=no
 fi
 AC_MSG_RESULT($enable_dap_remote_tests)
 
+AC_MSG_CHECKING([whether dap use of remotetest server should be enabled])
+AC_ARG_ENABLE([dap-remote-tests],
+              [AS_HELP_STRING([--disable-dap-remote-tests],
+                                 [disable dap remote tests])])
+test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes
+AC_MSG_RESULT($enable_dap_remote_tests)
+
+AC_MSG_CHECKING([whether use of external servers should be enabled])
+AC_ARG_ENABLE([external-server-tests],
+              [AS_HELP_STRING([--enable-external-server-tests (default off)],
+                                 [enable external server tests])])
+test "x$enable_external_server_tests" = xyes || enable_external_server_tests=no
+AC_MSG_RESULT($enable_external_server_tests)
+
+if test "x$enable_dap_remote_tests" = "xno" ; then
+  AC_MSG_NOTICE([--disable-dap_remote_tests => --disable-external-server-tests])
+  enable_external_server_tests=no
+fi
+
 # Default is not to do the remote authorization tests.
-AC_MSG_CHECKING([whether dap remote authorization testing should be enabled (default off)])
+AC_MSG_CHECKING([whether dap authorization testing should be enabled (default off)])
 AC_ARG_ENABLE([dap-auth-tests],
               [AS_HELP_STRING([--enable-dap-auth-tests],
                                  [enable dap remote authorization tests])])
 test "x$enable_dap_auth_tests" = xyes || enable_dap_auth_tests=no
+AC_MSG_RESULT($enable_dap_auth_tests)
+
 # dap must be enabled
+
 if test "x$enable_dap" = "xno" ; then
-  enable_dap_auth_tests=no
-fi
-# if remote tests are disabled, then so is this
-if test "x$enable_dap_remote_tests" = "xno" ; then
+  AC_MSG_NOTICE([--disable-dap => --disable-dap-remote-tests --disable-auth-tests --disable-external-server-tests])
   enable_dap_remote_tests=no
+  enable_dap_auth_tests=no
+  enable_external_server_tests=no  
 fi
-AC_MSG_RESULT($enable_dap_auth_tests)
 
 # Did the user specify a list of test servers to try for remote tests?
 AC_MSG_CHECKING([which remote test server(s) to use])
@@ -653,16 +673,20 @@ fi
 if test "x$enable_dap_remote_tests" = xyes; then
    AC_DEFINE([ENABLE_DAP_REMOTE_TESTS], [1], [if true, do remote tests])
 fi
+if test "x$enable_external_server_tests" = xyes; then
+   AC_DEFINE([ENABLE_EXTERNAL_SERVER_TESTS], [1], [if true, do remote external tests])
+fi
 
 AC_MSG_CHECKING([whether the time-consuming dap tests should be enabled (default off)])
 AC_ARG_ENABLE([dap-long-tests],
               [AS_HELP_STRING([--enable-dap-long-tests],
                                  [enable dap long tests])])
 test "x$enable_dap_long_tests" = xyes || enable_dap_long_tests=no
-if test "x$enable_dap_remote_tests" = "xno" ; then
+AC_MSG_RESULT([$enable_dap_long_tests])
+if test "x$enable_dap_remote_tests" = "xno" || test "x$enable_external_server_tests" = "xno" ; then
+  AC_MSG_NOTICE([--disable-dap-remote|external-server-tests => --disable_dap_long_tests])
   enable_dap_long_tests=no
 fi
-AC_MSG_RESULT([$enable_dap_long_tests])
 
 # Control zarr storage
 if test "x$enable_nczarr" = xyes ; then
@@ -1758,6 +1782,7 @@ AM_CONDITIONAL(ENABLE_DAP4, [test "x$enable_dap4" = xyes])
 AM_CONDITIONAL(USE_STRICT_NULL_BYTE_HEADER_PADDING, [test x$enable_strict_null_byte_header_padding = xyes])
 AM_CONDITIONAL(ENABLE_CDF5, [test "x$enable_cdf5" = xyes])
 AM_CONDITIONAL(ENABLE_DAP_REMOTE_TESTS, [test "x$enable_dap_remote_tests" = xyes])
+AM_CONDITIONAL(ENABLE_EXTERNAL_SERVER_TESTS, [test "x$enable_external_server_tests" = xyes])
 AM_CONDITIONAL(ENABLE_DAP_AUTH_TESTS, [test "x$enable_dap_auth_tests" = xyes])
 AM_CONDITIONAL(ENABLE_DAP_LONG_TESTS, [test "x$enable_dap_long_tests" = xyes])
 AM_CONDITIONAL(USE_PNETCDF_DIR, [test ! "x$PNETCDFDIR" = x])

diff --git a/dap4_test/test_thredds.sh b/dap4_test/test_thredds.sh
@@ -28,6 +28,7 @@ failure() {
 setresultdir results_test_thredds
 
 if test "x${RESET}" = x1 ; then rm -fr ${BASELINEH}/*.thredds ; fi
+if test "x$FEATURE_THREDDSTEST" = x1 ; then
 for f in $F ; do
     makeurl "dap4://thredds-test.unidata.ucar.edu/thredds/dap4/casestudies" "$f"
     echo "testing: $URL"
@@ -43,6 +44,7 @@ for f in $F ; do
 	cp ./results_test_thredds/${base}.thredds ${BASELINETH}/${base}.thredds
     fi
 done
+fi # FEATURE_THREDDSTEST
 
 echo "*** Pass"
 exit 0

diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c
@@ -907,7 +907,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
 	if(!modelcomplete(model)) {
 	    const char** p = ncurifragmentparams(uri); /* envv format */
 	    if(p != NULL) {
-	        for(;*p;p++) {
+	        for(;*p;p+=2) {
 		    const char* key = p[0];
 		    const char* value = p[1];;
         	    if((stat=processfragmentkeys(key,value,model))) goto done;

diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c
@@ -32,7 +32,7 @@
 #define MAXTAGS 256
 #define NCTAGDFALT "Log";
 
-#define NC_MAX_FRAMES 256
+#define NC_MAX_FRAMES 1024
 
 static int nclogginginitialized = 0;
 

diff --git a/nc_test/CMakeLists.txt b/nc_test/CMakeLists.txt
@@ -107,8 +107,10 @@ IF(BUILD_UTILITIES)
     ENDIF()
 
     IF(ENABLE_BYTERANGE)
+      IF(ENABLE_EXTERNAL_SERVER_TESTS)
         build_bin_test_no_prefix(tst_byterange)
         add_sh_test(nc_test test_byterange)
+      ENDIF()
     ENDIF()
 
   IF(BUILD_MMAP)

diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am
@@ -75,26 +75,27 @@ endif
 
 TESTS = $(TESTPROGRAMS)
 
-if ENABLE_BYTERANGE
 if BUILD_UTILITIES
+
+if ENABLE_BYTERANGE
+if ENABLE_EXTERNAL_SERVER_TESTS
 tst_byterange_SOURCES = tst_byterange.c
 check_PROGRAMS += tst_byterange
 TESTS += test_byterange.sh
 endif
 endif
 
-if BUILD_UTILITIES
 TESTS += run_diskless.sh run_diskless5.sh run_inmemory.sh
 if LARGE_FILE_TESTS
 if ! ENABLE_PARALLEL
 TESTS += run_diskless2.sh
 endif
 endif
+
 if BUILD_MMAP
 TESTS += run_mmap.sh
 run_mmap.log: run_diskless.log
 endif
-endif
 
 # If pnetcdf is enabled, these tests are run by a test
 # script. Otherwise, the are run by automake in the usual way.
@@ -104,6 +105,8 @@ else
 TESTS += t_nc tst_atts3 tst_nofill nc_test tst_small
 endif
 
+endif # BUILD_UTILITIES
+
 # The .c files that are generated with m4 are already distributed, but
 # we also include the original m4 files, plus test scripts data.
 EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh	\

diff --git a/nc_test/test_byterange.sh b/nc_test/test_byterange.sh
@@ -5,11 +5,13 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
 
 set -e
 
+
 # Test Urls
+if test "x$FEATURE_THREDDSTEST" = x1 ; then
 URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes&aws.profile=none"
-#URL3a="https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes&aws.profile=none"
-URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
 URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes&aws.profile=none"
+fi
+URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
 URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none"
 # Requires auth
 URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes&aws.profile=unidata"
@@ -81,12 +83,16 @@ testsetup https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data
 
 echo "*** Testing reading NetCDF-3 file with http"
 
-echo "***Test remote classic file"
-testbytes nc3 classic "$URL3"
+if test "x$FEATURE_THREDDSTEST" = x1 ; then
+  echo "***Test remote classic file"
+  testbytes nc3 classic "$URL3"
+fi
 
 if test "x$FEATURE_HDF5" = xyes ; then
-    echo "***Test remote netcdf-4 file: non-s3"
-    testbytes nc4b netCDF-4 "$URL4b"
+    if test "x$FEATURE_THREDDSTEST" = x1 ; then
+	echo "***Test remote netcdf-4 file: non-s3"
+        testbytes nc4b netCDF-4 "$URL4b"
+    fi
 fi
 if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_HDF5" = xyes ; then
     echo "***Test remote netdf-4 file: s3"

diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt
@@ -52,14 +52,14 @@ IF(ENABLE_TESTS)
       # not yet      add_sh_test(ncdap tst_hyrax)
       add_sh_test(ncdap tst_fillmismatch)
       IF(ENABLE_DAP_LONG_TESTS)
-        add_sh_test(ncdap tst_longremote3)
-        add_bin_test(ncdap test_manyurls)
-        SET_TESTS_PROPERTIES(ncdap_tst_longremote3 ncdap_test_manyurls PROPERTIES RUN_SERIAL TRUE)
+        add_sh_test(ncdap tst_longremote3)	
+        SET_TESTS_PROPERTIES(ncdap_tst_longremote3 PROPERTIES RUN_SERIAL TRUE)
       ENDIF(ENABLE_DAP_LONG_TESTS)
-
-
-
     ENDIF(BUILD_UTILITIES)
+    IF(ENABLE_EXTERNAL_SERVER_TESTS)
+          add_bin_test(ncdap test_manyurls)
+          SET_TESTS_PROPERTIES(ncdap_test_manyurls PROPERTIES RUN_SERIAL TRUE)
+    ENDIF()
 
     IF(ENABLE_DAP_AUTH_TESTS)
 ##obsolete      add_bin_test(ncdap t_auth)

diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am
@@ -52,10 +52,12 @@ TESTS += test_partvar
 if ENABLE_DAP_LONG_TESTS
  TESTS += tst_longremote3.sh
  tst_longremote3.log: tst_remote3.log
+if ENABLE_EXTERNAL_SERVER_TESTS
  test_manyurls_SOURCES = test_manyurls.c manyurls.h
  check_PROGRAMS += test_manyurls
- TESTS += test_manyurls
  test_manyurls.log: tst_longremote3.log
+ TESTS += test_manyurls
+endif
 endif
 
 test_partvar_SOURCES = test_partvar.c

diff --git a/test_common.in b/test_common.in
@@ -32,6 +32,9 @@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
 FEATURE_FILTERTESTS=@DO_FILTER_TESTS@
 FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@
 
+# Thredds-test server is currently disabled
+#FEATURE_THREDDSTEST=1
+
 set -e
 
 # Figure out various locations in the src/build tree.