diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 5f18aa1e0d..b9aaa5e13f 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -42,7 +42,7 @@ jobs: wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2 tar -jxf hdf-4.2.15.tar.bz2 pushd hdf-4.2.15 - ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib + ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib make -j make install -j popd @@ -164,7 +164,7 @@ jobs: - name: Configure shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ./configure --enable-hdf4 --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-doxygen --enable-external-server-tests if: ${{ success() }} - name: Look at config.log if error diff --git a/CMakeLists.txt b/CMakeLists.txt index 5dac91dd55..2db8dea02e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1092,9 +1092,12 @@ ENDIF() # Option to Enable DAP long tests, remote tests. OPTION(ENABLE_DAP_REMOTE_TESTS "Enable DAP remote tests." ON) +OPTION(ENABLE_EXTERNAL_SERVER_TESTS "Enable external Server remote tests." OFF) OPTION(ENABLE_DAP_LONG_TESTS "Enable DAP long tests." OFF) SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test") +SET(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test") + # See if we have zlib FIND_PACKAGE(ZLIB) @@ -1731,7 +1734,7 @@ ENDIF() # Set some of the options as advanced. MARK_AS_ADVANCED(ENABLE_INTERNAL_DOCS VALGRIND_TESTS ENABLE_COVERAGE_TESTS ) -MARK_AS_ADVANCED(ENABLE_DAP_REMOTE_TESTS ENABLE_DAP_LONG_TESTS USE_REMOTE_CDASH) +MARK_AS_ADVANCED(ENABLE_DAP_REMOTE_TESTS ENABLE_DAP_LONG_TESTS USE_REMOTE_CDASH ENABLE_EXTERNAL_SERVER_TESTS) MARK_AS_ADVANCED(ENABLE_DOXYGEN_BUILD_RELEASE_DOCS DOXYGEN_ENABLE_TASKS ENABLE_DOXYGEN_SERVER_SIDE_SEARCH) MARK_AS_ADVANCED(ENABLE_SHARED_LIBRARY_VERSION) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cdac66eadf..66e43a3adc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,12 +7,14 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.1 - T.B.D. -* [Bug Fix] Fix support for reading arrays of HDF5 fixed size strings. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* [Bug Fix] Split the remote tests into two parts: one for the remotetest server and one for all other external servers. Also add a configure option to enable the latter set. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* [Bug Fix] Fix support for reading arrays of HDF5 fixed size strings. See [Github #2462](https://github.com/Unidata/netcdf-c/pull/2466). * [Bug Fix] Provide a default enum const when fill value does not match any enum constant for the value zero. See [Github #2462](https://github.com/Unidata/netcdf-c/pull/2462). * [Bug Fix] Fix the json submodule symbol conflicts between libnetcdf and the plugin specific netcdf_json.h. See [Github #2448](https://github.com/Unidata/netcdf-c/pull/2448). * [Bug Fix] Fix quantize with CLASSIC_MODEL files. See [Github #2405](https://github.com/Unidata/netcdf-c/pull/2445). * [Enhancement] Add `--disable-quantize` option to `configure`. * [Bug Fix] Fix CMakeLists.txt to handle all acceptable boolean values for -DPLUGIN_INSTALL_DIR. See [Github #2430](https://github.com/Unidata/netcdf-c/pull/2430). +>>>>>>> master * [Enhancement] Provide a simple API to allow user access to the internal .rc file table: supports get/set/overwrite of entries of the form "key=value". See [Github #2408](https://github.com/Unidata/netcdf-c/pull/2408). * [Bug Fix] Use env variable USERPROFILE instead of HOME for windows and mingw. See [Github #2405](https://github.com/Unidata/netcdf-c/pull/2405). * [Bug Fix] Fix the nc_def_var_fletcher32 code in hdf5 to properly test value of the fletcher32 argument. See [Github #2403](https://github.com/Unidata/netcdf-c/pull/2403). @@ -27,8 +29,12 @@ This file contains a high-level description of this package's evolution. Release * [Enhancement] Allow the read/write of JSON-valued Zarr attributes to allow for domain specific info such as used by GDAL/Zarr. See [Github #2278](https://github.com/Unidata/netcdf-c/pull/2278). * [Enhancement] Turn on the XArray convention for NCZarr files by default. WARNING, this means that the mode should explicitly specify "nczarr" or "zarr" even if "xarray" or "noxarray" is specified. See [Github #2257](https://github.com/Unidata/netcdf-c/pull/2257). +<<<<<<< HEAD +* [Enhancement] Update the documentation to match the current filter capabilities See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249). +======= * [Enhancement] Update the documentation to match the current filter capabilities. See [Github #2249](https://github.com/Unidata/netcdf-c/pull/2249). +>>>>>>> master * [Enhancement] Support installation of pre-built standard filters into user-specified location. See [Github #2318](https://github.com/Unidata/netcdf-c/pull/2318). * [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #2245](https://github.com/Unidata/netcdf-c/pull/2245). * [Enhancement] Switch to tinyxml2 as the default xml parser implementation. See [Github #2170](https://github.com/Unidata/netcdf-c/pull/2170). diff --git a/configure.ac b/configure.ac index 1a1d7615e3..ff62d4b53f 100644 --- a/configure.ac +++ b/configure.ac @@ -604,31 +604,51 @@ AM_CONDITIONAL(ENABLE_QUANTIZE, [test x$enable_quantize = xyes]) # --enable-dap => enable-dap4 enable_dap4=$enable_dap -AC_MSG_CHECKING([whether dap remote testing should be enabled]) +AC_MSG_CHECKING([whether dap use of remotetest server should be enabled]) AC_ARG_ENABLE([dap-remote-tests], - [AS_HELP_STRING([--enable-dap-remote-tests], - [enable dap remote tests])]) -test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes + [AS_HELP_STRING([--disable-dap-remote-tests], + [disable dap remote tests])]) +test "x$enable_dap_remote_tests" = xyes || enable_dap_remote_tests=no if test "x$enable_dap" = "xno" ; then enable_dap_remote_tests=no fi AC_MSG_RESULT($enable_dap_remote_tests) +AC_MSG_CHECKING([whether dap use of remotetest server should be enabled]) +AC_ARG_ENABLE([dap-remote-tests], + [AS_HELP_STRING([--disable-dap-remote-tests], + [disable dap remote tests])]) +test "x$enable_dap_remote_tests" = xno || enable_dap_remote_tests=yes +AC_MSG_RESULT($enable_dap_remote_tests) + +AC_MSG_CHECKING([whether use of external servers should be enabled]) +AC_ARG_ENABLE([external-server-tests], + [AS_HELP_STRING([--enable-external-server-tests (default off)], + [enable external server tests])]) +test "x$enable_external_server_tests" = xyes || enable_external_server_tests=no +AC_MSG_RESULT($enable_external_server_tests) + +if test "x$enable_dap_remote_tests" = "xno" ; then + AC_MSG_NOTICE([--disable-dap_remote_tests => --disable-external-server-tests]) + enable_external_server_tests=no +fi + # Default is not to do the remote authorization tests. -AC_MSG_CHECKING([whether dap remote authorization testing should be enabled (default off)]) +AC_MSG_CHECKING([whether dap authorization testing should be enabled (default off)]) AC_ARG_ENABLE([dap-auth-tests], [AS_HELP_STRING([--enable-dap-auth-tests], [enable dap remote authorization tests])]) test "x$enable_dap_auth_tests" = xyes || enable_dap_auth_tests=no +AC_MSG_RESULT($enable_dap_auth_tests) + # dap must be enabled + if test "x$enable_dap" = "xno" ; then - enable_dap_auth_tests=no -fi -# if remote tests are disabled, then so is this -if test "x$enable_dap_remote_tests" = "xno" ; then + AC_MSG_NOTICE([--disable-dap => --disable-dap-remote-tests --disable-auth-tests --disable-external-server-tests]) enable_dap_remote_tests=no + enable_dap_auth_tests=no + enable_external_server_tests=no fi -AC_MSG_RESULT($enable_dap_auth_tests) # Did the user specify a list of test servers to try for remote tests? AC_MSG_CHECKING([which remote test server(s) to use]) @@ -653,16 +673,20 @@ fi if test "x$enable_dap_remote_tests" = xyes; then AC_DEFINE([ENABLE_DAP_REMOTE_TESTS], [1], [if true, do remote tests]) fi +if test "x$enable_external_server_tests" = xyes; then + AC_DEFINE([ENABLE_EXTERNAL_SERVER_TESTS], [1], [if true, do remote external tests]) +fi AC_MSG_CHECKING([whether the time-consuming dap tests should be enabled (default off)]) AC_ARG_ENABLE([dap-long-tests], [AS_HELP_STRING([--enable-dap-long-tests], [enable dap long tests])]) test "x$enable_dap_long_tests" = xyes || enable_dap_long_tests=no -if test "x$enable_dap_remote_tests" = "xno" ; then +AC_MSG_RESULT([$enable_dap_long_tests]) +if test "x$enable_dap_remote_tests" = "xno" || test "x$enable_external_server_tests" = "xno" ; then + AC_MSG_NOTICE([--disable-dap-remote|external-server-tests => --disable_dap_long_tests]) enable_dap_long_tests=no fi -AC_MSG_RESULT([$enable_dap_long_tests]) # Control zarr storage if test "x$enable_nczarr" = xyes ; then @@ -1758,6 +1782,7 @@ AM_CONDITIONAL(ENABLE_DAP4, [test "x$enable_dap4" = xyes]) AM_CONDITIONAL(USE_STRICT_NULL_BYTE_HEADER_PADDING, [test x$enable_strict_null_byte_header_padding = xyes]) AM_CONDITIONAL(ENABLE_CDF5, [test "x$enable_cdf5" = xyes]) AM_CONDITIONAL(ENABLE_DAP_REMOTE_TESTS, [test "x$enable_dap_remote_tests" = xyes]) +AM_CONDITIONAL(ENABLE_EXTERNAL_SERVER_TESTS, [test "x$enable_external_server_tests" = xyes]) AM_CONDITIONAL(ENABLE_DAP_AUTH_TESTS, [test "x$enable_dap_auth_tests" = xyes]) AM_CONDITIONAL(ENABLE_DAP_LONG_TESTS, [test "x$enable_dap_long_tests" = xyes]) AM_CONDITIONAL(USE_PNETCDF_DIR, [test ! "x$PNETCDFDIR" = x]) diff --git a/dap4_test/test_thredds.sh b/dap4_test/test_thredds.sh index 99cbcccc32..eae63d63ea 100755 --- a/dap4_test/test_thredds.sh +++ b/dap4_test/test_thredds.sh @@ -28,6 +28,7 @@ failure() { setresultdir results_test_thredds if test "x${RESET}" = x1 ; then rm -fr ${BASELINEH}/*.thredds ; fi +if test "x$FEATURE_THREDDSTEST" = x1 ; then for f in $F ; do makeurl "dap4://thredds-test.unidata.ucar.edu/thredds/dap4/casestudies" "$f" echo "testing: $URL" @@ -43,6 +44,7 @@ for f in $F ; do cp ./results_test_thredds/${base}.thredds ${BASELINETH}/${base}.thredds fi done +fi # FEATURE_THREDDSTEST echo "*** Pass" exit 0 diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index d991b9e0f5..74fd55a4fc 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -907,7 +907,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void if(!modelcomplete(model)) { const char** p = ncurifragmentparams(uri); /* envv format */ if(p != NULL) { - for(;*p;p++) { + for(;*p;p+=2) { const char* key = p[0]; const char* value = p[1];; if((stat=processfragmentkeys(key,value,model))) goto done; diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 823c83228b..b474b8c4b9 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -32,7 +32,7 @@ #define MAXTAGS 256 #define NCTAGDFALT "Log"; -#define NC_MAX_FRAMES 256 +#define NC_MAX_FRAMES 1024 static int nclogginginitialized = 0; diff --git a/nc_test/CMakeLists.txt b/nc_test/CMakeLists.txt index 96c7b6237a..61bff28074 100644 --- a/nc_test/CMakeLists.txt +++ b/nc_test/CMakeLists.txt @@ -107,8 +107,10 @@ IF(BUILD_UTILITIES) ENDIF() IF(ENABLE_BYTERANGE) + IF(ENABLE_EXTERNAL_SERVER_TESTS) build_bin_test_no_prefix(tst_byterange) add_sh_test(nc_test test_byterange) + ENDIF() ENDIF() IF(BUILD_MMAP) diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index c72bd23002..e7e7e5dfc0 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -75,26 +75,27 @@ endif TESTS = $(TESTPROGRAMS) -if ENABLE_BYTERANGE if BUILD_UTILITIES + +if ENABLE_BYTERANGE +if ENABLE_EXTERNAL_SERVER_TESTS tst_byterange_SOURCES = tst_byterange.c check_PROGRAMS += tst_byterange TESTS += test_byterange.sh endif endif -if BUILD_UTILITIES TESTS += run_diskless.sh run_diskless5.sh run_inmemory.sh if LARGE_FILE_TESTS if ! ENABLE_PARALLEL TESTS += run_diskless2.sh endif endif + if BUILD_MMAP TESTS += run_mmap.sh run_mmap.log: run_diskless.log endif -endif # If pnetcdf is enabled, these tests are run by a test # script. Otherwise, the are run by automake in the usual way. @@ -104,6 +105,8 @@ else TESTS += t_nc tst_atts3 tst_nofill nc_test tst_small endif +endif # BUILD_UTILITIES + # The .c files that are generated with m4 are already distributed, but # we also include the original m4 files, plus test scripts data. EXTRA_DIST = test_get.m4 test_put.m4 run_diskless.sh run_diskless2.sh \ diff --git a/nc_test/test_byterange.sh b/nc_test/test_byterange.sh index 28b5e6374d..276cdcd4f1 100755 --- a/nc_test/test_byterange.sh +++ b/nc_test/test_byterange.sh @@ -5,11 +5,13 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e + # Test Urls +if test "x$FEATURE_THREDDSTEST" = x1 ; then URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes&aws.profile=none" -#URL3a="https://remotetest.unidata.ucar.edu/thredds/fileServer/testdata/2004050300_eta_211.nc#bytes&aws.profile=none" -URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none" URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes&aws.profile=none" +fi +URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none" URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none" # Requires auth URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes&aws.profile=unidata" @@ -81,12 +83,16 @@ testsetup https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data echo "*** Testing reading NetCDF-3 file with http" -echo "***Test remote classic file" -testbytes nc3 classic "$URL3" +if test "x$FEATURE_THREDDSTEST" = x1 ; then + echo "***Test remote classic file" + testbytes nc3 classic "$URL3" +fi if test "x$FEATURE_HDF5" = xyes ; then - echo "***Test remote netcdf-4 file: non-s3" - testbytes nc4b netCDF-4 "$URL4b" + if test "x$FEATURE_THREDDSTEST" = x1 ; then + echo "***Test remote netcdf-4 file: non-s3" + testbytes nc4b netCDF-4 "$URL4b" + fi fi if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_HDF5" = xyes ; then echo "***Test remote netdf-4 file: s3" diff --git a/ncdap_test/CMakeLists.txt b/ncdap_test/CMakeLists.txt index 8c4f1af31e..ed07697e78 100644 --- a/ncdap_test/CMakeLists.txt +++ b/ncdap_test/CMakeLists.txt @@ -52,14 +52,14 @@ IF(ENABLE_TESTS) # not yet add_sh_test(ncdap tst_hyrax) add_sh_test(ncdap tst_fillmismatch) IF(ENABLE_DAP_LONG_TESTS) - add_sh_test(ncdap tst_longremote3) - add_bin_test(ncdap test_manyurls) - SET_TESTS_PROPERTIES(ncdap_tst_longremote3 ncdap_test_manyurls PROPERTIES RUN_SERIAL TRUE) + add_sh_test(ncdap tst_longremote3) + SET_TESTS_PROPERTIES(ncdap_tst_longremote3 PROPERTIES RUN_SERIAL TRUE) ENDIF(ENABLE_DAP_LONG_TESTS) - - - ENDIF(BUILD_UTILITIES) + IF(ENABLE_EXTERNAL_SERVER_TESTS) + add_bin_test(ncdap test_manyurls) + SET_TESTS_PROPERTIES(ncdap_test_manyurls PROPERTIES RUN_SERIAL TRUE) + ENDIF() IF(ENABLE_DAP_AUTH_TESTS) ##obsolete add_bin_test(ncdap t_auth) diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index 8cd39e51a9..3d045b5acd 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -52,10 +52,12 @@ TESTS += test_partvar if ENABLE_DAP_LONG_TESTS TESTS += tst_longremote3.sh tst_longremote3.log: tst_remote3.log +if ENABLE_EXTERNAL_SERVER_TESTS test_manyurls_SOURCES = test_manyurls.c manyurls.h check_PROGRAMS += test_manyurls - TESTS += test_manyurls test_manyurls.log: tst_longremote3.log + TESTS += test_manyurls +endif endif test_partvar_SOURCES = test_partvar.c diff --git a/test_common.in b/test_common.in index dc5be62e09..36412e02a3 100644 --- a/test_common.in +++ b/test_common.in @@ -32,6 +32,9 @@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@ FEATURE_FILTERTESTS=@DO_FILTER_TESTS@ FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@ +# Thredds-test server is currently disabled +#FEATURE_THREDDSTEST=1 + set -e # Figure out various locations in the src/build tree.