diff --git a/.clang-format b/.clang-format index 6fca7ffffb..64acd4ed81 100644 --- a/.clang-format +++ b/.clang-format @@ -2,4 +2,6 @@ Language: Cpp BasedOnStyle: LLVM DisableFormat: false +IndentWidth: 2 +ColumnLimit: 120 ... diff --git a/CMakeLists.txt b/CMakeLists.txt index f645cebab6..ba0b63646f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ # along with this program. If not, see http://www.gnu.org/licenses. # ----------------------------------------------------------------------------- -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.9) project(htm_core CXX) set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}") @@ -29,6 +29,7 @@ set($ENV{HTM_CPP} ${REPOSITORY_DIR}) # option(FORCE_CPP11 "Force compiler to use C++11 standard." OFF) option(FORCE_BOOST "Force compiler to install and use Boost." OFF) +set(YAML_PARSER "yamlcpp" CACHE STRING "Specify the YAML parser to use 'yamlcpp', 'RapidYaml', 'libyaml', default 'yamlcpp'." ) set(BINDING_BUILD "none" CACHE STRING "Specify the Binding to build 'Python2','Python3' or 'none', default 'none'." ) # Note: by setting the CXX environment variable, a non-default c++ compiler can be specified. @@ -77,6 +78,8 @@ message(STATUS "INTERNAL_LINKER_FLAGS_STR= ${INTERNAL_LINKER_FLAGS_STR}") message(STATUS "COMMON_COMPILER_DEFINITIONS_STR=${COMMON_COMPILER_DEFINITIONS_STR}") message(STATUS "COMMON_OS_LIBS = ${COMMON_OS_LIBS}") +add_definitions(-DYAML_PARSER_${YAML_PARSER}) + # Set up builds of external dependencies and get their exports. # (see individual external/*.cmake modules for exported settings and functions) include(external/bootstrap.cmake) diff --git a/bindings/py/cpp_src/plugin/PyBindRegion.cpp b/bindings/py/cpp_src/plugin/PyBindRegion.cpp index a227937d20..d9d836ddd9 100644 --- a/bindings/py/cpp_src/plugin/PyBindRegion.cpp +++ b/bindings/py/cpp_src/plugin/PyBindRegion.cpp @@ -67,56 +67,65 @@ namespace py = pybind11; } // switch } - static void prepareCreationParams(const ValueMap & vm, py::kwargs& kwargs) - { - ValueMap::const_iterator it; - for (it = vm.begin(); it != vm.end(); ++it) - { - try + // recurseive helper for prepareCreationParams + static py::object make_args(const Value& vm) { + if (vm.isScalar()) { + return py::str(vm.as()); + } + if (vm.isMap()) { + py::kwargs kw; + ValueMap::map_const_iterator it; + for (it = vm.begin(); it != vm.end(); ++it) { - auto key = it->first.c_str(); + std::string key = it->first.c_str(); + Value v = it->second; + kw[key.c_str()] = make_args(v); + } + return kw; + } + if (vm.isSequence()) { + auto a = py::list(); + for(size_t i = 0; i < vm.size(); i++) { + Value v = vm[i]; + a.append(make_args(v)); + } + return a; + } + throw Exception(__FILE__, __LINE__, "Not implemented."); + } - auto value = it->second; - if (value->isScalar()) - { - auto s = value->getScalar(); - switch (s->getType()) - { - case NTA_BasicType_Bool: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Byte: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Int16: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_UInt16: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Int32: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_UInt32: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Int64: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_UInt64: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Real32: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Real64: { kwargs[key] = s->getValue(); break; } - - default: - NTA_THROW << "Invalid type: " << s->getType(); - } - } - else if(value->isString()) - { - kwargs[key] = value->getString(); - } - else if (value->isArray()) - { - auto a = value->getArray(); - kwargs[key] = create_numpy_view(*a.get()); + // make kwargs from ValueMap. + static void prepareCreationParams(const ValueMap & vm, py::kwargs& kwargs) + { + if (vm.isMap()) { + ValueMap::map_const_iterator it; + for (it = vm.begin(); it != vm.end(); ++it) { + std::string key = it->first.c_str(); + try { + Value value = it->second; + kwargs[key.c_str()] = make_args(value); } - else - { - throw Exception(__FILE__, __LINE__, "Not implemented."); + catch (Exception& e) { + NTA_THROW << "Unable to create a Python object for parameter '" + << key << ": " << e.what(); } } + } + else if (vm.isScalar()) { + kwargs["arg"] = vm.as(); + } + else if (vm.isSequence()) { + try { + kwargs["array"] = make_args(vm); + } catch (Exception& e) { - NTA_THROW << "Unable to create a Python object for parameter '" - << it->first << ": " << e.what(); + NTA_THROW << "Unable to create a Python object for parameter 'array:' " << e.what(); } } + else { + NTA_THROW << "Unable to create a Python object for parameters."; + } }; PyBindRegion::PyBindRegion(const char * module, const ValueMap & nodeParams, Region * region, const char* className) @@ -184,10 +193,10 @@ namespace py = pybind11; args = py::make_tuple(node_, f, 2); // use type 2 protocol pickle.attr("dump")(*args); pickle.attr("close")(); - + // copy the pickle into the out string std::ifstream pfile(tmp_pickle.c_str(), std::ios::binary); - std::string content((std::istreambuf_iterator(pfile)), + std::string content((std::istreambuf_iterator(pfile)), std::istreambuf_iterator()); pfile.close(); Path::remove(tmp_pickle); @@ -206,7 +215,7 @@ namespace py = pybind11; // copy the extra data into the extra string std::ifstream efile(tmp_extra.c_str(), std::ios::binary); - std::string extra((std::istreambuf_iterator(efile)), + std::string extra((std::istreambuf_iterator(efile)), std::istreambuf_iterator()); efile.close(); Path::remove(tmp_extra); @@ -221,11 +230,11 @@ namespace py = pybind11; std::ofstream des; std::string tmp_pickle = "pickle.tmp"; - + std::ofstream pfile(tmp_pickle.c_str(), std::ios::binary); pfile.write(p.c_str(), p.size()); pfile.close(); - + // Tell Python to un-pickle using what is now in the pickle.tmp file. py::args args = py::make_tuple(tmp_pickle, "rb"); @@ -534,7 +543,7 @@ namespace py = pybind11; auto count = input["count"].cast(); bool required = false; - if (input.contains("required")) + if (input.contains("required")) { required = input["required"].cast(); } @@ -609,7 +618,7 @@ namespace py = pybind11; regionLevel = output["regionLevel"].cast(); } bool isDefaultOutput = false; - if (output.contains("isDefaultOutput")) + if (output.contains("isDefaultOutput")) { isDefaultOutput = output["isDefaultOutput"].cast(); } diff --git a/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp b/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp index 9fe9c658b5..0b03f2bc3d 100644 --- a/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp +++ b/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp @@ -65,6 +65,7 @@ #include #include +#include #include namespace py = pybind11; @@ -79,7 +80,6 @@ namespace htm class Spec; class PyRegionImpl; class Region; - class ValueMap; class RegisteredRegionImplPy: public RegisteredRegionImpl { diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index fde0d5c6e0..1bb1783a6d 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -38,7 +38,7 @@ # - cmake ../.. # Note: on MSVC, run startupMSVC.bat after deleting the build folder. -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.9) project(htm_core CXX) set(CMAKE_VERBOSE_MAKEFILE OFF) @@ -78,12 +78,20 @@ FILE(REMOVE "${EXPORT_FILE_NAME}") # common include(common.cmake) - ################ -# Yaml-cpp -include(YamlCppLib.cmake) - - +# Yaml Parser +if (${YAML_PARSER} STREQUAL "yamlcpp") + # Yaml-cpp + include(YamlCppLib.cmake) + +elseif(${YAML_PARSER} STREQUAL "RapidYaml") + # Rapid YAML + include(RapidYaml.cmake) + +elseif(${YAML_PARSER} STREQUAL "libyaml") + # libyaml + include(libyaml.cmake) +endif() ################ # Eigen include(eigen.cmake) diff --git a/external/RapidYaml.cmake b/external/RapidYaml.cmake new file mode 100644 index 0000000000..043a785916 --- /dev/null +++ b/external/RapidYaml.cmake @@ -0,0 +1,113 @@ +# ----------------------------------------------------------------------------- +# HTM Community Edition of NuPIC +# Copyright (C) 2019, Numenta, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see http://www.gnu.org/licenses. +# ----------------------------------------------------------------------------- + +# Rapid YAML. See https://github.com/biojppm/rapidyaml +# +# NOTE: Don't use this parser for now. It does not provide Exceptions correctly. +# On any error the program stops. +# David Keeney, 09/2019 +# +# Rapid YAML must be downloaded in four parts. +# - ryml from repository at https://github.com/biojppm/rapidyaml - the main library +# - c4core from repository at https://github.com/biojppm/c4core/ - the C library of utilities +# - cmake from repository at https://github.com/biojppm/cmake/ - Cmake scripts +# - debugbreak from repository at https://github.com/biojppm/debugbreak/ - debug break function +# There are links from rapidyaml to a specific branch of c4core in the extern folder of rapidyaml +# but the downloader does not follow it. So we need to do three downloads. + +message(STATUS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml.zip") +if( (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml.zip") + AND (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_c4core.zip") + AND (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_cmake.zip") + AND (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_debugbreak.zip")) + set(URL1 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml.zip") + set(URL2 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_c4core.zip") + set(URL3 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_cmake.zip") + set(URL4 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_debugbreak.zip") +else() + #There are no releases. Use the masters. + set(URL1 https://github.com/biojppm/rapidyaml/archive/master.zip) + set(URL2 https://github.com/biojppm/c4core/archive/master.zip) + set(URL3 https://github.com/biojppm/cmake/archive/master.zip) + set(URL4 https://github.com/biojppm/debugbreak/archive/master.zip) +endif() + +message(STATUS "Obtaining RapidYAML from ${URL1}" ) +include(DownloadProject/DownloadProject.cmake) +download_project(PROJ ryml + PREFIX "${EP_BASE}/ryml" + URL ${URL1} + UPDATE_DISCONNECTED 1 + QUIET + ) +download_project(PROJ ryml_c4core + PREFIX "${EP_BASE}/ryml_c4core" + SOURCE_DIR "${ryml_SOURCE_DIR}/extern/c4core" + URL ${URL2} + UPDATE_DISCONNECTED 1 + QUIET + ) +download_project(PROJ ryml_cmake + PREFIX "${EP_BASE}/ryml_cmake" + SOURCE_DIR "${ryml_SOURCE_DIR}/extern/c4core/cmake" + URL ${URL3} + UPDATE_DISCONNECTED 1 + QUIET + ) +download_project(PROJ ryml_debugbreak + PREFIX "${EP_BASE}/ryml_debugbreak" + SOURCE_DIR "${ryml_SOURCE_DIR}/extern/c4core/extern/debugbreak" + URL ${URL4} + UPDATE_DISCONNECTED 1 + QUIET + ) + +#Need to enable Exceptions +set(c4path "${ryml_SOURCE_DIR}/extern/c4core/src/c4") +file(RENAME "${c4path}/config.hpp" "${c4path}/config1.hpp") +file(READ "${c4path}/config1.hpp" content1) +string(REPLACE "//#define C4_ERROR_THROWS_EXCEPTION" + "#define C4_ERROR_EXCEPTIONS_ENABLED\n#define C4_ERROR_THROWS_EXCEPTION" + content + "${content1}") +file(WRITE "${c4path}/config.hpp" "${content}") + +file(RENAME "${c4path}/error.cpp" "${c4path}/error1.cpp") +file(READ "${c4path}/error1.cpp" content2) +string(REPLACE "if(s_error_flags & (ON_ERROR_LOG|ON_ERROR_CALLBACK))" + "if(s_error_flags & (ON_ERROR_LOG|ON_ERROR_CALLBACK|ON_ERROR_THROW))" + content1 + "${content2}") +string(REPLACE "throw Exception(ss.c_str())" + "throw Exception(buf)" + content + "${content1}") +file(WRITE "${c4path}/error.cpp" "${content}") + +set(C4_EXCEPTIONS_ENABLED ON) +set(C4_ERROR_THROWS_EXCEPTION ON) +add_subdirectory(${ryml_SOURCE_DIR} ${ryml_BINARY_DIR}) + +set(yaml_INCLUDE_DIRS "${ryml_SOURCE_DIR}/src@@@${ryml_SOURCE_DIR}/extern/c4core/src@@@${ryml_SOURCE_DIR}/extern/c4core/extern") +if (MSVC) + set(yaml_LIBRARIES "${ryml_BINARY_DIR}$<$:/Release/ryml.lib>$<$:/Debug/ryml.lib>") +else() + set(yaml_LIBRARIES ${ryml_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}ryml${CMAKE_STATIC_LIBRARY_SUFFIX}) +endif() +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_INCLUDE_DIRS@@@${yaml_INCLUDE_DIRS}\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_LIBRARIES@@@${yaml_LIBRARIES}\n") + diff --git a/external/YamlCppLib.cmake b/external/YamlCppLib.cmake index fa7d7e19bb..034c4d5085 100644 --- a/external/YamlCppLib.cmake +++ b/external/YamlCppLib.cmake @@ -39,12 +39,12 @@ set(YAML_CPP_BUILD_CONTRIB OFF CACHE BOOL "prevent contrib modules" FORCE) set(MSVC_SHARED_RT ON CACHE BOOL "Use compile option /MD rather than /MT" FORCE) add_subdirectory(${yaml-cpp_SOURCE_DIR} ${yaml-cpp_BINARY_DIR}) -set(yaml-cpp_INCLUDE_DIRS ${yaml-cpp_SOURCE_DIR}/include) +set(yaml_INCLUDE_DIRS ${yaml-cpp_SOURCE_DIR}/include) if (MSVC) - set(yaml-cpp_LIBRARIES "${yaml-cpp_BINARY_DIR}$<$:/Release/libyaml-cppmd.lib>$<$:/Debug/libyaml-cppmdd.lib>") + set(yaml_LIBRARIES "${yaml-cpp_BINARY_DIR}$<$:/Release/libyaml-cppmd.lib>$<$:/Debug/libyaml-cppmdd.lib>") else() - set(yaml-cpp_LIBRARIES ${yaml-cpp_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(yaml_LIBRARIES ${yaml-cpp_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}) endif() -FILE(APPEND "${EXPORT_FILE_NAME}" "yaml-cpp_INCLUDE_DIRS@@@${yaml-cpp_SOURCE_DIR}/include\n") -FILE(APPEND "${EXPORT_FILE_NAME}" "yaml-cpp_LIBRARIES@@@${yaml-cpp_LIBRARIES}\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_INCLUDE_DIRS@@@${yaml-cpp_SOURCE_DIR}/include\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_LIBRARIES@@@${yaml_LIBRARIES}\n") diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index f30cb3c9d3..b40223b55b 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -34,9 +34,10 @@ execute_process(COMMAND ${CMAKE_COMMAND} -D CMAKE_INSTALL_PREFIX=. -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} -D BINDING_BUILD:STRING=${BINDING_BUILD} - -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -D REPOSITORY_DIR=${REPOSITORY_DIR} - ../../external + -D YAML_PARSER:STRING=${YAML_PARSER} + -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -D REPOSITORY_DIR=${REPOSITORY_DIR} + ../../external WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty RESULT_VARIABLE result # OUTPUT_QUIET ### Disable this to debug external configuration @@ -91,7 +92,7 @@ FOREACH(line ${lines}) message(STATUS " ${name} = ${${name}}") ENDFOREACH() set(EXTERNAL_INCLUDES - ${yaml-cpp_INCLUDE_DIRS} + ${yaml_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${mnist_INCLUDE_DIRS} diff --git a/external/libYaml.cmake b/external/libYaml.cmake new file mode 100644 index 0000000000..c2ba260896 --- /dev/null +++ b/external/libYaml.cmake @@ -0,0 +1,54 @@ +# ----------------------------------------------------------------------------- +# Numenta Platform for Intelligent Computing (NuPIC) +# Copyright (C) 2019, Numenta, Inc. Unless you have purchased from +# Numenta, Inc. a separate commercial license for this software code, the +# following terms and conditions apply: +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see http://www.gnu.org/licenses. +# +# http://numenta.org/licenses/ +# ----------------------------------------------------------------------------- +# This downloads and builds the libyaml library. +# +# libyaml - see https://github.com/yaml/libyaml +# This is a SAX parser which means that it performs callbacks +# for each token it parses from the yaml text. Therefore +# the interface (Value-libyaml.cpp) must create the internal structure. +# +if(EXISTS ${REPOSITORY_DIR}/build/ThirdParty/share/libyaml.zip + set(URL ${REPOSITORY_DIR}/build/ThirdParty/share/libyaml.zip) +else() + set(URL "https://github.com/yaml/libyaml/archive/master.zip") +endif() + +message(STATUS "Obtaining libyaml") +include(DownloadProject/DownloadProject.cmake) +download_project(PROJ libyaml + PREFIX ${EP_BASE}/libyaml + URL ${URL} + GIT_SHALLOW ON + UPDATE_DISCONNECTED 1 + QUIET + ) + +add_subdirectory(${libyaml_SOURCE_DIR} ${libyaml_BINARY_DIR}) + +set(yaml_INCLUDE_DIRS ${libyaml_SOURCE_DIR}/include) +if (MSVC) + set(yaml_LIBRARIES "${libyaml_BINARY_DIR}$<$:/Release/liblibyamlmd.lib>$<$:/Debug/liblibyamlmdd.lib>") +else() + set(yaml_LIBRARIES ${libyaml_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}libyaml${CMAKE_STATIC_LIBRARY_SUFFIX}) +endif() +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_INCLUDE_DIRS@@@${yaml_SOURCE_DIR}\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_LIBRARIES@@@${yaml_LIBRARIES}\n") + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d59a01d2e9..733c7d12ae 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -112,8 +112,6 @@ set(engine_files htm/engine/RegisteredRegionImplCpp.hpp htm/engine/Spec.cpp htm/engine/Spec.hpp - htm/engine/YAMLUtils.cpp - htm/engine/YAMLUtils.hpp htm/engine/Watcher.cpp htm/engine/Watcher.hpp ) @@ -127,8 +125,6 @@ set(ntypes_files htm/ntypes/BasicType.hpp htm/ntypes/Collection.hpp htm/ntypes/Dimensions.hpp - htm/ntypes/Scalar.cpp - htm/ntypes/Scalar.hpp htm/ntypes/Value.cpp htm/ntypes/Value.hpp ) @@ -261,7 +257,7 @@ endif() # set(src_combined_htmcore_source_archives ${src_lib_static} - ${yaml-cpp_LIBRARIES} + ${yaml_LIBRARIES} ${Boost_LIBRARIES} ${common_LIBRARIES} ) @@ -281,7 +277,7 @@ else() # Note that these libraries must already exist (from depenancies) set(all_object_locations) set(src_externLibs - ${yaml-cpp_LIBRARIES} + ${yaml_LIBRARIES} ${Boost_LIBRARIES} ${common_LIBRARIES} ) diff --git a/src/htm/algorithms/SDRClassifier.cpp b/src/htm/algorithms/SDRClassifier.cpp index 9762361cd8..7c0bfc2255 100644 --- a/src/htm/algorithms/SDRClassifier.cpp +++ b/src/htm/algorithms/SDRClassifier.cpp @@ -129,7 +129,7 @@ void htm::softmax(PDF::iterator begin, PDF::iterator end) { *itr = std::exp(*itr - maxVal); // x[i] = e ^ (x[i] - maxVal) } // Sum of all elements raised to exp(elem) each. - const Real sum = (Real) std::accumulate(begin, end, 0.0f); + const Real sum = (Real) std::accumulate(begin, end, 0.0); NTA_ASSERT(sum > 0.0f); for (auto itr = begin; itr != end; ++itr) { *itr /= sum; diff --git a/src/htm/engine/RegionImpl.cpp b/src/htm/engine/RegionImpl.cpp index ec81e3708d..ff890b402c 100644 --- a/src/htm/engine/RegionImpl.cpp +++ b/src/htm/engine/RegionImpl.cpp @@ -252,5 +252,36 @@ Dimensions RegionImpl::getOutputDimensions(const std::string &name) const { return region_->getOutputDimensions(name); } +/** + * Checks the parameters in the ValueMap and gives an error if it + * is not consistant with the Spec. If a field in the Spec is not given + * in the ValueMap, insert it with its default value. + */ +void RegionImpl::ValidateParameters(ValueMap &vm) { + std::shared_ptr ns = region_->getSpec(); + + // Look for parameters that don't belong + for (auto p: vm) { + std::string key = p.first; + Value v = p.second; + if (key == "dim") + continue; + if (!ns->parameters.contains(key)) + NTA_THROW << "Parameter '" << key << "' is not expected for this Region."; + } + + // Look for missing parameters and apply their default value. + for (auto p : ns->parameters) { + std::string key = p.first; + ParameterSpec &ps = p.second; + if (vm.getString(key, "").length() == 0) { + // a missing or empty parameter. + vm[key] = ps.defaultValue; + } + } + +} + + } // namespace htm diff --git a/src/htm/engine/RegionImpl.hpp b/src/htm/engine/RegionImpl.hpp index 4ed7446b58..6fd1b17d64 100644 --- a/src/htm/engine/RegionImpl.hpp +++ b/src/htm/engine/RegionImpl.hpp @@ -140,6 +140,8 @@ #include #include #include +#include +#include namespace htm { @@ -294,8 +296,9 @@ class RegionImpl virtual void setDimensions(Dimensions dim) { dim_ = std::move(dim); } virtual Dimensions getDimensions() const { return dim_; } + virtual void ValidateParameters(ValueMap &vm); -protected: + protected: // A pointer to the Region object. This is the portion visible // to the applications. This class and it's subclasses are the // hidden implementations behind the Region class. diff --git a/src/htm/engine/RegionImplFactory.cpp b/src/htm/engine/RegionImplFactory.cpp index 2be4085688..a697852504 100644 --- a/src/htm/engine/RegionImplFactory.cpp +++ b/src/htm/engine/RegionImplFactory.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -112,9 +111,8 @@ RegionImpl *RegionImplFactory::createRegionImpl(const std::string nodeType, Region *region) { RegionImpl *impl = nullptr; - std::shared_ptr& ns = getSpec(nodeType); - ValueMap vm = YAMLUtils::toValueMap(nodeParams.c_str(), ns->parameters, - nodeType, region->getName()); + ValueMap vm; + vm.parse(nodeParams); if (regionTypeMap.find(nodeType) != regionTypeMap.end()) { impl = regionTypeMap[nodeType]->createRegionImpl(vm, region); @@ -124,9 +122,8 @@ RegionImpl *RegionImplFactory::createRegionImpl(const std::string nodeType, // If the parameter 'dim' was defined, parse that out as a global parameter. if (vm.contains("dim")) { - std::shared_ptr dim = vm.getArray("dim"); - Dimensions d(dim->asVector()); - impl->setDimensions(d); + std::vector dim = vm["dim"].asVector(); + impl->setDimensions(dim); } return impl; diff --git a/src/htm/engine/RegionImplFactory.hpp b/src/htm/engine/RegionImplFactory.hpp index ecb6723338..def7bfe80b 100644 --- a/src/htm/engine/RegionImplFactory.hpp +++ b/src/htm/engine/RegionImplFactory.hpp @@ -41,7 +41,7 @@ namespace htm { class RegionImpl; class Region; class Spec; -class ValueMap; +class Value; class RegisteredRegionImpl; class RegionImplFactory { diff --git a/src/htm/engine/RegisteredRegionImpl.hpp b/src/htm/engine/RegisteredRegionImpl.hpp index 7f8f7ebbbd..0bf418fdf7 100644 --- a/src/htm/engine/RegisteredRegionImpl.hpp +++ b/src/htm/engine/RegisteredRegionImpl.hpp @@ -58,6 +58,7 @@ #define NTA_REGISTERED_REGION_IMPL_HPP #include +#include namespace htm { @@ -65,7 +66,6 @@ namespace htm class ArWrapper; class RegionImpl; class Region; - class ValueMap; class RegisteredRegionImpl { public: diff --git a/src/htm/engine/RegisteredRegionImplCpp.hpp b/src/htm/engine/RegisteredRegionImplCpp.hpp index 698f582d9a..3dd0cd46e3 100644 --- a/src/htm/engine/RegisteredRegionImplCpp.hpp +++ b/src/htm/engine/RegisteredRegionImplCpp.hpp @@ -59,7 +59,6 @@ namespace htm class ArWrapper; class RegionImpl; class Region; - class ValueMap; template diff --git a/src/htm/engine/YAMLUtils.cpp b/src/htm/engine/YAMLUtils.cpp deleted file mode 100644 index feb4e65e95..0000000000 --- a/src/htm/engine/YAMLUtils.cpp +++ /dev/null @@ -1,306 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -#include -#include -#include -#include -#include -#include // strlen -#include - -#include - -namespace htm { -namespace YAMLUtils { - -/* - * These functions are used internally by toValue and toValueMap - */ -static void _toScalar(const YAML::Node& node, std::shared_ptr& s); -static void _toArray(const YAML::Node& node, std::shared_ptr& a); -static Value toValue(const YAML::Node& node, NTA_BasicType dataType); - -static void _toScalar(const YAML::Node &node, std::shared_ptr &s) { - NTA_CHECK(node.Type() == YAML::NodeType::Scalar); - switch (s->getType()) { - case NTA_BasicType_Byte: - // We should have already detected this and gone down the string path - NTA_THROW << "Internal error: attempting to convert YAML string to scalar of type Byte"; - break; - case NTA_BasicType_UInt16: - s->value.uint16 = node.as(); - break; - case NTA_BasicType_Int16: - s->value.int16 = node.as(); - break; - case NTA_BasicType_UInt32: - s->value.uint32 = node.as(); - break; - case NTA_BasicType_Int32: - s->value.int32 = node.as(); - break; - case NTA_BasicType_UInt64: - s->value.uint64 = node.as(); - break; - case NTA_BasicType_Int64: - s->value.int64 = node.as(); - break; - case NTA_BasicType_Real32: - s->value.real32 = node.as(); - break; - case NTA_BasicType_Real64: - s->value.real64 = node.as(); - break; - case NTA_BasicType_Bool: - s->value.boolean = node.as(); - break; - case NTA_BasicType_Handle: - NTA_THROW << "Attempt to specify a YAML value for a scalar of type Handle"; - break; - default: - // should not happen - const std::string val = node.as(); - NTA_THROW << "Unknown data type " << s->getType() << " for yaml node '" << val << "'"; - } -} - -static void _toArray(const YAML::Node& node, std::shared_ptr& a) { - NTA_CHECK(node.Type() == YAML::NodeType::Sequence); - - a->allocateBuffer(node.size()); - void *buffer = a->getBuffer(); - - for (size_t i = 0; i < node.size(); i++) { - const YAML::Node &item = node[i]; - NTA_CHECK(item.Type() == YAML::NodeType::Scalar); - switch (a->getType()) { - case NTA_BasicType_Byte: - // We should have already detected this and gone down the string path - NTA_THROW << "Internal error: attempting to convert YAML string to array " - "of type Byte"; - break; - case NTA_BasicType_UInt16: - ((UInt16*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Int16: - ((Int16*)buffer)[i] = item.as(); - break; - case NTA_BasicType_UInt32: - ((UInt32*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Int32: - ((Int32*)buffer)[i] = item.as(); - break; - case NTA_BasicType_UInt64: - ((UInt64*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Int64: - ((Int64*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Real32: - ((Real32*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Real64: - ((Real64*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Bool: - ((bool*)buffer)[i] = item.as(); - break; - default: - // should not happen - NTA_THROW << "Unknown data type " << a->getType(); - } - } -} - -static Value toValue(const YAML::Node &node, NTA_BasicType dataType) { - if (node.Type() == YAML::NodeType::Map || - node.Type() == YAML::NodeType::Null) { - NTA_THROW << "YAML string does not represent a value."; - } - if (node.Type() == YAML::NodeType::Scalar) { - if (dataType == NTA_BasicType_Byte) { - // node >> *str; - const std::string val = node.as(); - Value v(val); - return v; - } else { - std::shared_ptr s(new Scalar(dataType)); - _toScalar(node, s); - Value v(s); - return v; - } - } else { - // array - std::shared_ptr a(new Array(dataType)); - _toArray(node, a); - Value v(a); - return v; - } -} - -/* - * For converting default values specified in nodespec string - */ -Value toValue(const std::string& yamlstring, NTA_BasicType dataType) -{ - // TODO -- return value? exceptions? - const YAML::Node doc = YAML::Load(yamlstring); - return toValue(doc, dataType); -} - -/* - * For converting param specs for Regions and LinkPolicies - */ -ValueMap toValueMap(const char *yamlstring, - Collection ¶meters, - const std::string &nodeType, - const std::string ®ionName) { - - ValueMap vm; - - // special values that applies to all regions. - // NOTE: these values are used in RegionImplFactory to set region impl. - ParameterSpec dim_spec("Buffer dimensions for region's global dimensions. " - "Syntax: {dim: [2,3]}", // description - NTA_BasicType_UInt32, - 0, // elementCount (an array of unknown size) - "", // constraints - "", // defaultValue - ParameterSpec::ReadWriteAccess); - - - std::string paddedstring(yamlstring); - // TODO: strip white space to determine if empty - bool empty = (paddedstring.size() == 0); - - // TODO: utf-8 compatible? - const YAML::Node doc = YAML::Load(paddedstring); - if(!empty) { - // A ValueMap is specified as a dictionary - if (doc.Type() != YAML::NodeType::Map) { - std::string ys(yamlstring); - if (ys.size() > 30) { - ys = ys.substr(0, 30) + "..."; - } - NTA_THROW - << "YAML string '" << ys - << "' does not not specify a dictionary of key-value pairs. " - << "Region and Link parameters must be specified as a dictionary"; - } - } - // Grab each value out of the YAML dictionary and put into the ValueMap - // if it is allowed by the nodespec. - for (auto i = doc.begin(); i != doc.end(); i++) - { - ParameterSpec ps; - - const auto key = i->first.as(); - if (key == "dim") - ps = dim_spec; - else { - if (!parameters.contains(key)) - { - std::stringstream ss; - for (UInt j = 0; j < parameters.getCount(); j++){ - ss << " " << parameters.getByIndex(j).first << "\n"; - } - - if (nodeType == std::string("")) { - NTA_THROW << "Unknown parameter '" << key << "'\n" - << "Valid parameters are:\n" << ss.str(); - } else { - NTA_CHECK(regionName != std::string("")); - NTA_THROW << "Unknown parameter '" << key << "' for region '" - << regionName << "' of type '" << nodeType << "'\n" - << "Valid parameters are:\n" - << ss.str(); - } - } - ps = parameters.getByName(key); // makes a copy of ParameterSpec - } - if (vm.contains(key)) - NTA_THROW << "Parameter '" << key << "' specified more than once in YAML document"; - try - { - if (ps.accessMode == ParameterSpec::ReadOnlyAccess) { - NTA_THROW << "Parameter '" << key << "'. This is ReadOnly access. Cannot be set."; - } - Value v = toValue(i->second, ps.dataType); - if (v.isScalar() && ps.count != 1) - { - NTA_THROW << "Parameter '" << key << "'. Bad value in runtime parameters. Expected array value but got scalar value"; - } - if (!v.isScalar() && ps.count == 1) - { - NTA_THROW << "Parameter '" << key << "'. Bad value in runtime parameters. Expected scalar value but got array value"; - } - vm.add(key, v); - } catch (std::runtime_error &e) { - NTA_THROW << "Unable to set parameter '" << key << "'. " << e.what(); - } - } //end for - - // Populate ValueMap with default values if they were not specified in the YAML dictionary. - for (size_t i = 0; i < parameters.getCount(); i++) - { - const std::pair& item = parameters.getByIndex(i); - if (!vm.contains(item.first)) - { - const ParameterSpec & ps = item.second; - if (ps.defaultValue != "") - { - // TODO: This check should be uncommented after dropping NuPIC 1.x nodes (which don't comply) //FIXME try this - // if (ps.accessMode != ParameterSpec::CreateAccess) - // { - // NTA_THROW << "Default value for non-create parameter: " << item.first; - // } - - try { -#ifdef YAMLDEBUG - NTA_DEBUG << "Adding default value '" << ps.defaultValue - << "' to parameter " << item.first << " of type " - << BasicType::getName(ps.dataType) << " count " << ps.count; -#endif - // NOTE: this can handle both scalers and arrays - // Arrays MUST be in Yaml sequence format even if one element. - // i.e. [1,2,3] - Value v = toValue(ps.defaultValue, ps.dataType); - if (v.isScalar() && ps.count != 1) - { - NTA_THROW << "Parameter '" << item.first << "'. Bad default value in spec. Expected array value but got scalar value"; - } - if (!v.isScalar() && ps.count == 1) - { - NTA_THROW << "Parameter '" << item.first << "'. Bad default value in spec. Expected scalar value but got array value"; - } - vm.add(item.first, v); - } catch (...) { - NTA_THROW << "Unable to set default value for item '" << item.first - << "' of datatype " << BasicType::getName(ps.dataType) - << " with value '" << ps.defaultValue << "'"; - } - } - } - } - - return vm; -} - -} // end of YAMLUtils namespace -} // end of namespace htm diff --git a/src/htm/engine/YAMLUtils.hpp b/src/htm/engine/YAMLUtils.hpp deleted file mode 100644 index bb5dad7838..0000000000 --- a/src/htm/engine/YAMLUtils.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ -#ifndef NTA_YAML_HPP -#define NTA_YAML_HPP - -#include -#include -#include -#include - -namespace htm -{ - - namespace YAMLUtils - { - /* - * For converting default values - * @param yamlstring - is a string in parsable format (eg. '[1 2 3]' ), not a path to yaml file! - */ - Value toValue(const std::string& yamlstring, NTA_BasicType dataType); - -/* - * For converting param specs for Regions and LinkPolicies - */ -ValueMap toValueMap(const char *yamlstring, - Collection ¶meters, - const std::string &nodeType = "", - const std::string ®ionName = ""); - -} // namespace YAMLUtils -} // namespace htm - -#endif // NTA_YAML_HPP diff --git a/src/htm/ntypes/Scalar.cpp b/src/htm/ntypes/Scalar.cpp deleted file mode 100644 index f4144cba22..0000000000 --- a/src/htm/ntypes/Scalar.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -/** @file - * Implementation of the Scalar class - * - * A Scalar object is an instance of an NTA_BasicType -- essentially a union - * It is used internally in the conversion of YAML strings to C++ objects. - */ - -#include -#include - -using namespace htm; - -Scalar::Scalar(NTA_BasicType theTypeParam) { - theType_ = theTypeParam; - value.uint64 = 0; -} - -NTA_BasicType Scalar::getType() { return theType_; } - -// gcc 4.2 complains about the template specializations -// in a different namespace if we don't include this -namespace htm { - -template <> Handle Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Handle); - return value.handle; -} -template <> Byte Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Byte); - return value.byte; -} -template <> UInt16 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_UInt16); - return value.uint16; -} -template <> Int16 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Int16); - return value.int16; -} -template <> UInt32 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_UInt32); - return value.uint32; -} -template <> Int32 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Int32); - return value.int32; -} -template <> UInt64 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_UInt64); - return value.uint64; -} -template <> Int64 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Int64); - return value.int64; -} -template <> Real32 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Real32); - return value.real32; -} -template <> Real64 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Real64); - return value.real64; -} -template <> bool Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Bool); - return value.boolean; -} -} // namespace htm diff --git a/src/htm/ntypes/Scalar.hpp b/src/htm/ntypes/Scalar.hpp deleted file mode 100644 index 183dc053b6..0000000000 --- a/src/htm/ntypes/Scalar.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -/** @file - * Definitions for the Scalar class - * - * A Scalar object is an instance of an NTA_BasicType -- essentially a union - * It is used internally in the conversion of YAML strings to C++ objects. - */ - -#ifndef NTA_SCALAR_HPP -#define NTA_SCALAR_HPP - -#include -#include // temporary, while implementation is in hpp -#include - -namespace htm { -class Scalar { -public: - Scalar(NTA_BasicType theTypeParam); - - NTA_BasicType getType(); - - template T getValue() const; - - union { - Handle handle; - Byte byte; - Int16 int16; - UInt16 uint16; - Int32 int32; - UInt32 uint32; - Int64 int64; - UInt64 uint64; - Real32 real32; - Real64 real64; - bool boolean; - } value; - -private: - NTA_BasicType theType_; -}; - -} // namespace htm - -#endif // NTA_SCALAR_HPP diff --git a/src/htm/ntypes/Value-RapidYaml.cpp b/src/htm/ntypes/Value-RapidYaml.cpp new file mode 100644 index 0000000000..2b7db0791c --- /dev/null +++ b/src/htm/ntypes/Value-RapidYaml.cpp @@ -0,0 +1,368 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2019, Numenta, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ + +#ifdef YAML_PARSER_RapidYaml + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace htm; +using namespace ryml; +using namespace c4; + +// some static helper functions +// The convert to and from the special string types used by ryml +static csubstr _S(const std::string &s) { return to_csubstr(s.c_str()); } +static std::string S_(csubstr s) { return std::string(s.data(), s.size()); } + +// Error handling. This C function bootstraps into C++ to throw an Exception. +static void handleErrors_c(const char *msg, size_t msg_len, void *user_data) { + ((ValueMap *)user_data)->handleErrors(std::string(msg, msg_len)); +} + +// This is just so ryml.hpp declarations are hidden from our .hpp +struct htm::ValueMap::OpaqueTree { + c4::yml::NodeRef node; + c4::yml::Tree t; +}; +struct htm::ValueMap::iterator::OpaqueIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + ValueMap vm; +}; +struct htm::ValueMap::const_iterator::OpaqueConstIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + const ValueMap vm; +}; + + + + +// Constructor +ValueMap::ValueMap() { + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root +} + +// Parse YAML or JSON string document into the tree. +ValueMap &ValueMap::parse(const std::string &yaml_string) { + set_error_flags(ON_ERROR_LOG|ON_ERROR_THROW); + + // This must be the top node so set the callbacks on this. + //Callbacks const &cb_orig = get_callbacks(); + //Callbacks cb_new = cb_orig; + //cb_new.m_error = handleErrors_c; + //cb_new.m_user_data = this; + //set_callbacks(cb_new); + + // Clear the tree, just in case there has been something already there + // from a previous parse. + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root + + ryml::parse(_S(yaml_string), doc_->node); // populate tree at this node + + std::cout << "Tree: " << doc_->t; + + // If the value parsed is just a scalar, then the top node is set to a sequence + // and the first child is the scalar. If a sequence was explictly specified with [ ] + // or a leading '- ' and there is only one element we cannot tell the difference. + // So, we have to see what the yaml_string starts with. If it is a '[' or '- ' then leave + // it as a Sequence. If not, point to the single scalar child as the top node. + if (doc_->node.is_seq() && doc_->node.num_children() == 1) { + const char *p = yaml_string.c_str(); + while (isspace(*p)) p++; // skip over leading spaces + if (*p == '[' || (*p == '-' && *(p + 1) == ' ')) { + // leave top node as a sequence + } else { + // make the top node a Scaler + doc_->node = doc_->node.first_child(); + } + } + return *this; +} + + +// checking content of a parameter +//enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category ValueMap::getCategory() const { + if (doc_->node.is_map()) return ValueMap::Category::Map; + if (doc_->node.is_seq()) return ValueMap::Category::Sequence; + if (doc_->node.has_val()) return ValueMap::Category::Scalar; + return ValueMap::Category::Empty; +} + +bool ValueMap::contains(const std::string &key) const {return doc_->node[_S(key)].valid();} + +bool ValueMap::isScalar() const { return doc_->node.has_val(); } +bool ValueMap::isSequence() const { return doc_->node.is_seq(); } +bool ValueMap::isMap() const { return doc_->node.is_map(); } +bool ValueMap::isEmpty() const { return getCategory() == ValueMap::Category::Empty; } + +size_t ValueMap::size() const { return doc_->node.num_children(); } + +// Accessing members of a map +ValueMap ValueMap::operator[](const std::string &key) { + ValueMap vm = *this; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} +const ValueMap ValueMap::operator[](const std::string &key) const { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_map()) << "This is not a map."; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} + +// accessing members of a sequence +ValueMap ValueMap::operator[](size_t index) { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_seq()) << "This is not a sequence."; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '"+std::to_string(index)+"' is out of bounds."; + } + return vm; +} +const ValueMap ValueMap::operator[](size_t index) const { + ValueMap vm = *this; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '" + std::to_string(index) + "' is out of bounds."; + } + return vm; +} + +std::string ValueMap::str() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + return S_(r.val()); + } else { + NTA_THROW << "Value not found."; + } +} + +// Return a value converted to the specified type T. +template T ValueMap::as() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + T v; + r >> v; + return v; + } else { + NTA_THROW << "Value not found."; + } +} +// explicit instantiation. Can only be used with these types. +template int8_t ValueMap::as() const; +template int16_t ValueMap::as() const; +template uint16_t ValueMap::as() const; +template int32_t ValueMap::as() const; +template uint32_t ValueMap::as() const; +template int64_t ValueMap::as() const; +template uint64_t ValueMap::as() const; +template bool ValueMap::as() const; +template float ValueMap::as() const; +template double ValueMap::as() const; +template std::string ValueMap::as() const; + + +std::string ValueMap::key() const { + NodeRef r = doc_->node; + return (r.has_key()) ? S_(r.key()) : ""; +} + + +// Iterator +ValueMap::iterator ValueMap::iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last)? NodeRef(): ptr_->node.next_sibling(); + return *this; +} +ValueMap::iterator ValueMap::iterator::operator++(int junk) {return operator++(); } +ValueMap &ValueMap::iterator::operator*() { ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +ValueMap *ValueMap::iterator::operator->() { ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::iterator::operator==(const iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::iterator::operator!=(const iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::iterator ValueMap::begin() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } + else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} +ValueMap::iterator ValueMap::end() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + +ValueMap::const_iterator ValueMap::const_iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last) ? NodeRef() : ptr_->node.next_sibling(); + return *this; +} +ValueMap::const_iterator ValueMap::const_iterator::operator++(int junk) { return operator++();} +const ValueMap &ValueMap::const_iterator::operator*() {ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +const ValueMap *ValueMap::const_iterator::operator->() {ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::const_iterator::operator==(const const_iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::const_iterator::operator!=(const const_iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::const_iterator ValueMap::cbegin() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} + +ValueMap::const_iterator ValueMap::cend() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + + +std::vector ValueMap::getKeys() const { + std::vector v; + NodeRef root = doc_->t.rootref(); + for (NodeRef n : root.children()) { + if (n.has_key()) + v.push_back(S_(n.key())); + } + return v; +} + + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } + } +} + + +static void to_json(std::ostream& f, const htm::ValueMap& vm) { + bool first = true; + switch (vm.getCategory()) { + case ValueMap::Empty: + return; + case ValueMap::Scalar: + f << '"'; + escape_json(f, vm.as()); + f << '"'; + break; + case ValueMap::Sequence: + f << "["; + for (size_t i = 0; i < vm.size(); i++) { + if (!first) + f << ", "; + first = false; + to_json(f, vm[i]); + } + f << "]"; + break; + case ValueMap::Map: + f << "{"; + for (auto it = vm.cbegin(); it != vm.cend(); ++it) { + if (!first) + f << ", "; + first = false; + f << it->key() << ": "; + to_json(f, *it); + } + f << "}"; + break; + } +} + +std::string ValueMap::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); +} + +std::string ValueMap::to_yaml() const { + std::stringstream ss; + ss << doc_->t; + return ss.str(); +} + + +std::ostream &operator<<(std::ostream &f, const htm::ValueMap &vm) { + f << vm.to_json(); + return f; +} + +void ValueMap::handleErrors(const std::string &msg) +{ + throw htm::LoggingException(__FILE__, __LINE__) << "Yaml parse error: " + msg; +} + +#endif // YAML_PARSER_RapidYaml \ No newline at end of file diff --git a/src/htm/ntypes/Value-libyaml.cpp b/src/htm/ntypes/Value-libyaml.cpp new file mode 100644 index 0000000000..d90a9590a4 --- /dev/null +++ b/src/htm/ntypes/Value-libyaml.cpp @@ -0,0 +1,368 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2013, Numenta, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ + +#ifdef YAML_PARSER_libyaml + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace htm; +using namespace ryml; +using namespace c4; + +// some static helper functions +// The convert to and from the special string types used by ryml +static csubstr _S(const std::string &s) { return to_csubstr(s.c_str()); } +static std::string S_(csubstr s) { return std::string(s.data(), s.size()); } + +// Error handling. This C function bootstraps into C++ to throw an Exception. +static void handleErrors_c(const char *msg, size_t msg_len, void *user_data) { + ((ValueMap *)user_data)->handleErrors(std::string(msg, msg_len)); +} + +// This is just so ryml.hpp declarations are hidden from our .hpp +struct htm::ValueMap::OpaqueTree { + c4::yml::NodeRef node; + c4::yml::Tree t; +}; +struct htm::ValueMap::iterator::OpaqueIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + ValueMap vm; +}; +struct htm::ValueMap::const_iterator::OpaqueConstIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + const ValueMap vm; +}; + + + + +// Constructor +ValueMap::ValueMap() { + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root +} + +// Parse YAML or JSON string document into the tree. +ValueMap &ValueMap::parse(const std::string &yaml_string) { + set_error_flags(ON_ERROR_LOG|ON_ERROR_THROW); + + // This must be the top node so set the callbacks on this. + //Callbacks const &cb_orig = get_callbacks(); + //Callbacks cb_new = cb_orig; + //cb_new.m_error = handleErrors_c; + //cb_new.m_user_data = this; + //set_callbacks(cb_new); + + // Clear the tree, just in case there has been something already there + // from a previous parse. + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root + + ryml::parse(_S(yaml_string), doc_->node); // populate tree at this node + + std::cout << "Tree: " << doc_->t; + + // If the value parsed is just a scalar, then the top node is set to a sequence + // and the first child is the scalar. If a sequence was explictly specified with [ ] + // or a leading '- ' and there is only one element we cannot tell the difference. + // So, we have to see what the yaml_string starts with. If it is a '[' or '- ' then leave + // it as a Sequence. If not, point to the single scalar child as the top node. + if (doc_->node.is_seq() && doc_->node.num_children() == 1) { + const char *p = yaml_string.c_str(); + while (isspace(*p)) p++; // skip over leading spaces + if (*p == '[' || (*p == '-' && *(p + 1) == ' ')) { + // leave top node as a sequence + } else { + // make the top node a Scaler + doc_->node = doc_->node.first_child(); + } + } + return *this; +} + + +// checking content of a parameter +//enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category ValueMap::getCategory() const { + if (doc_->node.is_map()) return ValueMap::Category::Map; + if (doc_->node.is_seq()) return ValueMap::Category::Sequence; + if (doc_->node.has_val()) return ValueMap::Category::Scalar; + return ValueMap::Category::Empty; +} + +bool ValueMap::contains(const std::string &key) const {return doc_->node[_S(key)].valid();} + +bool ValueMap::isScalar() const { return doc_->node.has_val(); } +bool ValueMap::isSequence() const { return doc_->node.is_seq(); } +bool ValueMap::isMap() const { return doc_->node.is_map(); } +bool ValueMap::isEmpty() const { return getCategory() == ValueMap::Category::Empty; } + +size_t ValueMap::size() const { return doc_->node.num_children(); } + +// Accessing members of a map +ValueMap ValueMap::operator[](const std::string &key) { + ValueMap vm = *this; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} +const ValueMap ValueMap::operator[](const std::string &key) const { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_map()) << "This is not a map."; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} + +// accessing members of a sequence +ValueMap ValueMap::operator[](size_t index) { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_seq()) << "This is not a sequence."; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '"+std::to_string(index)+"' is out of bounds."; + } + return vm; +} +const ValueMap ValueMap::operator[](size_t index) const { + ValueMap vm = *this; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '" + std::to_string(index) + "' is out of bounds."; + } + return vm; +} + +std::string ValueMap::str() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + return S_(r.val()); + } else { + NTA_THROW << "Value not found."; + } +} + +// Return a value converted to the specified type T. +template T ValueMap::as() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + T v; + r >> v; + return v; + } else { + NTA_THROW << "Value not found."; + } +} +// explicit instantiation. Can only be used with these types. +template int8_t ValueMap::as() const; +template int16_t ValueMap::as() const; +template uint16_t ValueMap::as() const; +template int32_t ValueMap::as() const; +template uint32_t ValueMap::as() const; +template int64_t ValueMap::as() const; +template uint64_t ValueMap::as() const; +template bool ValueMap::as() const; +template float ValueMap::as() const; +template double ValueMap::as() const; +template std::string ValueMap::as() const; + + +std::string ValueMap::key() const { + NodeRef r = doc_->node; + return (r.has_key()) ? S_(r.key()) : ""; +} + + +// Iterator +ValueMap::iterator ValueMap::iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last)? NodeRef(): ptr_->node.next_sibling(); + return *this; +} +ValueMap::iterator ValueMap::iterator::operator++(int junk) {return operator++(); } +ValueMap &ValueMap::iterator::operator*() { ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +ValueMap *ValueMap::iterator::operator->() { ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::iterator::operator==(const iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::iterator::operator!=(const iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::iterator ValueMap::begin() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } + else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} +ValueMap::iterator ValueMap::end() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + +ValueMap::const_iterator ValueMap::const_iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last) ? NodeRef() : ptr_->node.next_sibling(); + return *this; +} +ValueMap::const_iterator ValueMap::const_iterator::operator++(int junk) { return operator++();} +const ValueMap &ValueMap::const_iterator::operator*() {ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +const ValueMap *ValueMap::const_iterator::operator->() {ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::const_iterator::operator==(const const_iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::const_iterator::operator!=(const const_iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::const_iterator ValueMap::cbegin() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} + +ValueMap::const_iterator ValueMap::cend() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + + +std::vector ValueMap::getKeys() const { + std::vector v; + NodeRef root = doc_->t.rootref(); + for (NodeRef n : root.children()) { + if (n.has_key()) + v.push_back(S_(n.key())); + } + return v; +} + + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } + } +} + + +static void to_json(std::ostream& f, const htm::ValueMap& vm) { + bool first = true; + switch (vm.getCategory()) { + case ValueMap::Empty: + return; + case ValueMap::Scalar: + f << '"'; + escape_json(f, vm.as()); + f << '"'; + break; + case ValueMap::Sequence: + f << "["; + for (size_t i = 0; i < vm.size(); i++) { + if (!first) + f << ", "; + first = false; + to_json(f, vm[i]); + } + f << "]"; + break; + case ValueMap::Map: + f << "{"; + for (auto it = vm.cbegin(); it != vm.cend(); ++it) { + if (!first) + f << ", "; + first = false; + f << it->key() << ": "; + to_json(f, *it); + } + f << "}"; + break; + } +} + +std::string ValueMap::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); +} + +std::string ValueMap::to_yaml() const { + std::stringstream ss; + ss << doc_->t; + return ss.str(); +} + + +std::ostream &operator<<(std::ostream &f, const htm::ValueMap &vm) { + f << vm.to_json(); + return f; +} + +void ValueMap::handleErrors(const std::string &msg) +{ + throw htm::LoggingException(__FILE__, __LINE__) << "Yaml parse error: " + msg; +} + +#endif // YAML_PARSER_libyaml diff --git a/src/htm/ntypes/Value-yamlcpp.cpp b/src/htm/ntypes/Value-yamlcpp.cpp new file mode 100644 index 0000000000..35b8339484 --- /dev/null +++ b/src/htm/ntypes/Value-yamlcpp.cpp @@ -0,0 +1,375 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2019, Numenta, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ +#ifdef YAML_PARSER_yamlcpp + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace htm; + +// This is just so yaml-cpp/yaml.h declarations are hidden from our .hpp +struct htm::Value::OpaqueTree { + YAML::Node node; +}; +struct htm::Value::map_iterator::OpaqueIterator { + YAML::iterator it; + std::pair current_pair; + Value current_node; +}; +struct htm::Value::map_const_iterator::OpaqueConstIterator { + YAML::const_iterator it; + std::pair current_pair; + Value current_node; +}; + +// Constructor +Value::Value() { + doc_ = std::make_shared(); // creates an empty Tree +} + +// Parse YAML or JSON string document into the tree. +Value &Value::parse(const std::string &yaml_string) { + doc_->node = YAML::Load(yaml_string); + return *this; +} + +// checking content of a parameter +// enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category Value::getCategory() const { + if (doc_->node.IsMap()) + return ValueMap::Category::Map; + if (doc_->node.IsSequence()) + return ValueMap::Category::Sequence; + if (doc_->node.IsScalar()) + return ValueMap::Category::Scalar; + return ValueMap::Category::Empty; +} + + +bool Value::contains(const std::string &key) const { return (doc_->node[key]); } + +bool Value::isScalar() const { return doc_->node.IsScalar(); } +bool Value::isSequence() const { return doc_->node.IsSequence(); } +bool Value::isMap() const { return doc_->node.IsMap(); } +bool Value::isEmpty() const { return getCategory() == Value::Category::Empty; } + +size_t Value::size() const { return doc_->node.size(); } + +// Accessing members of a map +// If not found, a Zombie Value object is returned. +// An error will be displayed when you try to access the value in the Zombie Value object. +// If you assign something to the Zombie Value, it will insert it into the tree with the saved key. +Value Value::operator[](const std::string &key) { + Value v; + v.doc_->node = doc_->node[key]; + return v; +} +const Value Value::operator[](const std::string &key) const { + Value v; + v.doc_->node = doc_->node[key]; + return v; +} + + +// accessing members of a sequence +Value Value::operator[](size_t index) { + Value v; + v.doc_->node = doc_->node[index]; + return v; +} +const Value Value::operator[](size_t index) const { + Value v; + v.doc_->node = doc_->node[index]; + return v; +} + +std::string Value::str() const { + return doc_->node.as(); +} + +// Assign a value converted from a specified type T. +void Value::operator=(char *val) { doc_->node = val; } +void Value::operator=(const std::string&val) { doc_->node = val; } +void Value::operator=(int8_t val) { doc_->node = val; } +void Value::operator=(int16_t val) { doc_->node = val; } +void Value::operator=(uint16_t val) { doc_->node = val; } +void Value::operator=(int32_t val) { doc_->node = val; } +void Value::operator=(uint32_t val) { doc_->node = val; } +void Value::operator=(int64_t val) { doc_->node = val; } +void Value::operator=(uint64_t val) { doc_->node = val; } +void Value::operator=(bool val) { doc_->node = val; } +void Value::operator=(float val) { doc_->node = val; } +void Value::operator=(double val) { doc_->node = val; } +void Value::operator=(std::vector val) { doc_->node = val; } + +// Compare two nodes recursively to see if content is same. +// yaml-cpp does equals by compairing pointers so we have to do our own. +static bool equals(const YAML::Node &n1, const YAML::Node &n2) { + if (n1.IsScalar() && n2.IsScalar() && n1.as() == n2.as()) + return true; + if (n1.IsSequence() && n2.IsSequence() && n1.size() == n2.size()) { + for (size_t i = 0; i < n1.size(); i++) + if (!equals(n1[i], n2[i])) return false; + return true; + } + if (n1.IsMap() && n2.IsMap() && n1.size() == n2.size()) { + for (auto it : n1) { + if (!n2[it.first.as()]) + return false; + if (!equals(it.second, n2[it.first.as()])) + return false; + } + return true; + } + return false; +} +bool Value::operator==(const Value &v) { return equals(v.doc_->node,doc_->node); } + +// Return a value converted to the specified type T. +template T Value::as() const { + return doc_->node.as(); +} +// explicit instantiation. Can only be used with these types. +template int8_t Value::as() const; +template int16_t Value::as() const; +template uint16_t Value::as() const; +template int32_t Value::as() const; +template uint32_t Value::as() const; +template int64_t Value::as() const; +template uint64_t Value::as() const; +template float Value::as() const; +template double Value::as() const; +template std::string Value::as() const; +template <> bool Value::as() const { std::string val = doc_->node.as(); + transform(val.begin(), val.end(), val.begin(), ::tolower); + if (val == "true" || val == "on" || val == "1") + return true; + if (val == "false" || val == "off" || val == "0") + return false; + NTA_THROW << "Invalid value for a boolean. " << val; +} + + +template T Value::as(T default_value) const { return doc_->node.as(default_value); } +// explicit instantiation. Can only be used with these types. +template int8_t Value::as(int8_t d) const; +template int16_t Value::as(int16_t d) const; +template uint16_t Value::as(uint16_t d) const; +template int32_t Value::as(int32_t d) const; +template uint32_t Value::as(uint32_t d) const; +template int64_t Value::as(int64_t d) const; +template uint64_t Value::as(uint64_t d) const; +template float Value::as(float d) const; +template double Value::as(double d) const; +template std::string Value::as(std::string d) const; +template <> bool Value::as(bool d) const { + std::string val = doc_->node.as(d?"true":"false"); + transform(val.begin(), val.end(), val.begin(), ::tolower); + if (val == "true" || val == "on" || val == "1") + return true; + if (val == "false" || val == "off" || val == "0") + return false; + NTA_THROW << "Invalid value for a boolean. " << val; +} + +// Map Iterator +Value::map_iterator Value::begin() { + NTA_CHECK(!isEmpty()) << "Not found"; + Value::map_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.begin(); + return itr; +} +Value::map_iterator Value::end() { + NTA_CHECK(!isEmpty()) << "Not found"; + Value::map_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.end(); + return itr; +} +Value::map_iterator Value::map_iterator::operator++() { + ptr_->it++; + return *this; +} +Value::map_iterator Value::map_iterator::operator++(int junk) { return operator++(); } + +Value &Value::map_iterator::operator*() { + ptr_->current_node.doc_->node.reset(ptr_->it->second); + return ptr_->current_node; +} +std::pair *Value::map_iterator::operator->() { + ptr_->current_node.doc_->node.reset(ptr_->it->second); + std::pair p(ptr_->it->first.as(), ptr_->current_node); + ptr_->current_pair = p; + return &ptr_->current_pair; +} +bool Value::map_iterator::operator==(const Value::map_iterator &rhs) const { return ptr_->it == rhs.ptr_->it; } +bool Value::map_iterator::operator!=(const Value::map_iterator &rhs) const { return ptr_->it != rhs.ptr_->it; } + + +// Const Iterator +Value::map_const_iterator Value::cbegin() const { + Value::map_const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.begin(); + return itr; +} + +Value::map_const_iterator Value::cend() const { + Value::map_const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.end(); + return itr; +} + +Value::map_const_iterator Value::map_const_iterator::operator++() { + ptr_->it++; + return *this; +} +Value::map_const_iterator Value::map_const_iterator::operator++(int junk) { return operator++(); } + +const Value &Value::map_const_iterator::operator*() { + ptr_->current_node.doc_->node.reset( ptr_->it->second ); + return ptr_->current_node; +} +const std::pair *Value::map_const_iterator::operator->() { + ptr_->current_node.doc_->node.reset(ptr_->it->second); + ptr_->current_pair.first = ptr_->it->first.as(); + ptr_->current_pair.second = ptr_->current_node; + return &ptr_->current_pair; +} + +bool Value::map_const_iterator::operator==(const Value::map_const_iterator &rhs) const { + return ptr_->it == rhs.ptr_->it; +} +bool Value::map_const_iterator::operator!=(const Value::map_const_iterator &rhs) const { + return ptr_->it != rhs.ptr_->it; +} + + + + +std::vector Value::getKeys() const { + NTA_CHECK(isEmpty()) << "Not found"; + NTA_CHECK(isMap()) << "This is not a map."; + std::vector v; + for (auto it = doc_->node.begin(); it != doc_->node.end(); it++) { + v.push_back(it->first.as()); + } + return v; +} + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } + } +} + +static void to_json(std::ostream &f, const htm::Value &v) { + bool first = true; + switch (v.getCategory()) { + case Value::Empty: + return; + case Value::Scalar: + f << '"'; + escape_json(f, v.as()); + f << '"'; + break; + case Value::Sequence: + f << "["; + for (size_t i = 0; i < v.size(); i++) { + if (!first) + f << ", "; + first = false; + Value n = v[i]; + to_json(f, n); + } + f << "]"; + break; + case Value::Map: + f << "{"; + for (auto it = v.cbegin(); it != v.cend(); it++) { + if (!first) + f << ", "; + first = false; + f << it->first << ": "; + Value n = *it; + to_json(f, n); + } + f << "}"; + break; + } +} + +std::string Value::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); +} + +std::string Value::to_yaml() const { + std::stringstream ss; + ss << doc_->node; + return ss.str(); +} + +namespace htm { +std::ostream &operator<<(std::ostream &f, const htm::Value &v) { + f << v.to_json(); + return f; +} +} // htm namespace + +#endif // YAML_PARSER_yamlcpp diff --git a/src/htm/ntypes/Value.cpp b/src/htm/ntypes/Value.cpp index 9975c3da88..0fb5d522f0 100644 --- a/src/htm/ntypes/Value.cpp +++ b/src/htm/ntypes/Value.cpp @@ -1,6 +1,9 @@ /* --------------------------------------------------------------------- * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. + * Copyright (C) 2019, Numenta, Inc. + * + * Author: David Keeney, 10/2019 + * dkeeney@gmail.com * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero Public License version 3 as @@ -15,271 +18,493 @@ * along with this program. If not, see http://www.gnu.org/licenses. * --------------------------------------------------------------------- */ -/** @file - * Implementation of the Value class - */ - +#include #include #include -using namespace htm; +#include +#include +#include +#include +#include -Value::Value(std::shared_ptr &s) { - category_ = scalarCategory; - scalar_ = s; -} +using namespace htm; -Value::Value(std::shared_ptr &a) { - category_ = arrayCategory; - array_ = a; -} +#define ZOMBIE_MAP ((size_t)-1) // Means the key of zombie was a map key +#define ZOMBIE_SEQ 0 // Means the key of zombie was a seq key -Value::Value(const std::string& s) { - category_ = stringCategory; - string_ = s; -} +////////////////////////////////////////////////////////////// +#ifdef YAML_PARSER_yamlcpp +#include -bool Value::isScalar() const { return category_ == scalarCategory; } +// Parse YAML or JSON string document into the tree. +Value &Value::parse(const std::string &yaml_string) { + // If this Value node is being re-used (like in unit tests) + // we need to clear variables. + vec_.clear(); + map_.clear(); + scalar_ = ""; + type_ = Value::Category::Empty; + parent_ = nullptr; -bool Value::isArray() const { return category_ == arrayCategory; } + YAML::Node node = YAML::Load(yaml_string); + // walk the tree and copy data into our structure -bool Value::isString() const { return category_ == stringCategory; } + setNode(&node); + return *this; +} -NTA_BasicType Value::getType() const { - switch (category_) { - case scalarCategory: - return scalar_->getType(); - break; - case arrayCategory: - return array_->getType(); - break; - default: - // string - return NTA_BasicType_Byte; - break; +void Value::setNode(void *n) { + YAML::Node &node = *((YAML::Node *)n); + std::pair::iterator, bool> ret; + bool first = true; + if (node.IsScalar()) { + type_ = Value::Scalar; + scalar_ = node.as(); + } else if (node.IsSequence()) { + type_ = Value::Sequence; + for (size_t i = 0; i < node.size(); i++) { + Value itm; + itm.parent_ = this; + itm.index_ = vec_.size(); + itm.key_ = std::to_string(i); + // insert empty sequence into parent. (makes copy of itm) + ret = map_.insert(std::pair(itm.key_, itm)); + vec_.push_back(ret.first); + + // Now populate the new node. + Value *new_location = &ret.first->second; + new_location->setNode(&node[i]); + } + } else if (node.IsMap()) { + type_ = Value::Map; + for (auto it = node.begin(); it != node.end(); it++) { + Value itm; + itm.parent_ = this; + itm.index_ = vec_.size(); + itm.key_ = it->first.as(); // get the key + // insert empty map into parent. (makes copy of itm). + ret = map_.insert(std::pair(itm.key_, itm)); + vec_.push_back(ret.first); + + // Now populate the new node. + Value *new_location = &ret.first->second; + new_location->setNode(&it->second); + } } } -std::shared_ptr Value::getScalar() const { - NTA_CHECK(category_ == scalarCategory); - return scalar_; -} +#endif // YAML_PARSER_yamlcpp -std::shared_ptr Value::getArray() const { - NTA_CHECK(category_ == arrayCategory); - return array_; +///////////////////////////////////////////////////////////////////////////////////////// + +// Constructor +Value::Value() { + type_ = Value::Category::Empty; + parent_ = nullptr; + zombie_ = nullptr; + assigned_ = nullptr; + index_ = ZOMBIE_MAP; } -std::string Value::getString() const { - NTA_CHECK(category_ == stringCategory); - return string_; + +// checking content of a parameter +// enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category Value::getCategory() const { return type_; } + +bool Value::contains(const std::string& key) const { return (map_.find(key) != map_.end()); } + +bool Value::isScalar() const { return type_ == Value::Category::Scalar; } +bool Value::isSequence() const { return type_ == Value::Category::Sequence; } +bool Value::isMap() const { return type_ == Value::Category::Map; } +bool Value::isEmpty() const { return type_ == Value::Category::Empty; } + +size_t Value::size() const { + NTA_CHECK(map_.size() == vec_.size()) << "Detected Corruption of ValueMap structure"; + return map_.size(); } -template T Value::getScalarT() const { - NTA_CHECK(category_ == scalarCategory); - if (BasicType::getType() != scalar_->getType()) { - NTA_THROW << "Attempt to access scalar of type " - << BasicType::getName(scalar_->getType()) << " as type " - << BasicType::getName(); +// Accessing members of a map +// If not found, a Zombie Value object is returned. +// An error will be displayed when you try to access the value in the Zombie Value object. +// If you assign something to the Zombie Value, it will insert it into the tree with the saved key. +Value& Value::operator[](const std::string& key) { + if (assigned_) + return (*assigned_)[key]; + auto it = map_.find(key); + if (it == map_.end()) { + // not found. Create a zombie in case we will later assign something to this key. + // Its type is Value::Category::Empty. + // NOTE: only one zombie per parent can exist at a time. + zombie_.reset(new Value()); + zombie_->parent_ = this; + zombie_->scalar_ = key; + zombie_->key_ = key; + zombie_->index_ = ZOMBIE_MAP; + return *zombie_; } - return scalar_->getValue(); + else + return it->second; } - -const std::string Value::getDescription() const { - switch (category_) { - case stringCategory: - return std::string("string") + " (" + string_ + ")"; - break; - case scalarCategory: - return std::string("Scalar of type ") + BasicType::getName(scalar_->getType()); - break; - case arrayCategory: - return std::string("Array of type ") + BasicType::getName(array_->getType()); - break; +const Value& Value::operator[](const std::string& key) const { + if (assigned_) + return (*assigned_)[key]; + auto it = map_.find(key); + if (it == map_.end()) { + // not found. Create a (const) zombie which signals if found or not but + // cannot be used with an assignment. Its type is Value::Category::Empty. + static Value const_zombie; // This is a constant + return const_zombie; } - return "NOT REACHED"; + else + return it->second; } -void ValueMap::add(const std::string &key, const Value &value) { - if (map_.find(key) != map_.end()) { - NTA_THROW << "Key '" << key << "' specified twice"; +// accessing members of a sequence +Value& Value::operator[](size_t index) { + if (assigned_) + return (*assigned_)[index]; + if (index < vec_.size()) + return vec_[index]->second; + else if (index == vec_.size()) { + // Not found, create a zombie in case we later assign it. + // Note that the index can ONLY be the size-of-vector. + // Make sure the key is uneque. append '-'s until it is. + std::string key = std::to_string(index); + while (true) { + if (map_.find(key) == map_.end()) + break; + key += "-"; + } + zombie_.reset(new Value()); + zombie_->parent_ = this; + zombie_->key_ = key; + zombie_->index_ = ZOMBIE_SEQ; + return *zombie_; } - auto vp = new Value(value); + NTA_THROW << "Index out of range; " << index; +} +const Value& Value::operator[](size_t index) const { + if (assigned_) + return (*assigned_)[index]; + if (index < vec_.size()) + return vec_[index]->second; + NTA_THROW << "Index out of range; " << index; // is const so cannot make a zombie +} - map_.insert(std::make_pair(key, vp)); +std::string Value::str() const { + NTA_CHECK(type_ == Value::Category::Scalar); + return scalar_; +} +const char* Value::c_str() const { + NTA_CHECK(type_ == Value::Category::Scalar); + return scalar_.c_str(); } -Value::Category Value::getCategory() const { return category_; } +std::string Value::key() const { return key_; } -ValueMap::const_iterator ValueMap::begin() const { return map_.begin(); } +std::vector Value::getKeys() const { + NTA_CHECK(isMap()) << "This is not a map."; + std::vector v; + for (auto it = begin(); it != end(); it++) { + v.push_back(it->first); + } + return v; +} -ValueMap::const_iterator ValueMap::end() const { return map_.end(); } +// Insert this node into the parent. +// Requires that there was a key (either string or index) unless it is root +// and if it was a string key, its index will be ZOMBIE_MAP. +void Value::addToParent() { + std::pair::iterator, bool> ret; + if (parent_ == nullptr) + return; // This is the root + NTA_CHECK(!key_.empty()) << "No key was provided. Use node[key] = value."; + if (parent_->type_ == Value::Category::Empty) { + parent_->addToParent(); + if (parent_->assigned_) + parent_ = parent_->assigned_; + } + bool map_key = (index_ == ZOMBIE_MAP); -// specializations of getValue() -// gcc 4.2 complains if they are not inside the namespace declaration -namespace htm { -template Byte Value::getScalarT() const; -template Int16 Value::getScalarT() const; -template Int32 Value::getScalarT() const; -template Int64 Value::getScalarT() const; -template UInt16 Value::getScalarT() const; -template UInt32 Value::getScalarT() const; -template UInt64 Value::getScalarT() const; -template Real32 Value::getScalarT() const; -template Real64 Value::getScalarT() const; -template Handle Value::getScalarT() const; -template bool Value::getScalarT() const; -} // namespace htm + // Add the node to the parent. + index_ = parent_->vec_.size(); + ret = parent_->map_.insert(std::pair(key_, *this)); + parent_->vec_.push_back(ret.first); + assigned_ = &ret.first->second; -ValueMap::ValueMap(){}; + NTA_CHECK(parent_->map_.size() == parent_->vec_.size()) << "Detected Corruption of ValueMap structure"; -ValueMap::~ValueMap() { - for (auto &elem : map_) { - delete elem.second; - elem.second = nullptr; - } - map_.clear(); + if (map_key) + parent_->type_ = Value::Category::Map; + else if (parent_->type_ == Value::Category::Empty) + parent_->type_ = Value::Category::Sequence; } -ValueMap::ValueMap(const ValueMap &rhs) { - for (auto &elem : map_) { - delete elem.second; - elem.second = nullptr; +void Value::assign(std::string val) { + std::pair::iterator, bool> ret; + if (type_ == Value::Category::Empty) { // previous search was false + // This is a zombie node. By assigning a value we add it to the tree. + // The key was already set in the operator[]. + + // Add to parent. + // If its parent is also a zombie, add it to the tree as well. + addToParent(); + assigned_->scalar_ = val; + assigned_->type_ = Value::Category::Scalar; + } else { + // Must be a value already in the tree. Do a replace. + if (type_ != Value::Category::Scalar) { + map_.clear(); + vec_.clear(); + type_ = Value::Category::Scalar; + } + scalar_ = val; } +} +// Assign a value converted from a specified type T. +void Value::operator=(char *val) { assign(val); } +void Value::operator=(const std::string &val) { assign(val); } +void Value::operator=(int8_t val) { assign(std::to_string(val)); } +void Value::operator=(int16_t val) { assign(std::to_string(val)); } +void Value::operator=(uint16_t val) { assign(std::to_string(val)); } +void Value::operator=(int32_t val) { assign(std::to_string(val)); } +void Value::operator=(uint32_t val) { assign(std::to_string(val)); } +void Value::operator=(int64_t val) { assign(std::to_string(val)); } +void Value::operator=(uint64_t val) { assign(std::to_string(val)); } +void Value::operator=(bool val) { assign((val) ? "true" : "false"); } +void Value::operator=(float val) { assign(std::to_string(val)); } +void Value::operator=(double val) { assign(std::to_string(val)); } +void Value::operator=(std::vector val) { + // Insert the contents of the vector into this node. map_.clear(); - - for (const auto &rh : rhs) { - auto vp = new Value(*(rh.second)); - - map_.insert(std::make_pair(rh.first, vp)); + vec_.clear(); + for (size_t i = 0; i < val.size(); i++) { + operator[](i) = std::to_string(val[i]); } } -void ValueMap::dump() const { - NTA_DEBUG << "===== Value Map:"; - for (const auto &elem : map_) { - std::string key = elem.first; - Value *value = elem.second; - NTA_DEBUG << "key: " << key - << " datatype: " << BasicType::getName(value->getType()) - << " category: " << value->getCategory(); - } - NTA_DEBUG << "===== End of Value Map"; -} -bool ValueMap::contains(const std::string &key) const { - return (map_.find(key) != map_.end()); -} -Value &ValueMap::getValue(const std::string &key) const { - auto item = map_.find(key); - if (item == map_.end()) { - NTA_THROW << "No value '" << key << "' found in Value Map"; +void Value::remove() { + Value *node = (assigned_) ? assigned_ : this; + NTA_CHECK(!node->isEmpty()) << "Item not found."; // current node is a zombie. + if (node->parent_ == nullptr) { + // This is root. Just clear the map. + node->vec_.clear(); + node->map_.clear(); + node->type_ = Value::Category::Empty; + return; + } + NTA_CHECK(node->parent_->vec_[index_]->second == *node); + if (node->parent_->vec_.size() == 1) { + // Last node in parent, remove parent. + node->parent_->remove(); + return; } - return *(item->second); + std::string key = node->parent_->vec_[index_]->first; + auto itr = node->parent_->map_.find(key); + + // adjust the index on all following items. + // We have to do it here because as soon as we erase the map item + // it will delete 'this'. + for (size_t i = index_+1; i < node->parent_->vec_.size(); i++) { + node->parent_->vec_[i]->second.index_ = i-1; + } + + node->parent_->vec_.erase(node->parent_->vec_.begin() + index_); + node->parent_->map_.erase(itr); + // The node object is deleted. Do no try to access it. } -template -T ValueMap::getScalarT(const std::string &key, T defaultValue) const { - auto item = map_.find(key); - if (item == map_.end()) { - return defaultValue; - } else { - return getScalarT(key); +// Compare two nodes recursively to see if content is same. +static bool equals(const Value &n1, const Value &n2) { + if (n1.getCategory() != n2.getCategory()) + return false; + if (n1.isSequence()) { + if (n1.size() != n2.size()) + return false; + for (size_t i = 0; i < n1.size(); i++) + if (!equals(n1[i], n2[i])) + return false; + return true; + } + if (n1.isMap()) { + if (n1.size() != n2.size()) + return false; + for (auto it : n1) { + if (!n2[it.first]) + return false; + if (!equals(it.second, n2[it.first])) + return false; + } + return true; } + if (n1.isScalar()) { + if (n1.str() == n2.str()) + return true; + } + return false; } - -template T ValueMap::getScalarT(const std::string &key) const { - std::shared_ptr s = getScalar(key); - if (s->getType() != BasicType::getType()) { - NTA_THROW << "Invalid attempt to access parameter '" << key - << "' as type a " << BasicType::getName() - << " but the Spec defines it as type " << BasicType::getName(s->getType()); +bool Value::operator==(const Value &v) const { return equals(*this, v); } + +// Explicit instantiations for as() +template int8_t Value::as() const; +template int16_t Value::as() const; +template uint16_t Value::as() const; +template int32_t Value::as() const; +template uint32_t Value::as() const; +template int64_t Value::as() const; +template uint64_t Value::as() const; +template float Value::as() const; +template double Value::as() const; +template std::string Value::as() const; +template bool Value::as() const; + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } } - - return s->getValue(); } -std::shared_ptr ValueMap::getArray(const std::string &key) const { - Value &v = getValue(key); - if (!v.isArray()) { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as an array but it is a '" - << v.getDescription(); +static void to_json(std::ostream &f, const htm::Value &v) { + bool first = true; + std::string s; + switch (v.getCategory()) { + case Value::Empty: + return; + case Value::Scalar: + s = v.str(); + if (std::regex_match(s, std::regex("^[-+]?[0-9]+([.][0-9]+)?$"))) { + escape_json(f, s); + } else { + f << '"'; + escape_json(f, s); + f << '"'; + } + break; + case Value::Sequence: + f << "["; + for (size_t i = 0; i < v.size(); i++) { + if (!first) + f << ", "; + first = false; + const Value &n = v[i]; + to_json(f, n); + } + f << "]"; + break; + case Value::Map: + f << "{"; + for (size_t i = 0; i < v.size(); i++) { + if (!first) + f << ", "; + first = false; + const Value &n = v[i]; + f << n.key() << ": "; + to_json(f, n); + } + f << "}"; + break; } - return v.getArray(); } -std::shared_ptr ValueMap::getScalar(const std::string &key) const { - Value &v = getValue(key); - if (!v.isScalar()) { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as an array but it is a '" - << v.getDescription(); - } - return v.getScalar(); +std::string Value::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); } -std::string ValueMap::getString(const std::string& key) const { - Value& v = getValue(key); - if (! v.isString()) - { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as a string but it is a '" - << v.getDescription(); +static void escape_yaml(std::ostream &o, const std::string &s, const std::string &indent) { + if (std::strchr(s.c_str(), '\n')) { + // contains newlines + o << " |"; // all blanks are significant + const char *from = s.c_str(); + const char *to = from; + while ((to = std::strchr(to, '\n')) != NULL) { + std::string line(from, to); + o << "\n" + indent + line; + ++to; + from = to; + } + o << "\n" + indent + from; + } else { + o << s; } - return v.getString(); } -std::string ValueMap::getString(const std::string &key, const std::string defaultValue) const { - auto item = map_.find(key); - if (item == map_.end()) { - return defaultValue; - } else { - Value &v = getValue(key); - if (!v.isString()) { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as a string but it is a '" - << v.getDescription(); + +static void to_yaml(std::ostream & f, const htm::Value &v, std::string indent) { + bool first = true; + std::string s; + switch (v.getCategory()) { + case Value::Empty: + return; + case Value::Scalar: + s = v.str(); + escape_yaml(f, s, indent); + f << "\n"; + break; + case Value::Sequence: + for (size_t i = 0; i < v.size(); i++) { + const Value &n = v[i]; + f << indent << "- "; + if (n.isMap() || n.isSequence()) + f << "\n"; + to_yaml(f, n, indent + " "); } - return v.getString(); + break; + case Value::Map: + for (size_t i = 0; i < v.size(); i++) { + const Value &n = v[i]; + f << indent << n.key() << ": "; + if (n.isMap() || n.isSequence()) + f << "\n"; + to_yaml(f, n, indent + " "); + } + break; } } +std::string Value::to_yaml() const { + std::stringstream f; + ::to_yaml(f, *this, ""); + return f.str(); +} -// explicit instantiations of getScalarT namespace htm { -template Byte ValueMap::getScalarT(const std::string &key, - Byte defaultValue) const; -template UInt16 ValueMap::getScalarT(const std::string &key, - UInt16 defaultValue) const; -template Int16 ValueMap::getScalarT(const std::string &key, - Int16 defaultValue) const; -template UInt32 ValueMap::getScalarT(const std::string &key, - UInt32 defaultValue) const; -template Int32 ValueMap::getScalarT(const std::string &key, - Int32 defaultValue) const; -template UInt64 ValueMap::getScalarT(const std::string &key, - UInt64 defaultValue) const; -template Int64 ValueMap::getScalarT(const std::string &key, - Int64 defaultValue) const; -template Real32 ValueMap::getScalarT(const std::string &key, - Real32 defaultValue) const; -template Real64 ValueMap::getScalarT(const std::string &key, - Real64 defaultValue) const; -template Handle ValueMap::getScalarT(const std::string &key, - Handle defaultValue) const; -template bool ValueMap::getScalarT(const std::string &key, - bool defaultValue) const; - -template Byte ValueMap::getScalarT(const std::string &key) const; -template UInt16 ValueMap::getScalarT(const std::string &key) const; -template Int16 ValueMap::getScalarT(const std::string &key) const; -template UInt32 ValueMap::getScalarT(const std::string &key) const; -template Int32 ValueMap::getScalarT(const std::string &key) const; -template UInt64 ValueMap::getScalarT(const std::string &key) const; -template Int64 ValueMap::getScalarT(const std::string &key) const; -template Real32 ValueMap::getScalarT(const std::string &key) const; -template Real64 ValueMap::getScalarT(const std::string &key) const; -template Handle ValueMap::getScalarT(const std::string &key) const; -template bool ValueMap::getScalarT(const std::string &key) const; +std::ostream &operator<<(std::ostream &f, const htm::Value &v) { + f << v.to_json(); + return f; +} } // namespace htm diff --git a/src/htm/ntypes/Value.hpp b/src/htm/ntypes/Value.hpp index 08ca996376..441ec9eb4a 100644 --- a/src/htm/ntypes/Value.hpp +++ b/src/htm/ntypes/Value.hpp @@ -1,6 +1,8 @@ /* --------------------------------------------------------------------- * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. + * Copyright (C) 2019, Numenta, Inc. + * + * David Keeney dkeeney@gmail.com * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero Public License version 3 as @@ -14,122 +16,226 @@ * You should have received a copy of the GNU Affero Public License * along with this program. If not, see http://www.gnu.org/licenses. * --------------------------------------------------------------------- */ - -/** @file - * Definitions for the Value class - * - * A Value object holds a Scalar or an Array - * A ValueMap is essentially a map - * It is used internally in the conversion of YAML strings to C++ objects. - * The API and implementation are geared towards clarify rather than - * performance, since it is expected to be used only during network construction. - */ - #ifndef NTA_VALUE_HPP #define NTA_VALUE_HPP - -#include +#include #include +#include +#include +#include +#include +#include +#include // transform +#include +#include // std::strerror(errno) -#include +namespace htm { -#include -#include -namespace htm { -/** - * The Value class is used to store construction parameters - * for regions and links. A YAML string specified by the user - * is parsed and converted into a set of Values. - * - * A Value is essentially a union of Scalar/Array/string. - * In turn, a Scalar is a union of NTA_BasicType types, - * and an Array is an array of such types. - * - * A string is similar to an Array of NTA_BasicType_Byte, but - * is handled differently, so it is separated in the API. - * - * The Value API uses std::shared_ptr instead of directly - * using the underlying objects, to avoid copying, and because - * Array may not be copied. - */ class Value { public: - Value(std::shared_ptr &s); - Value(std::shared_ptr &a); - Value(const std::string &s); - - enum Category { scalarCategory, arrayCategory, stringCategory }; - - bool isArray() const; - bool isString() const; - bool isScalar() const; - Category getCategory() const; - - NTA_BasicType getType() const; - - std::shared_ptr getScalar() const; - - std::shared_ptr getArray() const; - - std::string getString() const; - - template T getScalarT() const; - - const std::string getDescription() const; - -private: - // Default constructor would not be useful Value(); - Category category_; - std::shared_ptr scalar_; - std::shared_ptr array_; - std::string string_; -}; -class ValueMap { -public: - ValueMap(); - ValueMap(const ValueMap &rhs); - ~ValueMap(); - void add(const std::string &key, const Value &value); + // Parse a Yaml or JSON string an assign it to this node in the tree. + Value &parse(const std::string &yaml_string); - // map.find(key) != map.end() bool contains(const std::string &key) const; + size_t size() const; - // map.find(key) + exception if not found - Value &getValue(const std::string &key) const; - - // Method below are for convenience, bypassing the Value - std::shared_ptr getArray(const std::string &key) const; - std::shared_ptr getScalar(const std::string &key) const; - std::string getString(const std::string &key) const; - std::string getString(const std::string &key, const std::string defaultValue) const; + // type of value in the Value node + enum Category { Empty = 0, Scalar, Sequence, Map }; + Category getCategory() const; + bool isScalar() const; + bool isSequence() const; + bool isMap() const; + bool isEmpty() const; // false if current node has a value,sequence,or map + // true if operator[] did not find a value or was not assigned to. + + // Access + std::string str() const; // return copy of raw value as a string + std::string key() const; // return copy of raw key as a string + const char *c_str() const; // return a const pointer to raw value + std::vector getKeys() const; + + Value &operator[](const std::string &key); + Value &operator[](size_t index); + const Value &operator[](const std::string &key) const; + const Value &operator[](size_t index) const; + + template T as() const { + if (type_ != Category::Scalar) + NTA_THROW << "not found. '" << scalar_ << "'"; + T result; + std::stringstream ss(scalar_); + ss >> result; + return result; + } +#define NTA_CONVERT(T, I) \ + if (type_ != Value::Category::Scalar) \ + NTA_THROW << "value not found."; \ + errno = 0; \ + char *end; \ + T val = (T)I; \ + if (errno) \ + NTA_THROW << "In '" << scalar_ << "' numeric conversion error: " << std::strerror(errno); \ + if (*end != '\0') \ + NTA_THROW << "In '" << scalar_ << "' numeric conversion error: invalid char."; \ + return val; + + // Return a value converted to the specified type T. + // explicit specializations. By default it uses streams to convert. + template <> int8_t as() const { NTA_CONVERT(int8_t, std::strtol(scalar_.c_str(), &end, 0)); } + template <> int16_t as() const { NTA_CONVERT(int16_t, std::strtol(scalar_.c_str(), &end, 0)); } + template <> uint16_t as() const { NTA_CONVERT(uint16_t, std::strtoul(scalar_.c_str(), &end, 0)); } + template <> int32_t as() const { NTA_CONVERT(int32_t, std::strtol(scalar_.c_str(), &end, 0)); } + template <> uint32_t as() const { NTA_CONVERT(uint32_t, std::strtoul(scalar_.c_str(), &end, 0)); } + template <> int64_t as() const { NTA_CONVERT(int64_t, std::strtoll(scalar_.c_str(), &end, 0)); } + template <> uint64_t as() const { NTA_CONVERT(uint64_t, std::strtoull(scalar_.c_str(), &end, 0)); } + template <> float as() const { NTA_CONVERT(float, std::strtof(scalar_.c_str(), &end)); } + template <> double as() const { NTA_CONVERT(double, std::strtod(scalar_.c_str(), &end)); } + template <> std::string as() const { + if (type_ != Value::Category::Scalar) + NTA_THROW << "value not found."; + return scalar_; + } + template <> bool as() const { + if (type_ != Value::Category::Scalar) + NTA_THROW << "value not found. " << scalar_; + std::string val = str(); + transform(val.begin(), val.end(), val.begin(), ::tolower); + if (val == "true" || val == "on" || val == "1") + return true; + if (val == "false" || val == "off" || val == "0") + return false; + NTA_THROW << "Invalid value for a boolean. " << val; + } + + template T as(T d) const { + if (type_ != Value::Category::Scalar) + return d; + return as(); + } + + /** Assign a value to a Value node in the tree. + * If a previous operator[] found a match, this does a replace. + * If a previous operator[] did not find a match in the tree + * the current Value is a Zombie and not attached to the tree. + * But in this case its requested key is remembered. A subsequent + * operator= will assign this node to the tree with the remembered key. + * The parent will become a map if it is not already. + */ + void operator=(char *val); + void operator=(const std::string &val); + void operator=(int8_t val); + void operator=(int16_t val); + void operator=(uint16_t val); + void operator=(int32_t val); + void operator=(uint32_t val); + void operator=(int64_t val); + void operator=(uint64_t val); + void operator=(bool val); + void operator=(float val); + void operator=(double val); + void operator=(std::vector); + + /** + * Remove a node from the tree. (an element of map or sequence) + * It is assumed that this is a low use function because + * it will not be very efficent. + */ + void remove(); + + // compare two nodes in the tree. + bool operator==(const Value &v) const; + + // return false if node is empty: if (vm) { do something } + explicit operator bool() const { return !isEmpty(); } + + // extract a Vector + template std::vector asVector() const { + std::vector v; + if (!isSequence()) + NTA_THROW << "Not a sequence node."; + for (size_t i = 0; i < size(); i++) { // iterate through the children of this node. + const Value &n = (*this)[i]; + try { + if (n.isScalar()) { + v.push_back(n.as()); + } + } catch (std::exception &e) { + NTA_THROW << "Invalid vector element; " << e.what(); + } + } + return v; + } + + // extract a map. Key is always a string. + template std::map asMap() const { + std::map v; + for (auto iter = cbegin(); iter != cend(); iter++) { // iterate through the children of this node. + const Value &n = iter->second; + try { + if (n.isScalar()) { + v[n.key_] = n.as(); + } else { + // non-scalar field. Ignore + } + } catch (std::exception &e) { + // probably bad conversion of scalar to requested type. + NTA_THROW << "Invalid map element[\"" << n.key_ << "\"] " << e.what(); + } + } + return v; + } + + // serializing routines + std::string to_yaml() const; + std::string to_json() const; + + // Access for backward compatability + template T getScalarT(const std::string &key) const { // throws + return (*this)[key].as(); + } + template T getScalarT(const std::string &key, T defaultValue) const { // with default + return (*this)[key].as(defaultValue); + } + std::string getString(const std::string &key, const std::string &defaultValue) const { + return (*this)[key].as(defaultValue); + } + + friend std::ostream &operator<<(std::ostream &f, const htm::Value &vm); + + std::map::iterator begin() { return map_.begin(); } + std::map::iterator end() { return map_.end(); } + std::map::const_iterator begin() const { return map_.begin(); } + std::map::const_iterator end() const { return map_.end(); } + std::map::const_iterator cbegin() const { return map_.cbegin(); } + std::map::const_iterator cend() const { return map_.cend(); } - // More convenience methods, bypassing the Value and the contained Scalar +private: + void setNode(void *node); // add opaque node. + void assign(std::string val); // add a scalar + void addToParent(); // Assign to the map/vector in the parent + + enum Category type_; // type of node + std::map map_; + std::vector::iterator> vec_; + Value *parent_; // a pointer to the parent node (do not delete) + std::string scalar_; // scalar value. Not used on Map or Sequence nodes + std::string key_; // The key for this node. + size_t index_; // initial index Also, ZOMBIE_MAP in zombie nodes. + std::shared_ptr zombie_; // pointer to a zombie child Value node. Returned when operator[] not found. + Value *assigned_; // For a zombie, a pointer to assigned node in tree. +}; - // use default value if not specified in map - template - T getScalarT(const std::string &key, T defaultValue) const; - // raise exception if value is not specified in map - template T getScalarT(const std::string &key) const; +using ValueMap = Value; - void dump() const; +} // namespace htm - typedef std::map::const_iterator const_iterator; - const_iterator begin() const; - const_iterator end() const; -private: - // must be a Value* since Value doesn't have a default constructor - // We own all the items in the map and must delete them in our destructor - typedef std::map::iterator iterator; - std::map map_; -}; -} // namespace htm -#endif // NTA_VALUE_HPP +#endif // NTA_VALUE_HPP diff --git a/src/htm/regions/SPRegion.cpp b/src/htm/regions/SPRegion.cpp index aa5db33e5f..985837d861 100644 --- a/src/htm/regions/SPRegion.cpp +++ b/src/htm/regions/SPRegion.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include @@ -275,7 +274,7 @@ Spec *SPRegion::createSpec() { NTA_BasicType_Bool, // type 1, // elementCount "bool", // constraints - "false", // defaultValue + "true", // defaultValue ParameterSpec::ReadWriteAccess)); // access ns->parameters.add( @@ -426,7 +425,7 @@ Spec *SPRegion::createSpec() { NTA_BasicType_Int32, // type 1, // elementCount "", // constraints - "-1", // defaultValue + "1", // defaultValue ParameterSpec::CreateAccess)); // access ns->parameters.add( diff --git a/src/htm/regions/SPRegion.hpp b/src/htm/regions/SPRegion.hpp index 07633bc64b..021e24d08f 100644 --- a/src/htm/regions/SPRegion.hpp +++ b/src/htm/regions/SPRegion.hpp @@ -30,6 +30,7 @@ #include #include +#include //---------------------------------------------------------------------- diff --git a/src/htm/regions/ScalarSensor.cpp b/src/htm/regions/ScalarSensor.cpp index 0d391d3877..b40bd0e04b 100644 --- a/src/htm/regions/ScalarSensor.cpp +++ b/src/htm/regions/ScalarSensor.cpp @@ -32,19 +32,19 @@ namespace htm { ScalarSensor::ScalarSensor(const ValueMap ¶ms, Region *region) : RegionImpl(region) { - params_.size = params.getScalarT("n"); - params_.activeBits = params.getScalarT("w"); - params_.resolution = params.getScalarT("resolution"); - params_.radius = params.getScalarT("radius"); - params_.minimum = params.getScalarT("minValue"); - params_.maximum = params.getScalarT("maxValue"); - params_.periodic = params.getScalarT("periodic"); - params_.clipInput = params.getScalarT("clipInput"); + params_.size = params.getScalarT("n", 0); + params_.activeBits = params.getScalarT("w", 0); + params_.resolution = params.getScalarT("resolution", 0.0); + params_.radius = params.getScalarT("radius", 0.0); + params_.minimum = params.getScalarT("minValue", -1.0); + params_.maximum = params.getScalarT("maxValue", +1.0); + params_.periodic = params.getScalarT("periodic", false); + params_.clipInput = params.getScalarT("clipInput", false); encoder_ = std::make_shared( params_ ); - sensedValue_ = params.getScalarT("sensedValue"); + sensedValue_ = params.getScalarT("sensedValue", -1.0); } ScalarSensor::ScalarSensor(ArWrapper &wrapper, Region *region):RegionImpl(region) { @@ -203,21 +203,40 @@ ScalarSensor::~ScalarSensor() {} Real64 ScalarSensor::getParameterReal64(const std::string &name, Int64 index) { if (name == "sensedValue") { return sensedValue_; - } + } else if (name == "resolution") return encoder_->parameters.resolution; + else if (name == "radius") + return encoder_->parameters.radius; + else if (name == "minValue") + return encoder_->parameters.minimum; + else if (name == "maxValue") + return encoder_->parameters.maximum; else { return RegionImpl::getParameterReal64(name, index); } } +bool ScalarSensor::getParameterBool(const std::string& name, Int64 index) { + if (name == "periodic") + return encoder_->parameters.periodic; + if (name == "clipInput") + return encoder_->parameters.clipInput; + else { + return RegionImpl::getParameterBool(name, index); + } +} + UInt32 ScalarSensor::getParameterUInt32(const std::string &name, Int64 index) { if (name == "n") { return (UInt32)encoder_->size; } - else { + else if (name == "w") { + return encoder_->parameters.activeBits; + } else { return RegionImpl::getParameterUInt32(name, index); } } + void ScalarSensor::setParameterReal64(const std::string &name, Int64 index, Real64 value) { if (name == "sensedValue") { sensedValue_ = value; diff --git a/src/htm/regions/ScalarSensor.hpp b/src/htm/regions/ScalarSensor.hpp index afa53f51fa..ed1491108f 100644 --- a/src/htm/regions/ScalarSensor.hpp +++ b/src/htm/regions/ScalarSensor.hpp @@ -51,6 +51,7 @@ class ScalarSensor : public RegionImpl, Serializable { virtual Real64 getParameterReal64(const std::string &name, Int64 index = -1) override; virtual UInt32 getParameterUInt32(const std::string &name, Int64 index = -1) override; + virtual bool getParameterBool(const std::string &name, Int64 index = -1) override; virtual void setParameterReal64(const std::string &name, Int64 index, Real64 value) override; virtual void initialize() override; diff --git a/src/htm/regions/TMRegion.cpp b/src/htm/regions/TMRegion.cpp index d08fdbb620..75eeebbed6 100644 --- a/src/htm/regions/TMRegion.cpp +++ b/src/htm/regions/TMRegion.cpp @@ -338,7 +338,7 @@ Spec *TMRegion::createSpec() { NTA_BasicType_UInt32, // type 1, // elementCount "", // constraints - "8", // defaultValue + "10", // defaultValue ParameterSpec::CreateAccess)); // access ns->parameters.add( diff --git a/src/htm/regions/TestNode.cpp b/src/htm/regions/TestNode.cpp index 9ec534c2cf..0d8ce81106 100644 --- a/src/htm/regions/TestNode.cpp +++ b/src/htm/regions/TestNode.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include @@ -52,7 +51,7 @@ TestNode::TestNode(const ValueMap ¶ms, Region *region) shouldCloneParam_ = params.getScalarT("shouldCloneParam", 1) != 0; - stringParam_ = params.getString("stringParam"); + stringParam_ = params.getString("stringParam", "nodespec value"); real32ArrayParam_.resize(8); for (size_t i = 0; i < 8; i++) { diff --git a/src/htm/regions/VectorFileEffector.cpp b/src/htm/regions/VectorFileEffector.cpp index 84ebc1ad3a..00da1dec04 100644 --- a/src/htm/regions/VectorFileEffector.cpp +++ b/src/htm/regions/VectorFileEffector.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include @@ -39,7 +38,7 @@ VectorFileEffector::VectorFileEffector(const ValueMap ¶ms, Region* region) : RegionImpl(region), dataIn_(NTA_BasicType_Real32), filename_(""), outFile_(nullptr) { if (params.contains("outputFile")) { - std::string s = params.getString("outputFile"); + std::string s = params.getString("outputFile", ""); openFile(s); } else diff --git a/src/htm/regions/VectorFileEffector.hpp b/src/htm/regions/VectorFileEffector.hpp index 30f571599d..d1bb8f9792 100644 --- a/src/htm/regions/VectorFileEffector.hpp +++ b/src/htm/regions/VectorFileEffector.hpp @@ -30,10 +30,10 @@ #include #include #include +#include namespace htm { -class ValueMap; /** * VectorFileEffector is a node that takes its input vectors and diff --git a/src/htm/regions/VectorFileSensor.cpp b/src/htm/regions/VectorFileSensor.cpp index 7f947c3023..5839208b86 100644 --- a/src/htm/regions/VectorFileSensor.cpp +++ b/src/htm/regions/VectorFileSensor.cpp @@ -32,7 +32,6 @@ #include #include #include -#include using namespace std; namespace htm { @@ -58,12 +57,12 @@ VectorFileSensor::VectorFileSensor(const ValueMap ¶ms, Region *region) activeOutputCount_ = params.getScalarT("activeOutputCount", 0); hasCategoryOut_ = params.getScalarT("hasCategoryOut", 0) == 1; hasResetOut_ = params.getScalarT("hasResetOut", 0) == 1; - scalingMode_ = params.getString("scalingMode"); + scalingMode_ = params.getString("scalingMode", "none"); if (scalingMode_ != "standardForm") scalingMode_ = "none"; if (params.contains("inputFile")) - filename_ = params.getString("inputFile"); + filename_ = params.getString("inputFile", ""); } VectorFileSensor::VectorFileSensor(ArWrapper& wrapper, Region *region) diff --git a/src/htm/regions/VectorFileSensor.hpp b/src/htm/regions/VectorFileSensor.hpp index 2c8cfc2386..1a42ee14a5 100644 --- a/src/htm/regions/VectorFileSensor.hpp +++ b/src/htm/regions/VectorFileSensor.hpp @@ -33,9 +33,9 @@ #include #include #include +#include namespace htm { -class ValueMap; /** * VectorFileSensor is a sensor that reads in files containing lists of diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index e4bd4258b1..455d006ab3 100755 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -57,7 +57,6 @@ set(engine_tests unit/engine/InputTest.cpp unit/engine/LinkTest.cpp unit/engine/NetworkTest.cpp - unit/engine/YAMLUtilsTest.cpp unit/engine/WatcherTest.cpp ) @@ -71,7 +70,6 @@ set(ntypes_tests unit/ntypes/BasicTypeTest.cpp unit/ntypes/CollectionTest.cpp unit/ntypes/DimensionsTest.cpp - unit/ntypes/ScalarTest.cpp unit/ntypes/ValueTest.cpp ) @@ -86,8 +84,8 @@ set(regions_tests unit/regions/RegionTestUtilities.cpp unit/regions/RegionTestUtilities.hpp unit/regions/SPRegionTest.cpp - unit/regions/TMRegionTest.cpp - unit/regions/VectorFileTest.cpp + unit/regions/TMRegionTest.cpp + unit/regions/VectorFileTest.cpp ) diff --git a/src/test/unit/engine/CppRegionTest.cpp b/src/test/unit/engine/CppRegionTest.cpp index c57c42af34..c6966a7c08 100644 --- a/src/test/unit/engine/CppRegionTest.cpp +++ b/src/test/unit/engine/CppRegionTest.cpp @@ -118,6 +118,9 @@ TEST(CppRegionTest, testCppLinkingSDR) { net.initialize(); + EXPECT_EQ(region1->getParameterUInt32("n"), 6u); + EXPECT_EQ(region1->getParameterUInt32("w"), 2u); + region1->setParameterReal64("sensedValue", 0.8); //Note: default range setting is -1.0 to +1.0 const Dimensions r1dims = region1->getOutput("encoded")->getDimensions(); EXPECT_EQ(r1dims.size(), 2u) << " actual dims: " << r1dims.toString(); @@ -153,22 +156,17 @@ TEST(CppRegionTest, testCppLinkingSDR) { << "Expected dimensions on the output to match dimensions on the buffer."; VERBOSE << r2OutputArray << "\n"; SDR exp({20u, 3u}); - exp.setSparse(SDR_sparse_t{ - 4, 21, 32, 46 - }); - EXPECT_EQ(r2OutputArray, exp.getDense()) << "got " << r2OutputArray; + exp.setSparse(SDR_sparse_t{10, 38, 57}); + EXPECT_EQ(r2OutputArray, exp.getDense()) << "expected " << exp << " got " << r2OutputArray; } TEST(CppRegionTest, testYAML) { const char *params = "{count: 42, int32Param: 1234, real64Param: 23.1}"; - // badparams contains a non-existent parameter - const char *badparams = "{int32Param: 1234, real64Param: 23.1, badParam: 4}"; Network net; std::shared_ptr level1; - EXPECT_THROW(net.addRegion("level1", "TestNode", badparams), exception); EXPECT_NO_THROW({level1 = net.addRegion("level1", "TestNode", params);}); diff --git a/src/test/unit/engine/WatcherTest.cpp b/src/test/unit/engine/WatcherTest.cpp index e6688eb47e..2502eab37d 100644 --- a/src/test/unit/engine/WatcherTest.cpp +++ b/src/test/unit/engine/WatcherTest.cpp @@ -36,7 +36,7 @@ namespace testing { -static bool verbose = false; +static bool verbose = true; #define VERBOSE \ if (verbose) \ std::cerr << "[ ] " @@ -125,8 +125,7 @@ TEST(WatcherTest, FileTest1) { std::string tempString; if (inStream.is_open()) { getline(inStream, tempString); - ASSERT_EQ("Info: watchID, regionName, nodeType, nodeIndex, varName", - tempString); + ASSERT_EQ("Info: watchID, regionName, nodeType, nodeIndex, varName", tempString); getline(inStream, tempString); ASSERT_EQ("1, level1, TestNode, -1, uint32Param", tempString); getline(inStream, tempString); diff --git a/src/test/unit/engine/YAMLUtilsTest.cpp b/src/test/unit/engine/YAMLUtilsTest.cpp deleted file mode 100644 index e299f301cd..0000000000 --- a/src/test/unit/engine/YAMLUtilsTest.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -/** @file - * Implementation of YAMLUtils test - */ - -#include "gtest/gtest.h" -#include -#include -#include - -namespace testing { - -using namespace htm; - -TEST(YAMLUtilsTest, toValueTestInt) { - const char *s1 = "10"; - Value v = YAMLUtils::toValue(s1, NTA_BasicType_Int32); - EXPECT_TRUE(v.isScalar()) - << "assertion v.isScalar() failed at " << __FILE__ << ":" << __LINE__; - ASSERT_EQ(v.getType(), NTA_BasicType_Int32); - Int32 i = v.getScalarT(); - ASSERT_EQ(10, i); - std::shared_ptr s = v.getScalar(); - i = s->value.int32; - ASSERT_EQ(10, i); -} - -TEST(YAMLUtilsTest, handle1LetterInputString) -{ - const char* s1 = "1"; - EXPECT_NO_THROW(YAMLUtils::toValue(s1, NTA_BasicType_Int32)); -} - -TEST(YAMLUtilsTest, toValueTestReal32) -{ - const char* s1 = "10.1"; - Value v = YAMLUtils::toValue(s1, NTA_BasicType_Real32); - EXPECT_TRUE(v.isScalar()) - << "assertion v.isScalar() failed at " << __FILE__ << ":" << __LINE__; - ASSERT_EQ(v.getType(), NTA_BasicType_Real32); - Real32 x = v.getScalarT(); - EXPECT_NEAR(10.1f, x, 0.000001) << "assertion 10.1 == " << x << "\" failed at " - << __FILE__ << ":" << __LINE__; - - std::shared_ptr s = v.getScalar(); - x = s->value.real32; - EXPECT_NEAR(10.1f, x, 0.000001) << "assertion 10.1 == " << x << "\" failed at " - << __FILE__ << ":" << __LINE__; -} - -TEST(YAMLUtilsTest, toValueTestByte) { - const char *s1 = "this is a string"; - Value v = YAMLUtils::toValue(s1, NTA_BasicType_Byte); - EXPECT_TRUE(!v.isScalar()) - << "assertion !v.isScalar() failed at " << __FILE__ << ":" << __LINE__; - EXPECT_TRUE(v.isString()) - << "assertion v.isScalar() failed at " << __FILE__ << ":" << __LINE__; - ASSERT_EQ(v.getType(), NTA_BasicType_Byte); - std::string s = v.getString(); - EXPECT_STREQ(s1, s.c_str()); -} - -TEST(YAMLUtilsTest, toValueTestBool) { - const char *s1 = "true"; - Value v = YAMLUtils::toValue(s1, NTA_BasicType_Bool); - EXPECT_TRUE(v.isScalar()) - << "assertion v.isScalar() failed at " << __FILE__ << ":" << __LINE__; - ASSERT_EQ(v.getType(), NTA_BasicType_Bool); - bool b = v.getScalarT(); - ASSERT_EQ(true, b); - std::shared_ptr s = v.getScalar(); - b = s->value.boolean; - ASSERT_EQ(true, b); -} - -TEST(YAMLUtilsTest, ParameterSpec) { - Collection ps; - ps.add("int32Param", - ParameterSpec("Int32 scalar parameter", // description - NTA_BasicType_Int32, - 1, // elementCount - "", // constraints - "32", // defaultValue - ParameterSpec::ReadWriteAccess)); - - ps.add("uint32Param", - ParameterSpec("UInt32 scalar parameter", // description - NTA_BasicType_UInt32, - 1, // elementCount - "", // constraints - "33", // defaultValue - ParameterSpec::ReadWriteAccess)); - - ps.add("int64Param", - ParameterSpec("Int64 scalar parameter", // description - NTA_BasicType_Int64, - 1, // elementCount - "", // constraints - "64", // defaultValue - ParameterSpec::ReadWriteAccess)); - - ps.add("uint64Param", - ParameterSpec("UInt64 scalar parameter", // description - NTA_BasicType_UInt64, - 1, // elementCount - "", // constraints - "65", // defaultValue - ParameterSpec::ReadWriteAccess)); - - ps.add("real32Param", - ParameterSpec("Real32 scalar parameter", // description - NTA_BasicType_Real32, - 1, // elementCount - "", // constraints - "32.1", // defaultValue - ParameterSpec::ReadWriteAccess)); - - ps.add("real64Param", - ParameterSpec("Real64 scalar parameter", // description - NTA_BasicType_Real64, - 1, // elementCount - "", // constraints - "64.1", // defaultValue - ParameterSpec::ReadWriteAccess)); - - ps.add("real32ArrayParam", - ParameterSpec("int32 array parameter", NTA_BasicType_Real32, - 0, // array - "", "", ParameterSpec::ReadWriteAccess)); - - ps.add("int64ArrayParam", - ParameterSpec("int64 array parameter", NTA_BasicType_Int64, - 0, // array - "", "", ParameterSpec::ReadWriteAccess)); - - ps.add( - "computeCallback", - ParameterSpec("address of a function that is called at every compute()", - NTA_BasicType_Handle, 1, "", - "", // handles must not have a default value - ParameterSpec::ReadWriteAccess)); - - ps.add("stringParam", - ParameterSpec("string parameter", NTA_BasicType_Byte, - 0, // length=0 required for strings - "", "default value", ParameterSpec::ReadWriteAccess)); - - ps.add("boolParam", ParameterSpec("bool parameter", NTA_BasicType_Bool, 1, "", - "false", ParameterSpec::ReadWriteAccess)); - - NTA_DEBUG << "ps count: " << ps.getCount(); - - ValueMap vm = YAMLUtils::toValueMap("", ps); - EXPECT_TRUE(vm.contains("int32Param")) - << "assertion vm.contains(\"int32Param\") failed at " << __FILE__ << ":" - << __LINE__; - ASSERT_EQ((Int32)32, vm.getScalarT("int32Param")); - - EXPECT_TRUE(vm.contains("boolParam")) - << "assertion vm.contains(\"boolParam\") failed at " - << __FILE__ << ":" << __LINE__; - ASSERT_FALSE(vm.getScalarT("boolParam")); - - EXPECT_STREQ("default value", vm.getString("stringParam").c_str()); - - // Test error message in case of invalid parameter with and without nodeType - // and regionName - try { - YAMLUtils::toValueMap("{ blah: True }", ps, "nodeType", "regionName"); - } catch (htm::Exception &e) { - std::string s("Unknown parameter 'blah' for region 'regionName'"); - EXPECT_TRUE(std::string(e.getMessage()).find(s) == 0) - << "assertion std::string(e.getMessage()).find(s) == 0 failed at " - << __FILE__ << ":" << __LINE__; - } - - try { - YAMLUtils::toValueMap("{ blah: True }", ps); - } catch (htm::Exception &e) { - std::string s("Unknown parameter 'blah'\nValid"); - EXPECT_TRUE(std::string(e.getMessage()).find(s) == 0) - << "assertion std::string(e.getMessage()).find(s) == 0 failed at " - << __FILE__ << ":" << __LINE__; - } -} -} \ No newline at end of file diff --git a/src/test/unit/ntypes/ValueTest.cpp b/src/test/unit/ntypes/ValueTest.cpp index 39d12aece4..41406fb136 100644 --- a/src/test/unit/ntypes/ValueTest.cpp +++ b/src/test/unit/ntypes/ValueTest.cpp @@ -22,107 +22,339 @@ #include #include -namespace testing { - +#include +#include +#include + +namespace testing { + using namespace htm; +TEST(ValueTest, toValueNumber) { + ValueMap vm; + + const char *s1 = "10"; + vm.parse(s1); + EXPECT_TRUE(vm.isScalar()); + UInt32 u = vm.as(); + EXPECT_EQ(10u, u); -TEST(ValueTest, Scalar) { - std::shared_ptr s(new Scalar(NTA_BasicType_Int32)); - s->value.int32 = 10; - Value v(s); - ASSERT_TRUE(v.isScalar()); - ASSERT_TRUE(!v.isString()); - ASSERT_TRUE(!v.isArray()); - ASSERT_EQ(Value::scalarCategory, v.getCategory()); - ASSERT_EQ(NTA_BasicType_Int32, v.getType()); + vm.parse("-1"); + Int32 i = vm.as(); + EXPECT_EQ(-1, i); + UInt32 x = vm.as(); + EXPECT_EQ(4294967295u, x); - std::shared_ptr s1 = v.getScalar(); - ASSERT_TRUE(s1 == s); + vm.parse("- 1"); // "- " means a sequence element in YAML + EXPECT_TRUE(vm.isSequence()); + UInt32 u1 = vm[0].as(); + UInt32 u2 = 1u; + EXPECT_EQ(u1, u2); - ASSERT_ANY_THROW(v.getArray()); - ASSERT_ANY_THROW(v.getString()); + vm.parse("[123]"); // explicit sequence with one element. + EXPECT_TRUE(vm.isSequence()); + i = vm[0].as(); + EXPECT_EQ(123, i); - EXPECT_STREQ("Scalar of type Int32", v.getDescription().c_str()); + EXPECT_ANY_THROW(vm.parse("999999999999999999999999999").as()); + EXPECT_ANY_THROW(vm.parse("abc").as()); + EXPECT_ANY_THROW(vm.parse("").as()); - Int32 x = v.getScalarT(); - ASSERT_EQ(10, x); + vm.parse("A: 1"); + EXPECT_EQ(1, vm["A"].as(123)); // with a default value but found + EXPECT_EQ(123, vm["B"].as(123)); // with a default value and not found +} - ASSERT_ANY_THROW(v.getScalarT()); +TEST(ValueTest, toValueTestReal32) { + ValueMap vm; + vm.parse("10.1"); + Real32 x = vm.as(); + EXPECT_NEAR(10.1f, x, 0.000001); } -TEST(ValueTest, Array) { - std::shared_ptr s(new Array(NTA_BasicType_Int32)); - s->allocateBuffer(10); - Value v(s); - ASSERT_TRUE(v.isArray()); - ASSERT_TRUE(!v.isString()); - ASSERT_TRUE(!v.isScalar()); - ASSERT_EQ(Value::arrayCategory, v.getCategory()); - ASSERT_EQ(NTA_BasicType_Int32, v.getType()); +TEST(ValueTest, toValueString) { + ValueMap vm; + std::string s1 = "A: \"this is a string\""; + std::string s; + + // Positive tests + vm.parse(s1); + s = vm.getScalarT("A", "x"); + EXPECT_TRUE(s == "this is a string"); + s = vm.getScalarT("A"); + EXPECT_TRUE(s == "this is a string"); + s = vm.getString("A", "x"); + EXPECT_TRUE(s == "this is a string"); + s = vm["A"].as(); + EXPECT_TRUE(s == "this is a string"); + s = vm["A"].str(); + EXPECT_TRUE(s == "this is a string"); - std::shared_ptr s1 = v.getArray(); - ASSERT_TRUE(s1 == s); + // Negative tests + s = vm.getScalarT("B", "x"); + EXPECT_TRUE(s == "x"); + EXPECT_ANY_THROW(s = vm.getScalarT("B")); + s = vm.getString("B", "y"); + EXPECT_TRUE(s == "y"); + EXPECT_ANY_THROW(s = vm["B"].as()); + EXPECT_ANY_THROW(s = vm["B"].str()); +} - ASSERT_ANY_THROW(v.getScalar()); - ASSERT_ANY_THROW(v.getString()); - ASSERT_ANY_THROW(v.getScalarT()); +TEST(ValueTest, toValueBool) { + ValueMap vm; - EXPECT_STREQ("Array of type Int32", v.getDescription().c_str()); + EXPECT_TRUE(vm.parse("B: true").getScalarT("B", false)); + EXPECT_TRUE(vm.parse("B: True").getScalarT("B", false)); + EXPECT_TRUE(vm.parse("B: 1").getScalarT("B", false)); + EXPECT_TRUE(vm.parse("B: ON").getScalarT("B", false)); + EXPECT_FALSE(vm.parse("B: false").getScalarT("B", true)); + EXPECT_FALSE(vm.parse("B: FALSE").getScalarT("B", true)); + EXPECT_FALSE(vm.parse("B: 0").getScalarT("B", true)); + EXPECT_FALSE(vm.parse("B: off").getScalarT("B", true)); + vm.parse("B: false"); + EXPECT_FALSE(vm.getScalarT("B", true)); + EXPECT_FALSE(vm.parse("B: 0").getScalarT("B", true)); + EXPECT_ANY_THROW(vm.parse("B: 1234").getScalarT("B", false)); } +TEST(ValueTest, asArray) { + ValueMap vm; + std::vector s2 = {10u, 20u, 30u, 40u, 50u}; + + std::string json = "[10,20,30,40,50]"; + vm.parse(json); + + EXPECT_EQ(ValueMap::Sequence, vm.getCategory()); + EXPECT_TRUE(vm.isSequence()); + EXPECT_TRUE(!vm.isMap()); + EXPECT_TRUE(!vm.isScalar()); + EXPECT_TRUE(!vm.isEmpty()); + + std::vector s1 = vm.asVector(); + EXPECT_TRUE(s1 == s2); + + EXPECT_EQ(vm[0].as(), 10u); + EXPECT_STREQ(vm[0].str().c_str(), "10"); + + std::vector s3 = {100u, 200u, 300u, 400u, 500u}; + vm[5] = s3; // assign an array to the 6th element. + + std::string t = vm.to_json(); + EXPECT_STREQ(t.c_str(), "[10, 20, 30, 40, 50, [100, 200, 300, 400, 500]]"); + + EXPECT_EQ(vm[0].as(), 10u); + EXPECT_TRUE(vm[5].isSequence()); + EXPECT_TRUE(vm[5][4].isScalar()); + EXPECT_EQ(vm[5][4].as(), 500u); + EXPECT_ANY_THROW(vm.as()); // not a scaler + EXPECT_ANY_THROW(vm[5].as()); // not a sequence +} + + +TEST(ValueTest, asMap) { + ValueMap vm; + std::string src = "{scalar: 456, array: [1, 2, 3, 4], string: \"true\"}"; + vm.parse(src); + + //std::cout << vm << "\n"; + + std::map m; + m = vm.asMap(); + std::stringstream ss; + ss << "{"; + bool first = true; + for (auto itr = m.begin(); itr != m.end(); itr++) { + if (!first) ss << ", "; + first = false; + ss << itr->first << ": " << itr->second; + } + ss << "}"; + std::string result = ss.str(); + std::cout << result << "\n"; + EXPECT_STREQ(result.c_str(), "{scalar: 456, string: true}"); +} + + TEST(ValueTest, String) { std::string s("hello world"); - Value v(s); - ASSERT_TRUE(!v.isArray()); - ASSERT_TRUE(v.isString()); - ASSERT_TRUE(!v.isScalar()); - ASSERT_EQ(Value::stringCategory, v.getCategory()); - ASSERT_EQ(NTA_BasicType_Byte, v.getType()); - - std::string s1 = v.getString(); + Value v; + v.parse(s); + EXPECT_TRUE(!v.isSequence()); + EXPECT_TRUE(!v.isMap()); + EXPECT_TRUE(v.isScalar()); + + std::string s1 = v.str(); EXPECT_STREQ("hello world", s1.c_str()); - ASSERT_ANY_THROW(v.getScalar()); - ASSERT_ANY_THROW(v.getArray()); - ASSERT_ANY_THROW(v.getScalarT()); + EXPECT_ANY_THROW(v.asVector()); + EXPECT_ANY_THROW(v.as()); + + EXPECT_STREQ("\"hello world\"", v.to_json().c_str()); +} + - EXPECT_STREQ("string (hello world)", v.getDescription().c_str()); + +TEST(ValueTest, inserts) { + ValueMap vm; + vm[0][0][0] = 1; + //std::cout << "inserts1: " << vm << "\n"; + EXPECT_STREQ("[[[1]]]", vm.to_json().c_str()); + + Value v = vm[1]; // Create a zombie node + for (int i = 0; i < 3; i++) { + v[i] = i*100+100; // assign to an array on a zombie which should add it to the tree. + } + //std::cout << "inserts2: " << vm << "\n"; + EXPECT_STREQ("[[[1]], [100, 200, 300]]", vm.to_json().c_str()); + + EXPECT_ANY_THROW(vm[3]["hello"] = "world"); } -TEST(ValueTest, ValueMap) { - std::shared_ptr s(new Scalar(NTA_BasicType_Int32)); - s->value.int32 = 10; - std::shared_ptr a(new Array(NTA_BasicType_Real32)); - std::string str("hello world"); +TEST(ValueTest, ValueMapTest) { + std::vector a = {1, 2, 3, 4}; ValueMap vm; - vm.add("scalar", s); - vm.add("array", a); - vm.add("string", str); - ASSERT_ANY_THROW(vm.add("scalar", s)); - - ASSERT_TRUE(vm.contains("scalar")); - ASSERT_TRUE(vm.contains("array")); - ASSERT_TRUE(vm.contains("string")); - ASSERT_TRUE(!vm.contains("foo")); - ASSERT_TRUE(!vm.contains("scalar2")); - ASSERT_TRUE(!vm.contains("xscalar")); - - std::shared_ptr s1 = vm.getScalar("scalar"); - ASSERT_TRUE(s1 == s); - - std::shared_ptr a1 = vm.getArray("array"); - ASSERT_TRUE(a1 == a); - - std::shared_ptr def(new Scalar(NTA_BasicType_Int32)); - Int32 x = vm.getScalarT("scalar", (Int32)20); - ASSERT_EQ((Int32)10, x); - - x = vm.getScalarT("scalar2", (Int32)20); - ASSERT_EQ((Int32)20, x); - - Value v = vm.getValue("array"); - ASSERT_EQ(Value::arrayCategory, v.getCategory()); - ASSERT_TRUE(v.getArray() == a); + vm["scalar"] = 123; + vm["scalar"] = 456; // should replace + vm["array"] = a; + vm["string"] = std::string("str"); + + EXPECT_TRUE(vm.isMap()); + EXPECT_TRUE(vm.contains("scalar")); + EXPECT_TRUE(vm.contains("array")); + EXPECT_TRUE(vm.contains("string")); + EXPECT_TRUE(!vm.contains("foo")); + EXPECT_TRUE(!vm.contains("scalar2")); + EXPECT_TRUE(!vm.contains("xscalar")); + + int s = vm["scalar"].as(); + EXPECT_TRUE(456 == s); + + std::vector a1 = vm["array"].asVector(); + EXPECT_TRUE(a1 == a); + + Int32 x = vm.getScalarT("scalar2", (Int32)20); + EXPECT_EQ((Int32)20, x); + + std::string expected = "{scalar: 456, array: [1, 2, 3, 4], string: \"str\"}"; + std::string result; + std::stringstream ss; + ss << vm; + result = ss.str(); + EXPECT_STREQ(result.c_str(), expected.c_str()); + + result = vm.to_json(); + EXPECT_STREQ(result.c_str(), expected.c_str()); + + ValueMap vm2; + vm2.parse(result); + //std::cout << "vm=" << vm << "\n"; + //std::cout << "vm2" << vm2 << "\n"; + EXPECT_TRUE(vm == vm2); } + +TEST(ValueTest, Iterations) { + ValueMap vm; + std::vector results; + int cnt = 0; + + std::string data = R"(scalar: 123.45 +array: + - 1 + - 2 + - 3 + - 4 +string: this is a string +)"; + + vm.parse(data); + //std::cout << vm << "\n"; + for (auto itr = vm.begin(); itr != vm.end(); itr++) { + std::string key = itr->first; + if (key == "scalar") + EXPECT_NEAR(itr->second.as(), 123.45f, 0.000001); + else if (key == "string") + EXPECT_STREQ(itr->second.str().c_str(), "this is a string"); + else if (key == "array") { + for (size_t i = 0; i < 4; i++) { + EXPECT_EQ(i + 1, itr->second[i].as()); + } + } else + NTA_THROW << "unexpected key"; + } + + // iterate with for range + cnt = 0; + for (auto itm : vm) { + if (itm.second.isScalar()) + cnt++; + if (itm.second.isSequence()) + cnt++; + } + EXPECT_EQ(cnt, 3); + + std::string result = vm.to_yaml(); + //std::cout << result << "\n"; + EXPECT_STREQ(result.c_str(), data.c_str()); + } + +TEST(ValueTest, deletes) { + ValueMap vm; + std::string src = "{scalar: 456, array: [1, 2, 3, 4], string: \"true\"}"; + vm.parse(src); + + EXPECT_EQ(vm.size(), 3); + EXPECT_EQ(vm["array"].size(), 4); + + vm["scalar"].remove(); + EXPECT_EQ(vm.size(), 2); + EXPECT_ANY_THROW(vm["scalar"].str()); + EXPECT_TRUE(vm[0].isSequence()); + EXPECT_TRUE(vm[0][0].isScalar()); + EXPECT_EQ(vm[0][0].as(), 1); + EXPECT_EQ(vm[0][3].as(), 4); + EXPECT_EQ(vm[0].size(), 4); + + vm[0][0].remove(); + EXPECT_EQ(vm[0].size(), 3); + EXPECT_TRUE(vm[0][0].isScalar()); + EXPECT_EQ(vm[0][0].as(), 2); + EXPECT_EQ(vm[0][2].as(), 4); + EXPECT_ANY_THROW(vm[0][3].as()); + + vm[0][2].remove(); + EXPECT_EQ(vm[0].size(), 2); + EXPECT_TRUE(vm[0][1].isScalar()); + EXPECT_EQ(vm[0][0].as(), 2); + EXPECT_EQ(vm[0][1].as(), 3); + EXPECT_ANY_THROW(vm[0][2].as()); + + vm[0][2] = 6; + EXPECT_EQ(vm[0].size(), 3); + EXPECT_TRUE(vm[0][1].isScalar()); + EXPECT_EQ(vm[0][0].as(), 2); + EXPECT_EQ(vm[0][1].as(), 3); + EXPECT_ANY_THROW(vm[0][3].as()); + std::vector v = vm[0].asVector(); + + std::vector expected({2, 3, 6}); + for (size_t i = 0; i < 3; i++) { + EXPECT_EQ(expected[i], v[i]); + } + + vm[0][2].remove(); + EXPECT_EQ(vm[0].size(), 2); + for (size_t i = 0; i < 2; i++) { + EXPECT_EQ(expected[i], v[i]); + } + + vm[0].remove(); + EXPECT_EQ(vm.size(), 1); + EXPECT_TRUE(vm[0].isScalar()); + EXPECT_TRUE(vm.contains("string")); + + vm.remove(); + EXPECT_TRUE(vm.isEmpty()); +} + +} // namespace testing diff --git a/src/test/unit/regions/RegionTestUtilities.cpp b/src/test/unit/regions/RegionTestUtilities.cpp index 3405fad352..088957990d 100644 --- a/src/test/unit/regions/RegionTestUtilities.cpp +++ b/src/test/unit/regions/RegionTestUtilities.cpp @@ -68,11 +68,6 @@ void checkGetSetAgainstSpec(std::shared_ptr region1, switch (p.second.dataType) { case NTA_BasicType_UInt32: { - if (negativeCheck) { - negativeCheck = false; - VERBOSE << "negative check..." << std::endl; - EXPECT_THROW(region1->getParameterInt32("bad_parameter"), htm::Exception); - } VERBOSE << "Parameter \"" << name << "\" type: " << BasicType::getName(p.second.dataType) << std::endl; // check the getter. diff --git a/src/test/unit/regions/SPRegionTest.cpp b/src/test/unit/regions/SPRegionTest.cpp index 2c9b182485..47a1bdb11c 100644 --- a/src/test/unit/regions/SPRegionTest.cpp +++ b/src/test/unit/regions/SPRegionTest.cpp @@ -50,7 +50,6 @@ #include #include #include -#include #include @@ -64,7 +63,6 @@ #include -#include "yaml-cpp/yaml.h" #include "gtest/gtest.h" #include "RegionTestUtilities.hpp" diff --git a/src/test/unit/regions/TMRegionTest.cpp b/src/test/unit/regions/TMRegionTest.cpp index 7d83942608..7201ada382 100644 --- a/src/test/unit/regions/TMRegionTest.cpp +++ b/src/test/unit/regions/TMRegionTest.cpp @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include @@ -66,7 +65,6 @@ #include #include "RegionTestUtilities.hpp" -#include "yaml-cpp/yaml.h" #include "gtest/gtest.h" #define VERBOSE if (verbose) std::cerr << "[ ] " @@ -216,7 +214,7 @@ TEST(TMRegionTest, testLinking) { std::shared_ptr region1 = net.addRegion("region1", "VectorFileSensor",parameters); std::shared_ptr region2 = net.addRegion("region2", "SPRegion", "{dim: [2,10]}"); std::shared_ptr region3 = net.addRegion("region3", "TMRegion", - "{activationThreshold: 9, cellsPerColumn: 5}"); + "{activationThreshold: 12, cellsPerColumn: 5}"); std::shared_ptr region4 = net.addRegion("region4", "VectorFileEffector", "{outputFile: '" + test_output_file + "'}"); @@ -273,7 +271,7 @@ TEST(TMRegionTest, testLinking) { VERBOSE << " " << r3InputArray << "\n"; std::vector expected3in = VectorHelpers::sparseToBinary( { - 7 + 19 }, (UInt32)r3InputArray.getCount()); EXPECT_EQ(r3InputArray, expected3in) << r3InputArray; @@ -290,9 +288,7 @@ TEST(TMRegionTest, testLinking) { EXPECT_TRUE(r3OutputArray.getType() == NTA_BasicType_SDR); VERBOSE << " " << r3OutputArray << "\n"; std::vector expected3out = VectorHelpers::sparseToBinary( //TODO replace with SDR - { - 35, 36, 37, 38, 39 - }, (UInt32)r3OutputArray.getCount()); + {95, 96, 97, 98, 99 }, (UInt32)r3OutputArray.getCount()); EXPECT_EQ(r3OutputArray, expected3out) << r3OutputArray; EXPECT_EQ(r3OutputArray.getSDR().getSparse().size(), 5u); @@ -309,9 +305,7 @@ TEST(TMRegionTest, testLinking) { << numberOfCols << " * " << cellsPerColumn; VERBOSE << " " << r3OutputArray << ")\n"; std::vector expected3outa = VectorHelpers::sparseToBinary( - { - 95, 96, 97, 98, 99 - }, (UInt32)r3OutputArray.getCount()); + {60, 61, 62, 63, 64 }, (UInt32)r3OutputArray.getCount()); EXPECT_EQ(r3OutputArray, expected3outa) << r3OutputArray; diff --git a/src/test/unit/regions/VectorFileTest.cpp b/src/test/unit/regions/VectorFileTest.cpp index 23d4721571..54db00f477 100644 --- a/src/test/unit/regions/VectorFileTest.cpp +++ b/src/test/unit/regions/VectorFileTest.cpp @@ -49,7 +49,6 @@ #include #include #include -#include #include #include @@ -67,7 +66,6 @@ -#include "yaml-cpp/yaml.h" #include "gtest/gtest.h" #include "RegionTestUtilities.hpp"