From 84faaa05c5f76abd5e0235480b84108fce679e75 Mon Sep 17 00:00:00 2001 From: David Keeney Date: Tue, 26 Feb 2019 11:55:26 -0800 Subject: [PATCH 01/10] A placeholder for this PR --- external/bootstrap.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index e7078f8983..a249bfbc73 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -34,10 +34,10 @@ FILE(MAKE_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty) execute_process(COMMAND ${CMAKE_COMMAND} - -G ${CMAKE_GENERATOR} + -G ${CMAKE_GENERATOR} -D CMAKE_INSTALL_PREFIX=. - -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} - -D BINDING_BUILD:STRING=${BINDING_BUILD} + -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} + -D BINDING_BUILD:STRING=${BINDING_BUILD} -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ../../external WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty From 22fa00c944592b5e514bcb56e15d7a4d42766589 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 28 Feb 2019 15:23:51 +0100 Subject: [PATCH 02/10] YAMLUtils cleanup --- src/nupic/engine/YAMLUtils.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/nupic/engine/YAMLUtils.cpp b/src/nupic/engine/YAMLUtils.cpp index a2f6166ef1..a16e4a1c00 100644 --- a/src/nupic/engine/YAMLUtils.cpp +++ b/src/nupic/engine/YAMLUtils.cpp @@ -89,6 +89,7 @@ static void _toArray(const YAML::Node& node, std::shared_ptr& a) { a->allocateBuffer(node.size()); void *buffer = a->getBuffer(); + NTA_CHECK(buffer != nullptr) << "buffer is null"; for (size_t i = 0; i < node.size(); i++) { const YAML::Node &item = node[i]; @@ -276,11 +277,6 @@ ValueMap toValueMap(const char *yamlstring, // } try { -#ifdef YAMLDEBUG - NTA_DEBUG << "Adding default value '" << ps.defaultValue - << "' to parameter " << item.first << " of type " - << BasicType::getName(ps.dataType) << " count " << ps.count; -#endif // NOTE: this can handle both scalers and arrays // Arrays MUST be in Yaml sequence format even if one element. // i.e. [1,2,3] From 4b9a97f38dec328431d850d887fc7401174f4ca1 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 28 Feb 2019 15:41:39 +0100 Subject: [PATCH 03/10] YAMLUtils toValueMap uses string, not char* --- src/nupic/engine/RegionImplFactory.cpp | 2 +- src/nupic/engine/YAMLUtils.cpp | 3 ++- src/nupic/engine/YAMLUtils.hpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/nupic/engine/RegionImplFactory.cpp b/src/nupic/engine/RegionImplFactory.cpp index ba18ba6761..9f3e2d720a 100644 --- a/src/nupic/engine/RegionImplFactory.cpp +++ b/src/nupic/engine/RegionImplFactory.cpp @@ -116,7 +116,7 @@ RegionImpl *RegionImplFactory::createRegionImpl(const std::string nodeType, RegionImpl *impl = nullptr; std::shared_ptr& ns = getSpec(nodeType); - ValueMap vm = YAMLUtils::toValueMap(nodeParams.c_str(), ns->parameters, + ValueMap vm = YAMLUtils::toValueMap(nodeParams, ns->parameters, nodeType, region->getName()); if (regionTypeMap.find(nodeType) != regionTypeMap.end()) { diff --git a/src/nupic/engine/YAMLUtils.cpp b/src/nupic/engine/YAMLUtils.cpp index a16e4a1c00..7c198bea81 100644 --- a/src/nupic/engine/YAMLUtils.cpp +++ b/src/nupic/engine/YAMLUtils.cpp @@ -166,6 +166,7 @@ static Value toValue(const YAML::Node &node, NTA_BasicType dataType) { Value toValue(const std::string& yamlstring, NTA_BasicType dataType) { // TODO -- return value? exceptions? + NTA_CHECK(yamlstring.size() > 0) << "string is empty!"; const YAML::Node doc = YAML::Load(yamlstring); return toValue(doc, dataType); } @@ -173,7 +174,7 @@ Value toValue(const std::string& yamlstring, NTA_BasicType dataType) /* * For converting param specs for Regions and LinkPolicies */ -ValueMap toValueMap(const char *yamlstring, +ValueMap toValueMap(const std::string yamlstring, Collection ¶meters, const std::string &nodeType, const std::string ®ionName) { diff --git a/src/nupic/engine/YAMLUtils.hpp b/src/nupic/engine/YAMLUtils.hpp index b2e2f4ad53..15f2075207 100644 --- a/src/nupic/engine/YAMLUtils.hpp +++ b/src/nupic/engine/YAMLUtils.hpp @@ -41,7 +41,7 @@ namespace nupic /* * For converting param specs for Regions and LinkPolicies */ -ValueMap toValueMap(const char *yamlstring, +ValueMap toValueMap(const std::string yamlstring, Collection ¶meters, const std::string &nodeType = "", const std::string ®ionName = ""); From 5752689fa43a488eb72ee35e8f17f1cd66c1b951 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 28 Feb 2019 16:20:52 +0100 Subject: [PATCH 04/10] CMake: external deps always build in Release mode this should fix yaml-cpp bug, gtest might need to match Debug/Release of project --- external/bootstrap.cmake | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index 5657000fb3..cd348cfc3c 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -29,16 +29,18 @@ # - create build/scripts (mkdir -d build/scripts) # - cd build/scripts # - cmake ../.. +# +# externals are always built in Release mode, CMake's build type is ignored FILE(MAKE_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty) execute_process(COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} - -D CMAKE_INSTALL_PREFIX=. + -D CMAKE_INSTALL_PREFIX=. -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} -D BINDING_BUILD:STRING=${BINDING_BUILD} - -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -D CMAKE_BUILD_TYPE="Release" ../../external WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty RESULT_VARIABLE result From 2a9d59f93e3f17fafaf5392b327350ed0bd03e75 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 28 Feb 2019 16:22:50 +0100 Subject: [PATCH 05/10] dummy --- external/bootstrap.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index cd348cfc3c..6e4c390aaa 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -51,7 +51,7 @@ if(result) endif() if(MSVC) # for MSVC builds we need to build both Release and Debug builds - # because this will not be ran again if we switch modes in the IDE. + # because this will not be ran again if we switch modes in the IDE. execute_process(COMMAND ${CMAKE_COMMAND} --build . --config Release WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty RESULT_VARIABLE result From 3694c908399a340fa9ff2bffc18c3ff038a735ff Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 28 Feb 2019 16:33:38 +0100 Subject: [PATCH 06/10] CMake: do not build both Release, Debug for Windows determine by CMAKE_BUILD_TYPE, IDEs will need to rebuild,fix if Debug/Release is switched --- external/bootstrap.cmake | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index 6e4c390aaa..eb86daa3c6 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -40,7 +40,7 @@ execute_process(COMMAND ${CMAKE_COMMAND} -D CMAKE_INSTALL_PREFIX=. -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} -D BINDING_BUILD:STRING=${BINDING_BUILD} - -D CMAKE_BUILD_TYPE="Release" + -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ../../external WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty RESULT_VARIABLE result @@ -49,36 +49,15 @@ execute_process(COMMAND ${CMAKE_COMMAND} if(result) message(FATAL_ERROR "CMake step for Third Party builds failed: ${result}") endif() -if(MSVC) - # for MSVC builds we need to build both Release and Debug builds - # because this will not be ran again if we switch modes in the IDE. - execute_process(COMMAND ${CMAKE_COMMAND} --build . --config Release - WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty - RESULT_VARIABLE result -# OUTPUT_QUIET ### Disable this to debug external buiilds - ) - if(result) - message(FATAL_ERROR "build step for MSVC Relase Third Party builds failed: ${result}") - endif() - execute_process(COMMAND ${CMAKE_COMMAND} --build . --config Debug - WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty - RESULT_VARIABLE result -# OUTPUT_QUIET ### Disable this to debug external buiilds - ) - if(result) - message(FATAL_ERROR "build step for MSVC Debug Third Party builds failed: ${result}") - endif() -else(MSVC) - # for linux and OSx builds - execute_process(COMMAND ${CMAKE_COMMAND} --build . --config ${CMAKE_BUILD_TYPE} + +execute_process(COMMAND ${CMAKE_COMMAND} --build . WORKING_DIRECTORY ${REPOSITORY_DIR}/build/ThirdParty RESULT_VARIABLE result -# OUTPUT_QUIET ### Disable this to debug external buiilds +# OUTPUT_QUIET ### Disable this to debug external builds ) if(result) message(FATAL_ERROR "build step for Third Party builds failed: ${result}") endif() -endif(MSVC) # extract the external directory paths # The external third party modules are being built From 36a441031012a3f8e301b2417cd289bf604713ed Mon Sep 17 00:00:00 2001 From: David Keeney Date: Thu, 28 Feb 2019 09:45:43 -0800 Subject: [PATCH 07/10] using libYaml --- external/CMakeLists.txt | 4 + external/libYaml.cmake | 49 ++++++ external/libyaml/YAMLParser.cpp | 145 +++++++++++++++++ external/libyaml/YAMLParser.hpp | 61 ++++++++ external/libyaml/yaml.hpp | 270 ++++++++++++++++++++++++++++++++ src/nupic/engine/Spec.hpp | 53 ++++--- src/nupic/engine/YAMLUtils.cpp | 6 + startupCodeBlocks.sh | 14 ++ 8 files changed, 576 insertions(+), 26 deletions(-) create mode 100644 external/libYaml.cmake create mode 100644 external/libyaml/YAMLParser.cpp create mode 100644 external/libyaml/YAMLParser.hpp create mode 100644 external/libyaml/yaml.hpp create mode 100644 startupCodeBlocks.sh diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 1784f75cd4..1f4d381403 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -76,6 +76,10 @@ FILE(REMOVE "${EXPORT_FILE_NAME}") # Yaml-cpp include(YamlCppLib.cmake) +################ +# libyaml +include(libyaml.cmake) + ################ # Eigen diff --git a/external/libYaml.cmake b/external/libYaml.cmake new file mode 100644 index 0000000000..01902f4763 --- /dev/null +++ b/external/libYaml.cmake @@ -0,0 +1,49 @@ +# ----------------------------------------------------------------------------- +# Numenta Platform for Intelligent Computing (NuPIC) +# Copyright (C) 2016, Numenta, Inc. Unless you have purchased from +# Numenta, Inc. a separate commercial license for this software code, the +# following terms and conditions apply: +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see http://www.gnu.org/licenses. +# +# http://numenta.org/licenses/ +# ----------------------------------------------------------------------------- +# This downloads and builds the libyaml library. +# +if(EXISTS ${REPOSITORY_DIR}/build/ThirdParty/share/libyaml.zip + set(URL ${REPOSITORY_DIR}/build/ThirdParty/share/libyaml.zip) +else() + set(URL "https://github.com/yaml/libyaml/archive/master.zip") +endif() + +message(STATUS "Obtaining libyaml") +include(DownloadProject/DownloadProject.cmake) +download_project(PROJ libyaml + PREFIX ${EP_BASE}/libyaml + URL ${URL} + GIT_SHALLOW ON + UPDATE_DISCONNECTED 1 + QUIET + ) + +add_subdirectory(${libyaml_SOURCE_DIR} ${libyaml_BINARY_DIR}) + +set(libyaml_INCLUDE_DIRS ${libyaml_SOURCE_DIR}/include) +if (MSVC) + set(libyaml_LIBRARIES "${libyaml_BINARY_DIR}$<$:/Release/liblibyamlmd.lib>$<$:/Debug/liblibyamlmdd.lib>") +else() + set(ylibyaml_LIBRARIES ${libyaml_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}libyaml${CMAKE_STATIC_LIBRARY_SUFFIX}) +endif() +FILE(APPEND "${EXPORT_FILE_NAME}" "libyaml_INCLUDE_DIRS@@@${libyaml_SOURCE_DIR}/include\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "libyaml_LIBRARIES@@@${libyaml_LIBRARIES}\n") + diff --git a/external/libyaml/YAMLParser.cpp b/external/libyaml/YAMLParser.cpp new file mode 100644 index 0000000000..e5ab4b8a15 --- /dev/null +++ b/external/libyaml/YAMLParser.cpp @@ -0,0 +1,145 @@ + +#include +#include +#include +#include +#include "YAMLParser.h" + +/****************************************************************************/ +// Helpers + +/** Wrapper class for yaml_parser_t object + * + * Using a wrapper ensures all resources will be properly freed even + * if an exception is throw during parsing. + */ +class Parser final +{ +public: + Parser(std::istream & stream) + { + m_done = false; + m_hasEvent = false; + if (yaml_parser_initialize(&m_parser) == 0) { + throw std::runtime_error("YAML parser initialization failed"); + } + yaml_parser_set_input(&m_parser, readHandler, &stream); + } + + ~Parser() + { + if (m_hasEvent) { yaml_event_delete(&event); } + yaml_parser_delete(&m_parser); + } + + void parseNext() + { + if (m_hasEvent) { + yaml_event_delete(&event); + m_hasEvent = false; + } + if (yaml_parser_parse(&m_parser, &event) == 0) { + throw YAMLParser::ParseError( + m_parser.problem, + m_parser.problem_mark.line, m_parser.problem_mark.column, + m_parser.context, m_parser.context_mark.line + ); + } + m_hasEvent = true; + if (event.type == YAML_STREAM_END_EVENT) { m_done = true; } + } + + bool done() const { return m_done; } + +private: + static int readHandler(void * stream_ptr, unsigned char * buffer, + size_t size, size_t * size_read) + { + std::istream & stream = *reinterpret_cast(stream_ptr); + if (!stream.eof()) { + stream.read(reinterpret_cast(buffer), size); + *size_read = stream.gcount(); + } else { + *size_read = 0; + } + return stream.bad() ? 0 : 1; + } + +public: + yaml_event_t event; + +private: + bool m_done; ///< Set when end of stream was reached + yaml_parser_t m_parser; ///< libYAML parser object + bool m_hasEvent; ///< Set when @ref event holds an event +}; + +/****************************************************************************/ + +static const char * toStr(const yaml_char_t * str) { + if (str == NULL) { return ""; } + return reinterpret_cast(str); +} + +void YAMLParser::parse(std::istream & stream) +{ + auto parser = Parser(stream); + + while (!parser.done()) { + parser.parseNext(); + m_line = parser.event.start_mark.line; + m_column = parser.event.start_mark.column; + + switch (parser.event.type) { + case YAML_STREAM_START_EVENT: streamStart(); break; + case YAML_STREAM_END_EVENT: streamEnd(); break; + case YAML_DOCUMENT_START_EVENT: documentStart(); break; + case YAML_DOCUMENT_END_EVENT: documentEnd(); break; + case YAML_SEQUENCE_START_EVENT: { + const auto & data = parser.event.data.sequence_start; + sequenceStart(toStr(data.tag), toStr(data.anchor)); + break; + } + case YAML_SEQUENCE_END_EVENT: sequenceEnd(); break; + case YAML_MAPPING_START_EVENT: { + const auto & data = parser.event.data.mapping_start; + mappingStart(toStr(data.tag), toStr(data.anchor)); + break; + } + case YAML_MAPPING_END_EVENT: mappingEnd(); break; + case YAML_ALIAS_EVENT: { + const auto & data = parser.event.data.alias; + alias(toStr(data.anchor)); + break; + } + case YAML_SCALAR_EVENT: { + const auto & data = parser.event.data.scalar; + scalar(std::string(toStr(data.value), data.length), + toStr(data.tag), toStr(data.anchor)); + break; + } + default: + throw std::logic_error("YamlParser: unexpected event type"); + } + } +} + +/****************************************************************************/ + +YAMLParser::ParseError::ParseError(const std::string & what, size_t line, size_t col) +{ + std::ostringstream error; + error < +#include +#include +#include + +class YAMLParser +{ +public: + class ParseError : public std::runtime_error + { + public: + ParseError(const std::string & what, size_t line, size_t col, + const std::string & context = "", size_t ctx_line = -1) + : std::runtime_error(genErrMsg(what, line, col, context, ctx_line)) + {} + + private: + std::string static genErrMsg(const std::string & what, size_t line, size_t col, + const std::string & context, size_t ctx_line) + { + std::ostringstream error; + error << what + << " line " << line + 1 << " column " << col + 1 + << " " << context; + if (ctx_line != -1) { + error << " from line " << ctx_line + 1; + } + return error.str(); + } + }; + +public: + virtual ~YAMLParser() {} + virtual void parse(std::istream & stream); + +protected: + virtual void streamStart() = 0; + virtual void streamEnd() = 0; + virtual void documentStart() = 0; + virtual void documentEnd() = 0; + virtual void sequenceStart(const std::string & tag, const std::string & anchor) = 0; + virtual void sequenceEnd() = 0; + virtual void mappingStart(const std::string & tag, const std::string & anchor) = 0; + virtual void mappingEnd() = 0; + virtual void alias(const std::string & anchor) = 0; + virtual void scalar(const std::string & value, const std::string & tag, + const std::string & anchor) = 0; + + ParseError makeError(const std::string & what) const + { + return ParseError(what, m_line, m_column); + } +private: + size_t m_line; + size_t m_column; +}; + +#endif diff --git a/external/libyaml/yaml.hpp b/external/libyaml/yaml.hpp new file mode 100644 index 0000000000..4fbcd32b9c --- /dev/null +++ b/external/libyaml/yaml.hpp @@ -0,0 +1,270 @@ + + + + + + + +# NOTE: not finished. +# REFERENCES: +# https://gist.github.com/meffie/89d106a86b81c579c2b2a1895ffa18b0 +# https://github.com/yaml/libyaml +# https://www.wpsoftware.net/andrew/pages/libyaml.html +# https://pyyaml.org/wiki/LibYAML +# https://github.com/jbeder/yaml-cpp/wiki/Tutorial for yaml-cpp +#pragma once + +#include + + +namespace YAML { + +enum NodeType +{ + Null = 0, + Scalar, + Sequence, + Map +}; + +/****************************************************************************/ + +class YamlError : public std::runtime_error +{ +public: + YamlError(const std::string & what, size_t line, size_t col, + const std::string & context = "", size_t ctx_line = -1) + : std::runtime_error(genErrMsg(what, line, col, context, ctx_line)) + {} + +private: + std::string static genErrMsg(const std::string & what, size_t line, size_t col, + const std::string & context, size_t ctx_line) { + std::ostringstream error; + error << what + << " line " << line + 1 << " column " << col + 1 + << " " << context; + if (ctx_line != -1) { + error << " from line " << ctx_line + 1; + } + return error.str(); + } +}; + +/****************************************************************************/ + + +class Node : public std::vector { +public: + + + Node() { type_ = Null; } + ~Node() { } + //Node(const Node& rhs); ?? + //Node(Node&& rhs); ?? + explicit Node(NodeType type) { type_ = type;} + + NodeType::value Type() const { return type_; } + //bool IsDefined() const; + bool IsNull() const { return Type() == NodeType::Null; } + bool IsScalar() const { return Type() == NodeType::Scalar; } + bool IsSequence() const { return Type() == NodeType::Sequence; } + bool IsMap() const { return Type() == NodeType::Map; } + + // bool conversions + YAML_CPP_OPERATOR_BOOL() + bool operator!() const { return !IsDefined(); } + + // access + template + T as() const; + char as() { return (char)stoi(value_); } + short as() { return (short)stoi(value_); } + unsigned short as() { return (unsigned short)stoul(value_); } + int as() { return stoul(value_); } + unsigned int as() { return stoul(value_); } + long long as() { return stoll(value_); } + unsigned long long as() { return stoull(value_); } + float as() { return stof(value_); } + double as() { return stod(value_); } + + template + T as(const S& fallback) const { if (value_.empty()) return (T)fallback; return as();} + + + const std::string& Tag() const { return tag_; } + + + // assignment + //bool is(const Node& rhs) const; + //template + //Node& operator=(const T& rhs); + //Node& operator=(const Node& rhs); + + + // Set Node to undefined + void clear() override { tag_ = ""; value_ = ""; type_ = Null; defined_=false; std::vector::clear(); } + + // indexing + //template + //const Node operator[](const Key& key) const; + + //template + //Node operator[](const Key& key); + + //template + //bool remove(const Key& key); + + //const Node operator[](const Node& key) const; + //Node operator[](const Node& key); + bool remove(const Node& key); + + // map + //template + //void force_insert(const Key& key, const Value& value); + + void addScaler(const std::string& value, const std::string& tag="") { + Node n(NodeType::Scalar); + n.parent_ = this; + n.tag_ = tag; + n.value_ = value; + parent->push_back(n); + } + void beginSeq() { + Node n(NodeType::Sequence); + parent_->push_back(n) + } + void endSeq() { + parent_ = parent_->parent_; + } + void beginMap() { + Node n(NodeType::Map); + parent_->push_back(n) + } + void endMap() { + + parent_ = parent_->parent_; + } + +private: + std::string tag_; + std::string value_; + enum NodeType type_; + Node *parent_; + +}; + +/****************************************************************************/ +class YAML +{ +public: + static YAML::Node Load(std::string& yamlstring) { + YAML yaml(yamlstrng); + return yaml.root; + } + YAMLParser(std::string& yamlstring) { + stringstream stream(yamlstring); + initialize(stream); + } + YAMLParser(std::istream & stream) { + initialize(stream); + } + ~YAMLParser() + { + if (m_hasEvent) { yaml_event_delete(&event); } + yaml_parser_delete(&m_parser); + } + +private: + void initialize(std::istream & stream) { + m_done = false; + m_hasEvent = false; + if (yaml_parser_initialize(&m_parser) == 0) { + throw std::runtime_error("YAML parser initialization failed"); + } + yaml_parser_set_input(&m_parser, readHandler, &stream); + parse(); + } + + static int readHandler(void * stream_ptr, unsigned char * buffer, + size_t size, size_t * size_read) { + std::istream & stream = *reinterpret_cast(stream_ptr); + if (!stream.eof()) { + stream.read(reinterpret_cast(buffer), size); + *size_read = stream.gcount(); + } else { + *size_read = 0; + } + return stream.bad() ? 0 : 1; + } + + YAML::Node parse() { + yaml_event_t event; + YAML::Node* node = &m_root; + + m_done = false; + do { + if (m_hasEvent) { + yaml_event_delete(&event); + m_hasEvent = false; + } + if (yaml_parser_parse(&m_parser, &event) == 0) { + throw YAMLParser::ParseError( + m_parser.problem, + m_parser.problem_mark.line, m_parser.problem_mark.column, + m_parser.context, m_parser.context_mark.line + ); + } + m_hasEvent = true; + + switch(event.type) + { + case YAML_NO_EVENT: puts("No event!"); break; + /* Stream start/end */ + case YAML_STREAM_START_EVENT: break; + case YAML_STREAM_END_EVENT: m_done = true; break; + /* Block delimeters */ + case YAML_DOCUMENT_START_EVENT: m_root.clear(); node = &m_root; break; + case YAML_DOCUMENT_END_EVENT: break; + case YAML_SEQUENCE_START_EVENT: node = node->beginSeq(); break; + case YAML_SEQUENCE_END_EVENT: node = node->endSeq(); break; + case YAML_MAPPING_START_EVENT: node = node->beginMap(); break; + case YAML_MAPPING_END_EVENT: node = node->endMap() break; + /* Data */ + case YAML_ALIAS_EVENT: node->setAnchor(event.data.alias.anchor); break; + case YAML_SCALAR_EVENT: node->setScalar(event.data.scalar.value); break; // TODO: set tag + } + if(event.type != YAML_STREAM_END_EVENT) + yaml_event_delete(&event); + } while(!m_done); + yaml_event_delete(&event); + + stream.close(); + return m_root; + } + + + + YAML::Node m_root; + bool m_done; ///< Set when end of stream was reached + yaml_parser_t m_parser; ///< libYAML parser object + bool m_hasEvent; ///< Set when @ref event holds an event + }; + + + + +protected: + + ParseError makeError(const std::string & what) const + { + return ParseError(what, m_line, m_column); + } +private: + size_t m_line; + size_t m_column; +}; + + + +} // namespace diff --git a/src/nupic/engine/Spec.hpp b/src/nupic/engine/Spec.hpp index e24b1cd47a..99b65f96a8 100644 --- a/src/nupic/engine/Spec.hpp +++ b/src/nupic/engine/Spec.hpp @@ -47,20 +47,20 @@ class InputSpec { return !operator==(other); } std::string description; // description of input - + NTA_BasicType dataType; // declare type of input // width of buffer if fixed. 0 means variable. // If non-zero positive value it means this region was developed // to accept a fixed sized 1D array only. - UInt32 count; - + UInt32 count; + bool required; // true if input must be connected. - - bool regionLevel; // if true, this means this input can propagate its + + bool regionLevel; // if true, this means this input can propagate its // dimensions to/from the region's dimensions. - - bool isDefaultInput; // if True, assume this if input name not given + + bool isDefaultInput; // if True, assume this if input name not given // in functions involving inputs of a region. }; @@ -77,11 +77,11 @@ class OutputSpec { return !operator==(other); } std::string description; // description of output - + NTA_BasicType dataType; // The type of the output buffer. - size_t count; // Size, in number of elements. If size is fixed. - // If non-zero value it means this region + size_t count; // Size, in number of elements. If size is fixed. + // If non-zero value it means this region // was developed to output a fixed sized 1D array only. // if 0, call askImplForOutputDimensions() to get dimensions. @@ -108,9 +108,6 @@ class ParameterSpec { typedef enum { CreateAccess, ReadOnlyAccess, ReadWriteAccess } AccessMode; ParameterSpec() {} - /** - * @param defaultValue -- a JSON-encoded value - */ ParameterSpec(std::string description, NTA_BasicType dataType, size_t count, std::string constraints, std::string defaultValue, AccessMode accessMode); @@ -118,18 +115,22 @@ class ParameterSpec { inline bool operator!=(const ParameterSpec &other) const { return !operator==(other); } - std::string description; + std::string description; // what this parameter is about + + NTA_BasicType dataType; // any NTA_BaseType type + + size_t count; // 0 - means array of unknown size + // 1 - means scalar + // >1 - means array of fixed size + + std::string constraints; // 'bool' means 0 or 1 value + + std::string defaultValue; // JSON representation; + // i.e. 11024.56 for scaler + // [1,2,3] for arrays + // empty string means no default available. - // [open: current basic types are bytes/{u}int16/32/64, real32/64, BytePtr. Is - // this the right list? Should we have std::string, jsonstd::string?] - NTA_BasicType dataType; - // 1 = scalar; > 1 = array o fixed sized; 0 = array of unknown size - // TODO: should be size_t? Serialization issues? - size_t count; - std::string constraints; - std::string defaultValue; // JSON representation; empty std::string means - // parameter is required - AccessMode accessMode; + AccessMode accessMode; // any AccessMode enum }; class Spec { @@ -161,11 +162,11 @@ class Spec { // a value that applys to the count field in inputs, outputs, parameters. // It means that the field is an array and its size is not fixed. - static const int VARIABLE = 0; + static const int VARIABLE = 0; // a value that applys to the count field in inputs, outputs, parameters. // It means that the field not an array and has a single scaler value. - static const int SCALER = 1; + static const int SCALER = 1; }; diff --git a/src/nupic/engine/YAMLUtils.cpp b/src/nupic/engine/YAMLUtils.cpp index a2f6166ef1..69ad1ba9ad 100644 --- a/src/nupic/engine/YAMLUtils.cpp +++ b/src/nupic/engine/YAMLUtils.cpp @@ -20,13 +20,19 @@ * --------------------------------------------------------------------- */ + #define USE_LIBYAML 1 + #include #include #include #include #include #include // strlen +#ifdef USE_LIBYAML +#include +#else #include +#endif #include diff --git a/startupCodeBlocks.sh b/startupCodeBlocks.sh new file mode 100644 index 0000000000..6c07ecae5a --- /dev/null +++ b/startupCodeBlocks.sh @@ -0,0 +1,14 @@ + + +# for starting up CodeBlocks +# To install codeblocks on Umbuntu +# sudo add-apt-repository ppa:damien-moore/codeblocks-stable +# sudo apt update +# sudo apt install codeblocks codeblocks-contrib +# cd to repository +# ./startupCodeBlocks.sh +# +mkdir -p build/scripts +cd build/scripts +cmake -G "CodeBlocks - Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug ../.. +codeblocks nupic_code.cbp From 6365eca9af2a05f8b0156771290a017247a86a53 Mon Sep 17 00:00:00 2001 From: David Keeney Date: Wed, 18 Sep 2019 18:36:56 -0700 Subject: [PATCH 08/10] WIP checking in what I have so far. --- .clang-format | 2 + CMakeLists.txt | 5 +- external/CMakeLists.txt | 20 +- external/RapidYaml.cmake | 107 +++++++ external/YamlCppLib.cmake | 10 +- external/bootstrap.cmake | 3 +- external/libYaml.cmake | 49 +++ src/CMakeLists.txt | 12 +- src/htm/engine/RegionImplFactory.cpp | 11 +- src/htm/engine/YAMLUtils.cpp | 306 ------------------- src/htm/engine/YAMLUtils.hpp | 47 --- src/htm/ntypes/Scalar.cpp | 85 ------ src/htm/ntypes/Scalar.hpp | 61 ---- src/htm/ntypes/Value-RapidYaml.cpp | 368 +++++++++++++++++++++++ src/htm/ntypes/Value-libyaml.cpp | 368 +++++++++++++++++++++++ src/htm/ntypes/Value-yamlcpp.cpp | 308 +++++++++++++++++++ src/htm/ntypes/Value.cpp | 285 ------------------ src/htm/ntypes/Value.hpp | 238 +++++++++------ src/htm/regions/SPRegion.cpp | 1 - src/htm/regions/SPRegion.hpp | 1 + src/htm/regions/ScalarSensor.cpp | 18 +- src/htm/regions/TestNode.cpp | 1 - src/htm/regions/VectorFileEffector.cpp | 1 - src/htm/regions/VectorFileEffector.hpp | 2 +- src/htm/regions/VectorFileSensor.cpp | 1 - src/htm/regions/VectorFileSensor.hpp | 1 + src/test/CMakeLists.txt | 6 +- src/test/unit/ntypes/ValueMapTest.cpp | 116 +++++++ src/test/unit/regions/SPRegionTest.cpp | 2 - src/test/unit/regions/TMRegionTest.cpp | 2 - src/test/unit/regions/VectorFileTest.cpp | 2 - 31 files changed, 1508 insertions(+), 931 deletions(-) create mode 100644 external/RapidYaml.cmake create mode 100644 external/libYaml.cmake delete mode 100644 src/htm/engine/YAMLUtils.cpp delete mode 100644 src/htm/engine/YAMLUtils.hpp delete mode 100644 src/htm/ntypes/Scalar.cpp delete mode 100644 src/htm/ntypes/Scalar.hpp create mode 100644 src/htm/ntypes/Value-RapidYaml.cpp create mode 100644 src/htm/ntypes/Value-libyaml.cpp create mode 100644 src/htm/ntypes/Value-yamlcpp.cpp delete mode 100644 src/htm/ntypes/Value.cpp create mode 100644 src/test/unit/ntypes/ValueMapTest.cpp diff --git a/.clang-format b/.clang-format index 6fca7ffffb..64acd4ed81 100644 --- a/.clang-format +++ b/.clang-format @@ -2,4 +2,6 @@ Language: Cpp BasedOnStyle: LLVM DisableFormat: false +IndentWidth: 2 +ColumnLimit: 120 ... diff --git a/CMakeLists.txt b/CMakeLists.txt index f645cebab6..ba0b63646f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ # along with this program. If not, see http://www.gnu.org/licenses. # ----------------------------------------------------------------------------- -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.9) project(htm_core CXX) set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}") @@ -29,6 +29,7 @@ set($ENV{HTM_CPP} ${REPOSITORY_DIR}) # option(FORCE_CPP11 "Force compiler to use C++11 standard." OFF) option(FORCE_BOOST "Force compiler to install and use Boost." OFF) +set(YAML_PARSER "yamlcpp" CACHE STRING "Specify the YAML parser to use 'yamlcpp', 'RapidYaml', 'libyaml', default 'yamlcpp'." ) set(BINDING_BUILD "none" CACHE STRING "Specify the Binding to build 'Python2','Python3' or 'none', default 'none'." ) # Note: by setting the CXX environment variable, a non-default c++ compiler can be specified. @@ -77,6 +78,8 @@ message(STATUS "INTERNAL_LINKER_FLAGS_STR= ${INTERNAL_LINKER_FLAGS_STR}") message(STATUS "COMMON_COMPILER_DEFINITIONS_STR=${COMMON_COMPILER_DEFINITIONS_STR}") message(STATUS "COMMON_OS_LIBS = ${COMMON_OS_LIBS}") +add_definitions(-DYAML_PARSER_${YAML_PARSER}) + # Set up builds of external dependencies and get their exports. # (see individual external/*.cmake modules for exported settings and functions) include(external/bootstrap.cmake) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index fde0d5c6e0..1bb1783a6d 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -38,7 +38,7 @@ # - cmake ../.. # Note: on MSVC, run startupMSVC.bat after deleting the build folder. -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.9) project(htm_core CXX) set(CMAKE_VERBOSE_MAKEFILE OFF) @@ -78,12 +78,20 @@ FILE(REMOVE "${EXPORT_FILE_NAME}") # common include(common.cmake) - ################ -# Yaml-cpp -include(YamlCppLib.cmake) - - +# Yaml Parser +if (${YAML_PARSER} STREQUAL "yamlcpp") + # Yaml-cpp + include(YamlCppLib.cmake) + +elseif(${YAML_PARSER} STREQUAL "RapidYaml") + # Rapid YAML + include(RapidYaml.cmake) + +elseif(${YAML_PARSER} STREQUAL "libyaml") + # libyaml + include(libyaml.cmake) +endif() ################ # Eigen include(eigen.cmake) diff --git a/external/RapidYaml.cmake b/external/RapidYaml.cmake new file mode 100644 index 0000000000..032e85084b --- /dev/null +++ b/external/RapidYaml.cmake @@ -0,0 +1,107 @@ +# ----------------------------------------------------------------------------- +# HTM Community Edition of NuPIC +# Copyright (C) 2019, Numenta, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see http://www.gnu.org/licenses. +# ----------------------------------------------------------------------------- + +# Rapid YAML must be downloaded in four parts. +# - ryml from repository at https://github.com/biojppm/rapidyaml - the main library +# - c4core from repository at https://github.com/biojppm/c4core/ - the C library of utilities +# - cmake from repository at https://github.com/biojppm/cmake/ - Cmake scripts +# - debugbreak from repository at https://github.com/biojppm/debugbreak/ - debug break function +# There are links from rapidyaml to a specific branch of c4core in the extern folder of rapidyaml +# but the downloader does not follow it. So we need to do three downloads. + +message(STATUS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml.zip") +if( (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml.zip") + AND (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_c4core.zip") + AND (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_cmake.zip") + AND (EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_debugbreak.zip")) + set(URL1 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml.zip") + set(URL2 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_c4core.zip") + set(URL3 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_cmake.zip") + set(URL4 "${REPOSITORY_DIR}/build/ThirdParty/share/ryml_debugbreak.zip") +else() + #There are no releases. Use the masters. + set(URL1 https://github.com/biojppm/rapidyaml/archive/master.zip) + set(URL2 https://github.com/biojppm/c4core/archive/master.zip) + set(URL3 https://github.com/biojppm/cmake/archive/master.zip) + set(URL4 https://github.com/biojppm/debugbreak/archive/master.zip) +endif() + +message(STATUS "Obtaining RapidYAML from ${URL1}" ) +include(DownloadProject/DownloadProject.cmake) +download_project(PROJ ryml + PREFIX "${EP_BASE}/ryml" + URL ${URL1} + UPDATE_DISCONNECTED 1 + QUIET + ) +download_project(PROJ ryml_c4core + PREFIX "${EP_BASE}/ryml_c4core" + SOURCE_DIR "${ryml_SOURCE_DIR}/extern/c4core" + URL ${URL2} + UPDATE_DISCONNECTED 1 + QUIET + ) +download_project(PROJ ryml_cmake + PREFIX "${EP_BASE}/ryml_cmake" + SOURCE_DIR "${ryml_SOURCE_DIR}/extern/c4core/cmake" + URL ${URL3} + UPDATE_DISCONNECTED 1 + QUIET + ) +download_project(PROJ ryml_debugbreak + PREFIX "${EP_BASE}/ryml_debugbreak" + SOURCE_DIR "${ryml_SOURCE_DIR}/extern/c4core/extern/debugbreak" + URL ${URL4} + UPDATE_DISCONNECTED 1 + QUIET + ) + +#Need to enable Exceptions +set(c4path "${ryml_SOURCE_DIR}/extern/c4core/src/c4") +file(RENAME "${c4path}/config.hpp" "${c4path}/config1.hpp") +file(READ "${c4path}/config1.hpp" content1) +string(REPLACE "//#define C4_ERROR_THROWS_EXCEPTION" + "#define C4_ERROR_EXCEPTIONS_ENABLED\n#define C4_ERROR_THROWS_EXCEPTION" + content + "${content1}") +file(WRITE "${c4path}/config.hpp" "${content}") + +file(RENAME "${c4path}/error.cpp" "${c4path}/error1.cpp") +file(READ "${c4path}/error1.cpp" content2) +string(REPLACE "if(s_error_flags & (ON_ERROR_LOG|ON_ERROR_CALLBACK))" + "if(s_error_flags & (ON_ERROR_LOG|ON_ERROR_CALLBACK|ON_ERROR_THROW))" + content1 + "${content2}") +string(REPLACE "throw Exception(ss.c_str())" + "throw Exception(buf)" + content + "${content1}") +file(WRITE "${c4path}/error.cpp" "${content}") + +set(C4_EXCEPTIONS_ENABLED ON) +set(C4_ERROR_THROWS_EXCEPTION ON) +add_subdirectory(${ryml_SOURCE_DIR} ${ryml_BINARY_DIR}) + +set(yaml_INCLUDE_DIRS "${ryml_SOURCE_DIR}/src@@@${ryml_SOURCE_DIR}/extern/c4core/src@@@${ryml_SOURCE_DIR}/extern/c4core/extern") +if (MSVC) + set(yaml_LIBRARIES "${ryml_BINARY_DIR}$<$:/Release/ryml.lib>$<$:/Debug/ryml.lib>") +else() + set(yaml_LIBRARIES ${ryml_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}ryml${CMAKE_STATIC_LIBRARY_SUFFIX}) +endif() +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_INCLUDE_DIRS@@@${yaml_INCLUDE_DIRS}\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_LIBRARIES@@@${yaml_LIBRARIES}\n") + diff --git a/external/YamlCppLib.cmake b/external/YamlCppLib.cmake index f3e977f07e..98f365261d 100644 --- a/external/YamlCppLib.cmake +++ b/external/YamlCppLib.cmake @@ -40,12 +40,12 @@ set(YAML_CPP_BUILD_CONTRIB OFF CACHE BOOL "prevent contrib modules" FORCE) set(MSVC_SHARED_RT ON CACHE BOOL "Use compile option /MD rather than /MT" FORCE) add_subdirectory(${yaml-cpp_SOURCE_DIR} ${yaml-cpp_BINARY_DIR}) -set(yaml-cpp_INCLUDE_DIRS ${yaml-cpp_SOURCE_DIR}/include) +set(yaml_INCLUDE_DIRS ${yaml-cpp_SOURCE_DIR}/include) if (MSVC) - set(yaml-cpp_LIBRARIES "${yaml-cpp_BINARY_DIR}$<$:/Release/libyaml-cppmd.lib>$<$:/Debug/libyaml-cppmdd.lib>") + set(yaml_LIBRARIES "${yaml-cpp_BINARY_DIR}$<$:/Release/libyaml-cppmd.lib>$<$:/Debug/libyaml-cppmdd.lib>") else() - set(yaml-cpp_LIBRARIES ${yaml-cpp_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(yaml_LIBRARIES ${yaml-cpp_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}yaml-cpp${CMAKE_STATIC_LIBRARY_SUFFIX}) endif() -FILE(APPEND "${EXPORT_FILE_NAME}" "yaml-cpp_INCLUDE_DIRS@@@${yaml-cpp_SOURCE_DIR}/include\n") -FILE(APPEND "${EXPORT_FILE_NAME}" "yaml-cpp_LIBRARIES@@@${yaml-cpp_LIBRARIES}\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_INCLUDE_DIRS@@@${yaml-cpp_SOURCE_DIR}/include\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_LIBRARIES@@@${yaml_LIBRARIES}\n") diff --git a/external/bootstrap.cmake b/external/bootstrap.cmake index 3beba401b5..d585dc67fd 100644 --- a/external/bootstrap.cmake +++ b/external/bootstrap.cmake @@ -34,6 +34,7 @@ execute_process(COMMAND ${CMAKE_COMMAND} -D CMAKE_INSTALL_PREFIX=. -D NEEDS_BOOST:BOOL=${NEEDS_BOOST} -D BINDING_BUILD:STRING=${BINDING_BUILD} + -D YAML_PARSER:STRING=${YAML_PARSER} -D CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -D REPOSITORY_DIR=${REPOSITORY_DIR} ../../external @@ -91,7 +92,7 @@ FOREACH(line ${lines}) message(STATUS " ${name} = ${${name}}") ENDFOREACH() set(EXTERNAL_INCLUDES - ${yaml-cpp_INCLUDE_DIRS} + ${yaml_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${eigen_INCLUDE_DIRS} ${mnist_INCLUDE_DIRS} diff --git a/external/libYaml.cmake b/external/libYaml.cmake new file mode 100644 index 0000000000..176fc133fd --- /dev/null +++ b/external/libYaml.cmake @@ -0,0 +1,49 @@ +# ----------------------------------------------------------------------------- +# Numenta Platform for Intelligent Computing (NuPIC) +# Copyright (C) 2019, Numenta, Inc. Unless you have purchased from +# Numenta, Inc. a separate commercial license for this software code, the +# following terms and conditions apply: +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero Public License for more details. +# +# You should have received a copy of the GNU Affero Public License +# along with this program. If not, see http://www.gnu.org/licenses. +# +# http://numenta.org/licenses/ +# ----------------------------------------------------------------------------- +# This downloads and builds the libyaml library. +# +if(EXISTS ${REPOSITORY_DIR}/build/ThirdParty/share/libyaml.zip + set(URL ${REPOSITORY_DIR}/build/ThirdParty/share/libyaml.zip) +else() + set(URL "https://github.com/yaml/libyaml/archive/master.zip") +endif() + +message(STATUS "Obtaining libyaml") +include(DownloadProject/DownloadProject.cmake) +download_project(PROJ libyaml + PREFIX ${EP_BASE}/libyaml + URL ${URL} + GIT_SHALLOW ON + UPDATE_DISCONNECTED 1 + QUIET + ) + +add_subdirectory(${libyaml_SOURCE_DIR} ${libyaml_BINARY_DIR}) + +set(yaml_INCLUDE_DIRS ${libyaml_SOURCE_DIR}/include) +if (MSVC) + set(yaml_LIBRARIES "${libyaml_BINARY_DIR}$<$:/Release/liblibyamlmd.lib>$<$:/Debug/liblibyamlmdd.lib>") +else() + set(yaml_LIBRARIES ${libyaml_BINARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}libyaml${CMAKE_STATIC_LIBRARY_SUFFIX}) +endif() +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_INCLUDE_DIRS@@@${yaml_SOURCE_DIR}\n") +FILE(APPEND "${EXPORT_FILE_NAME}" "yaml_LIBRARIES@@@${yaml_LIBRARIES}\n") + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6985f5a49f..db055f257e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -112,8 +112,6 @@ set(engine_files htm/engine/RegisteredRegionImplCpp.hpp htm/engine/Spec.cpp htm/engine/Spec.hpp - htm/engine/YAMLUtils.cpp - htm/engine/YAMLUtils.hpp htm/engine/Watcher.cpp htm/engine/Watcher.hpp ) @@ -127,9 +125,9 @@ set(ntypes_files htm/ntypes/BasicType.hpp htm/ntypes/Collection.hpp htm/ntypes/Dimensions.hpp - htm/ntypes/Scalar.cpp - htm/ntypes/Scalar.hpp - htm/ntypes/Value.cpp + htm/ntypes/Value-yamlcpp.cpp + htm/ntypes/Value-rapidyaml.cpp + htm/ntypes/Value-libyaml.cpp htm/ntypes/Value.hpp ) @@ -263,7 +261,7 @@ endif() # set(src_combined_htmcore_source_archives ${src_lib_static} - ${yaml-cpp_LIBRARIES} + ${yaml_LIBRARIES} ${Boost_LIBRARIES} ${common_LIBRARIES} ) @@ -283,7 +281,7 @@ else() # Note that these libraries must already exist (from depenancies) set(all_object_locations) set(src_externLibs - ${yaml-cpp_LIBRARIES} + ${yaml_LIBRARIES} ${Boost_LIBRARIES} ${common_LIBRARIES} ) diff --git a/src/htm/engine/RegionImplFactory.cpp b/src/htm/engine/RegionImplFactory.cpp index 2be4085688..a697852504 100644 --- a/src/htm/engine/RegionImplFactory.cpp +++ b/src/htm/engine/RegionImplFactory.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -112,9 +111,8 @@ RegionImpl *RegionImplFactory::createRegionImpl(const std::string nodeType, Region *region) { RegionImpl *impl = nullptr; - std::shared_ptr& ns = getSpec(nodeType); - ValueMap vm = YAMLUtils::toValueMap(nodeParams.c_str(), ns->parameters, - nodeType, region->getName()); + ValueMap vm; + vm.parse(nodeParams); if (regionTypeMap.find(nodeType) != regionTypeMap.end()) { impl = regionTypeMap[nodeType]->createRegionImpl(vm, region); @@ -124,9 +122,8 @@ RegionImpl *RegionImplFactory::createRegionImpl(const std::string nodeType, // If the parameter 'dim' was defined, parse that out as a global parameter. if (vm.contains("dim")) { - std::shared_ptr dim = vm.getArray("dim"); - Dimensions d(dim->asVector()); - impl->setDimensions(d); + std::vector dim = vm["dim"].asVector(); + impl->setDimensions(dim); } return impl; diff --git a/src/htm/engine/YAMLUtils.cpp b/src/htm/engine/YAMLUtils.cpp deleted file mode 100644 index feb4e65e95..0000000000 --- a/src/htm/engine/YAMLUtils.cpp +++ /dev/null @@ -1,306 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -#include -#include -#include -#include -#include -#include // strlen -#include - -#include - -namespace htm { -namespace YAMLUtils { - -/* - * These functions are used internally by toValue and toValueMap - */ -static void _toScalar(const YAML::Node& node, std::shared_ptr& s); -static void _toArray(const YAML::Node& node, std::shared_ptr& a); -static Value toValue(const YAML::Node& node, NTA_BasicType dataType); - -static void _toScalar(const YAML::Node &node, std::shared_ptr &s) { - NTA_CHECK(node.Type() == YAML::NodeType::Scalar); - switch (s->getType()) { - case NTA_BasicType_Byte: - // We should have already detected this and gone down the string path - NTA_THROW << "Internal error: attempting to convert YAML string to scalar of type Byte"; - break; - case NTA_BasicType_UInt16: - s->value.uint16 = node.as(); - break; - case NTA_BasicType_Int16: - s->value.int16 = node.as(); - break; - case NTA_BasicType_UInt32: - s->value.uint32 = node.as(); - break; - case NTA_BasicType_Int32: - s->value.int32 = node.as(); - break; - case NTA_BasicType_UInt64: - s->value.uint64 = node.as(); - break; - case NTA_BasicType_Int64: - s->value.int64 = node.as(); - break; - case NTA_BasicType_Real32: - s->value.real32 = node.as(); - break; - case NTA_BasicType_Real64: - s->value.real64 = node.as(); - break; - case NTA_BasicType_Bool: - s->value.boolean = node.as(); - break; - case NTA_BasicType_Handle: - NTA_THROW << "Attempt to specify a YAML value for a scalar of type Handle"; - break; - default: - // should not happen - const std::string val = node.as(); - NTA_THROW << "Unknown data type " << s->getType() << " for yaml node '" << val << "'"; - } -} - -static void _toArray(const YAML::Node& node, std::shared_ptr& a) { - NTA_CHECK(node.Type() == YAML::NodeType::Sequence); - - a->allocateBuffer(node.size()); - void *buffer = a->getBuffer(); - - for (size_t i = 0; i < node.size(); i++) { - const YAML::Node &item = node[i]; - NTA_CHECK(item.Type() == YAML::NodeType::Scalar); - switch (a->getType()) { - case NTA_BasicType_Byte: - // We should have already detected this and gone down the string path - NTA_THROW << "Internal error: attempting to convert YAML string to array " - "of type Byte"; - break; - case NTA_BasicType_UInt16: - ((UInt16*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Int16: - ((Int16*)buffer)[i] = item.as(); - break; - case NTA_BasicType_UInt32: - ((UInt32*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Int32: - ((Int32*)buffer)[i] = item.as(); - break; - case NTA_BasicType_UInt64: - ((UInt64*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Int64: - ((Int64*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Real32: - ((Real32*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Real64: - ((Real64*)buffer)[i] = item.as(); - break; - case NTA_BasicType_Bool: - ((bool*)buffer)[i] = item.as(); - break; - default: - // should not happen - NTA_THROW << "Unknown data type " << a->getType(); - } - } -} - -static Value toValue(const YAML::Node &node, NTA_BasicType dataType) { - if (node.Type() == YAML::NodeType::Map || - node.Type() == YAML::NodeType::Null) { - NTA_THROW << "YAML string does not represent a value."; - } - if (node.Type() == YAML::NodeType::Scalar) { - if (dataType == NTA_BasicType_Byte) { - // node >> *str; - const std::string val = node.as(); - Value v(val); - return v; - } else { - std::shared_ptr s(new Scalar(dataType)); - _toScalar(node, s); - Value v(s); - return v; - } - } else { - // array - std::shared_ptr a(new Array(dataType)); - _toArray(node, a); - Value v(a); - return v; - } -} - -/* - * For converting default values specified in nodespec string - */ -Value toValue(const std::string& yamlstring, NTA_BasicType dataType) -{ - // TODO -- return value? exceptions? - const YAML::Node doc = YAML::Load(yamlstring); - return toValue(doc, dataType); -} - -/* - * For converting param specs for Regions and LinkPolicies - */ -ValueMap toValueMap(const char *yamlstring, - Collection ¶meters, - const std::string &nodeType, - const std::string ®ionName) { - - ValueMap vm; - - // special values that applies to all regions. - // NOTE: these values are used in RegionImplFactory to set region impl. - ParameterSpec dim_spec("Buffer dimensions for region's global dimensions. " - "Syntax: {dim: [2,3]}", // description - NTA_BasicType_UInt32, - 0, // elementCount (an array of unknown size) - "", // constraints - "", // defaultValue - ParameterSpec::ReadWriteAccess); - - - std::string paddedstring(yamlstring); - // TODO: strip white space to determine if empty - bool empty = (paddedstring.size() == 0); - - // TODO: utf-8 compatible? - const YAML::Node doc = YAML::Load(paddedstring); - if(!empty) { - // A ValueMap is specified as a dictionary - if (doc.Type() != YAML::NodeType::Map) { - std::string ys(yamlstring); - if (ys.size() > 30) { - ys = ys.substr(0, 30) + "..."; - } - NTA_THROW - << "YAML string '" << ys - << "' does not not specify a dictionary of key-value pairs. " - << "Region and Link parameters must be specified as a dictionary"; - } - } - // Grab each value out of the YAML dictionary and put into the ValueMap - // if it is allowed by the nodespec. - for (auto i = doc.begin(); i != doc.end(); i++) - { - ParameterSpec ps; - - const auto key = i->first.as(); - if (key == "dim") - ps = dim_spec; - else { - if (!parameters.contains(key)) - { - std::stringstream ss; - for (UInt j = 0; j < parameters.getCount(); j++){ - ss << " " << parameters.getByIndex(j).first << "\n"; - } - - if (nodeType == std::string("")) { - NTA_THROW << "Unknown parameter '" << key << "'\n" - << "Valid parameters are:\n" << ss.str(); - } else { - NTA_CHECK(regionName != std::string("")); - NTA_THROW << "Unknown parameter '" << key << "' for region '" - << regionName << "' of type '" << nodeType << "'\n" - << "Valid parameters are:\n" - << ss.str(); - } - } - ps = parameters.getByName(key); // makes a copy of ParameterSpec - } - if (vm.contains(key)) - NTA_THROW << "Parameter '" << key << "' specified more than once in YAML document"; - try - { - if (ps.accessMode == ParameterSpec::ReadOnlyAccess) { - NTA_THROW << "Parameter '" << key << "'. This is ReadOnly access. Cannot be set."; - } - Value v = toValue(i->second, ps.dataType); - if (v.isScalar() && ps.count != 1) - { - NTA_THROW << "Parameter '" << key << "'. Bad value in runtime parameters. Expected array value but got scalar value"; - } - if (!v.isScalar() && ps.count == 1) - { - NTA_THROW << "Parameter '" << key << "'. Bad value in runtime parameters. Expected scalar value but got array value"; - } - vm.add(key, v); - } catch (std::runtime_error &e) { - NTA_THROW << "Unable to set parameter '" << key << "'. " << e.what(); - } - } //end for - - // Populate ValueMap with default values if they were not specified in the YAML dictionary. - for (size_t i = 0; i < parameters.getCount(); i++) - { - const std::pair& item = parameters.getByIndex(i); - if (!vm.contains(item.first)) - { - const ParameterSpec & ps = item.second; - if (ps.defaultValue != "") - { - // TODO: This check should be uncommented after dropping NuPIC 1.x nodes (which don't comply) //FIXME try this - // if (ps.accessMode != ParameterSpec::CreateAccess) - // { - // NTA_THROW << "Default value for non-create parameter: " << item.first; - // } - - try { -#ifdef YAMLDEBUG - NTA_DEBUG << "Adding default value '" << ps.defaultValue - << "' to parameter " << item.first << " of type " - << BasicType::getName(ps.dataType) << " count " << ps.count; -#endif - // NOTE: this can handle both scalers and arrays - // Arrays MUST be in Yaml sequence format even if one element. - // i.e. [1,2,3] - Value v = toValue(ps.defaultValue, ps.dataType); - if (v.isScalar() && ps.count != 1) - { - NTA_THROW << "Parameter '" << item.first << "'. Bad default value in spec. Expected array value but got scalar value"; - } - if (!v.isScalar() && ps.count == 1) - { - NTA_THROW << "Parameter '" << item.first << "'. Bad default value in spec. Expected scalar value but got array value"; - } - vm.add(item.first, v); - } catch (...) { - NTA_THROW << "Unable to set default value for item '" << item.first - << "' of datatype " << BasicType::getName(ps.dataType) - << " with value '" << ps.defaultValue << "'"; - } - } - } - } - - return vm; -} - -} // end of YAMLUtils namespace -} // end of namespace htm diff --git a/src/htm/engine/YAMLUtils.hpp b/src/htm/engine/YAMLUtils.hpp deleted file mode 100644 index bb5dad7838..0000000000 --- a/src/htm/engine/YAMLUtils.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ -#ifndef NTA_YAML_HPP -#define NTA_YAML_HPP - -#include -#include -#include -#include - -namespace htm -{ - - namespace YAMLUtils - { - /* - * For converting default values - * @param yamlstring - is a string in parsable format (eg. '[1 2 3]' ), not a path to yaml file! - */ - Value toValue(const std::string& yamlstring, NTA_BasicType dataType); - -/* - * For converting param specs for Regions and LinkPolicies - */ -ValueMap toValueMap(const char *yamlstring, - Collection ¶meters, - const std::string &nodeType = "", - const std::string ®ionName = ""); - -} // namespace YAMLUtils -} // namespace htm - -#endif // NTA_YAML_HPP diff --git a/src/htm/ntypes/Scalar.cpp b/src/htm/ntypes/Scalar.cpp deleted file mode 100644 index f4144cba22..0000000000 --- a/src/htm/ntypes/Scalar.cpp +++ /dev/null @@ -1,85 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -/** @file - * Implementation of the Scalar class - * - * A Scalar object is an instance of an NTA_BasicType -- essentially a union - * It is used internally in the conversion of YAML strings to C++ objects. - */ - -#include -#include - -using namespace htm; - -Scalar::Scalar(NTA_BasicType theTypeParam) { - theType_ = theTypeParam; - value.uint64 = 0; -} - -NTA_BasicType Scalar::getType() { return theType_; } - -// gcc 4.2 complains about the template specializations -// in a different namespace if we don't include this -namespace htm { - -template <> Handle Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Handle); - return value.handle; -} -template <> Byte Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Byte); - return value.byte; -} -template <> UInt16 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_UInt16); - return value.uint16; -} -template <> Int16 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Int16); - return value.int16; -} -template <> UInt32 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_UInt32); - return value.uint32; -} -template <> Int32 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Int32); - return value.int32; -} -template <> UInt64 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_UInt64); - return value.uint64; -} -template <> Int64 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Int64); - return value.int64; -} -template <> Real32 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Real32); - return value.real32; -} -template <> Real64 Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Real64); - return value.real64; -} -template <> bool Scalar::getValue() const { - NTA_CHECK(theType_ == NTA_BasicType_Bool); - return value.boolean; -} -} // namespace htm diff --git a/src/htm/ntypes/Scalar.hpp b/src/htm/ntypes/Scalar.hpp deleted file mode 100644 index 183dc053b6..0000000000 --- a/src/htm/ntypes/Scalar.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -/** @file - * Definitions for the Scalar class - * - * A Scalar object is an instance of an NTA_BasicType -- essentially a union - * It is used internally in the conversion of YAML strings to C++ objects. - */ - -#ifndef NTA_SCALAR_HPP -#define NTA_SCALAR_HPP - -#include -#include // temporary, while implementation is in hpp -#include - -namespace htm { -class Scalar { -public: - Scalar(NTA_BasicType theTypeParam); - - NTA_BasicType getType(); - - template T getValue() const; - - union { - Handle handle; - Byte byte; - Int16 int16; - UInt16 uint16; - Int32 int32; - UInt32 uint32; - Int64 int64; - UInt64 uint64; - Real32 real32; - Real64 real64; - bool boolean; - } value; - -private: - NTA_BasicType theType_; -}; - -} // namespace htm - -#endif // NTA_SCALAR_HPP diff --git a/src/htm/ntypes/Value-RapidYaml.cpp b/src/htm/ntypes/Value-RapidYaml.cpp new file mode 100644 index 0000000000..2b7db0791c --- /dev/null +++ b/src/htm/ntypes/Value-RapidYaml.cpp @@ -0,0 +1,368 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2019, Numenta, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ + +#ifdef YAML_PARSER_RapidYaml + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace htm; +using namespace ryml; +using namespace c4; + +// some static helper functions +// The convert to and from the special string types used by ryml +static csubstr _S(const std::string &s) { return to_csubstr(s.c_str()); } +static std::string S_(csubstr s) { return std::string(s.data(), s.size()); } + +// Error handling. This C function bootstraps into C++ to throw an Exception. +static void handleErrors_c(const char *msg, size_t msg_len, void *user_data) { + ((ValueMap *)user_data)->handleErrors(std::string(msg, msg_len)); +} + +// This is just so ryml.hpp declarations are hidden from our .hpp +struct htm::ValueMap::OpaqueTree { + c4::yml::NodeRef node; + c4::yml::Tree t; +}; +struct htm::ValueMap::iterator::OpaqueIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + ValueMap vm; +}; +struct htm::ValueMap::const_iterator::OpaqueConstIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + const ValueMap vm; +}; + + + + +// Constructor +ValueMap::ValueMap() { + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root +} + +// Parse YAML or JSON string document into the tree. +ValueMap &ValueMap::parse(const std::string &yaml_string) { + set_error_flags(ON_ERROR_LOG|ON_ERROR_THROW); + + // This must be the top node so set the callbacks on this. + //Callbacks const &cb_orig = get_callbacks(); + //Callbacks cb_new = cb_orig; + //cb_new.m_error = handleErrors_c; + //cb_new.m_user_data = this; + //set_callbacks(cb_new); + + // Clear the tree, just in case there has been something already there + // from a previous parse. + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root + + ryml::parse(_S(yaml_string), doc_->node); // populate tree at this node + + std::cout << "Tree: " << doc_->t; + + // If the value parsed is just a scalar, then the top node is set to a sequence + // and the first child is the scalar. If a sequence was explictly specified with [ ] + // or a leading '- ' and there is only one element we cannot tell the difference. + // So, we have to see what the yaml_string starts with. If it is a '[' or '- ' then leave + // it as a Sequence. If not, point to the single scalar child as the top node. + if (doc_->node.is_seq() && doc_->node.num_children() == 1) { + const char *p = yaml_string.c_str(); + while (isspace(*p)) p++; // skip over leading spaces + if (*p == '[' || (*p == '-' && *(p + 1) == ' ')) { + // leave top node as a sequence + } else { + // make the top node a Scaler + doc_->node = doc_->node.first_child(); + } + } + return *this; +} + + +// checking content of a parameter +//enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category ValueMap::getCategory() const { + if (doc_->node.is_map()) return ValueMap::Category::Map; + if (doc_->node.is_seq()) return ValueMap::Category::Sequence; + if (doc_->node.has_val()) return ValueMap::Category::Scalar; + return ValueMap::Category::Empty; +} + +bool ValueMap::contains(const std::string &key) const {return doc_->node[_S(key)].valid();} + +bool ValueMap::isScalar() const { return doc_->node.has_val(); } +bool ValueMap::isSequence() const { return doc_->node.is_seq(); } +bool ValueMap::isMap() const { return doc_->node.is_map(); } +bool ValueMap::isEmpty() const { return getCategory() == ValueMap::Category::Empty; } + +size_t ValueMap::size() const { return doc_->node.num_children(); } + +// Accessing members of a map +ValueMap ValueMap::operator[](const std::string &key) { + ValueMap vm = *this; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} +const ValueMap ValueMap::operator[](const std::string &key) const { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_map()) << "This is not a map."; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} + +// accessing members of a sequence +ValueMap ValueMap::operator[](size_t index) { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_seq()) << "This is not a sequence."; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '"+std::to_string(index)+"' is out of bounds."; + } + return vm; +} +const ValueMap ValueMap::operator[](size_t index) const { + ValueMap vm = *this; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '" + std::to_string(index) + "' is out of bounds."; + } + return vm; +} + +std::string ValueMap::str() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + return S_(r.val()); + } else { + NTA_THROW << "Value not found."; + } +} + +// Return a value converted to the specified type T. +template T ValueMap::as() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + T v; + r >> v; + return v; + } else { + NTA_THROW << "Value not found."; + } +} +// explicit instantiation. Can only be used with these types. +template int8_t ValueMap::as() const; +template int16_t ValueMap::as() const; +template uint16_t ValueMap::as() const; +template int32_t ValueMap::as() const; +template uint32_t ValueMap::as() const; +template int64_t ValueMap::as() const; +template uint64_t ValueMap::as() const; +template bool ValueMap::as() const; +template float ValueMap::as() const; +template double ValueMap::as() const; +template std::string ValueMap::as() const; + + +std::string ValueMap::key() const { + NodeRef r = doc_->node; + return (r.has_key()) ? S_(r.key()) : ""; +} + + +// Iterator +ValueMap::iterator ValueMap::iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last)? NodeRef(): ptr_->node.next_sibling(); + return *this; +} +ValueMap::iterator ValueMap::iterator::operator++(int junk) {return operator++(); } +ValueMap &ValueMap::iterator::operator*() { ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +ValueMap *ValueMap::iterator::operator->() { ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::iterator::operator==(const iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::iterator::operator!=(const iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::iterator ValueMap::begin() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } + else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} +ValueMap::iterator ValueMap::end() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + +ValueMap::const_iterator ValueMap::const_iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last) ? NodeRef() : ptr_->node.next_sibling(); + return *this; +} +ValueMap::const_iterator ValueMap::const_iterator::operator++(int junk) { return operator++();} +const ValueMap &ValueMap::const_iterator::operator*() {ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +const ValueMap *ValueMap::const_iterator::operator->() {ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::const_iterator::operator==(const const_iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::const_iterator::operator!=(const const_iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::const_iterator ValueMap::cbegin() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} + +ValueMap::const_iterator ValueMap::cend() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + + +std::vector ValueMap::getKeys() const { + std::vector v; + NodeRef root = doc_->t.rootref(); + for (NodeRef n : root.children()) { + if (n.has_key()) + v.push_back(S_(n.key())); + } + return v; +} + + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } + } +} + + +static void to_json(std::ostream& f, const htm::ValueMap& vm) { + bool first = true; + switch (vm.getCategory()) { + case ValueMap::Empty: + return; + case ValueMap::Scalar: + f << '"'; + escape_json(f, vm.as()); + f << '"'; + break; + case ValueMap::Sequence: + f << "["; + for (size_t i = 0; i < vm.size(); i++) { + if (!first) + f << ", "; + first = false; + to_json(f, vm[i]); + } + f << "]"; + break; + case ValueMap::Map: + f << "{"; + for (auto it = vm.cbegin(); it != vm.cend(); ++it) { + if (!first) + f << ", "; + first = false; + f << it->key() << ": "; + to_json(f, *it); + } + f << "}"; + break; + } +} + +std::string ValueMap::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); +} + +std::string ValueMap::to_yaml() const { + std::stringstream ss; + ss << doc_->t; + return ss.str(); +} + + +std::ostream &operator<<(std::ostream &f, const htm::ValueMap &vm) { + f << vm.to_json(); + return f; +} + +void ValueMap::handleErrors(const std::string &msg) +{ + throw htm::LoggingException(__FILE__, __LINE__) << "Yaml parse error: " + msg; +} + +#endif // YAML_PARSER_RapidYaml \ No newline at end of file diff --git a/src/htm/ntypes/Value-libyaml.cpp b/src/htm/ntypes/Value-libyaml.cpp new file mode 100644 index 0000000000..d90a9590a4 --- /dev/null +++ b/src/htm/ntypes/Value-libyaml.cpp @@ -0,0 +1,368 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2013, Numenta, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ + +#ifdef YAML_PARSER_libyaml + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace htm; +using namespace ryml; +using namespace c4; + +// some static helper functions +// The convert to and from the special string types used by ryml +static csubstr _S(const std::string &s) { return to_csubstr(s.c_str()); } +static std::string S_(csubstr s) { return std::string(s.data(), s.size()); } + +// Error handling. This C function bootstraps into C++ to throw an Exception. +static void handleErrors_c(const char *msg, size_t msg_len, void *user_data) { + ((ValueMap *)user_data)->handleErrors(std::string(msg, msg_len)); +} + +// This is just so ryml.hpp declarations are hidden from our .hpp +struct htm::ValueMap::OpaqueTree { + c4::yml::NodeRef node; + c4::yml::Tree t; +}; +struct htm::ValueMap::iterator::OpaqueIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + ValueMap vm; +}; +struct htm::ValueMap::const_iterator::OpaqueConstIterator { + c4::yml::NodeRef node; + c4::yml::NodeRef last; + const ValueMap vm; +}; + + + + +// Constructor +ValueMap::ValueMap() { + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root +} + +// Parse YAML or JSON string document into the tree. +ValueMap &ValueMap::parse(const std::string &yaml_string) { + set_error_flags(ON_ERROR_LOG|ON_ERROR_THROW); + + // This must be the top node so set the callbacks on this. + //Callbacks const &cb_orig = get_callbacks(); + //Callbacks cb_new = cb_orig; + //cb_new.m_error = handleErrors_c; + //cb_new.m_user_data = this; + //set_callbacks(cb_new); + + // Clear the tree, just in case there has been something already there + // from a previous parse. + doc_ = std::make_shared(); // creates an empty Tree + doc_->node = doc_->t.rootref(); // Node corresponding to the Tree root + + ryml::parse(_S(yaml_string), doc_->node); // populate tree at this node + + std::cout << "Tree: " << doc_->t; + + // If the value parsed is just a scalar, then the top node is set to a sequence + // and the first child is the scalar. If a sequence was explictly specified with [ ] + // or a leading '- ' and there is only one element we cannot tell the difference. + // So, we have to see what the yaml_string starts with. If it is a '[' or '- ' then leave + // it as a Sequence. If not, point to the single scalar child as the top node. + if (doc_->node.is_seq() && doc_->node.num_children() == 1) { + const char *p = yaml_string.c_str(); + while (isspace(*p)) p++; // skip over leading spaces + if (*p == '[' || (*p == '-' && *(p + 1) == ' ')) { + // leave top node as a sequence + } else { + // make the top node a Scaler + doc_->node = doc_->node.first_child(); + } + } + return *this; +} + + +// checking content of a parameter +//enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category ValueMap::getCategory() const { + if (doc_->node.is_map()) return ValueMap::Category::Map; + if (doc_->node.is_seq()) return ValueMap::Category::Sequence; + if (doc_->node.has_val()) return ValueMap::Category::Scalar; + return ValueMap::Category::Empty; +} + +bool ValueMap::contains(const std::string &key) const {return doc_->node[_S(key)].valid();} + +bool ValueMap::isScalar() const { return doc_->node.has_val(); } +bool ValueMap::isSequence() const { return doc_->node.is_seq(); } +bool ValueMap::isMap() const { return doc_->node.is_map(); } +bool ValueMap::isEmpty() const { return getCategory() == ValueMap::Category::Empty; } + +size_t ValueMap::size() const { return doc_->node.num_children(); } + +// Accessing members of a map +ValueMap ValueMap::operator[](const std::string &key) { + ValueMap vm = *this; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} +const ValueMap ValueMap::operator[](const std::string &key) const { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_map()) << "This is not a map."; + vm.doc_->node = doc_->node[_S(key)]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "No value found for key '"+key+"'."; + } + return vm; +} + +// accessing members of a sequence +ValueMap ValueMap::operator[](size_t index) { + ValueMap vm = *this; + NTA_CHECK(doc_->node.is_seq()) << "This is not a sequence."; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '"+std::to_string(index)+"' is out of bounds."; + } + return vm; +} +const ValueMap ValueMap::operator[](size_t index) const { + ValueMap vm = *this; + vm.doc_->node = doc_->node[index]; + if (!vm.doc_->node.valid()) { + NTA_THROW << "Index '" + std::to_string(index) + "' is out of bounds."; + } + return vm; +} + +std::string ValueMap::str() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + return S_(r.val()); + } else { + NTA_THROW << "Value not found."; + } +} + +// Return a value converted to the specified type T. +template T ValueMap::as() const { + NodeRef r = doc_->node; + if (r.valid() && r.has_val()) { + T v; + r >> v; + return v; + } else { + NTA_THROW << "Value not found."; + } +} +// explicit instantiation. Can only be used with these types. +template int8_t ValueMap::as() const; +template int16_t ValueMap::as() const; +template uint16_t ValueMap::as() const; +template int32_t ValueMap::as() const; +template uint32_t ValueMap::as() const; +template int64_t ValueMap::as() const; +template uint64_t ValueMap::as() const; +template bool ValueMap::as() const; +template float ValueMap::as() const; +template double ValueMap::as() const; +template std::string ValueMap::as() const; + + +std::string ValueMap::key() const { + NodeRef r = doc_->node; + return (r.has_key()) ? S_(r.key()) : ""; +} + + +// Iterator +ValueMap::iterator ValueMap::iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last)? NodeRef(): ptr_->node.next_sibling(); + return *this; +} +ValueMap::iterator ValueMap::iterator::operator++(int junk) {return operator++(); } +ValueMap &ValueMap::iterator::operator*() { ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +ValueMap *ValueMap::iterator::operator->() { ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::iterator::operator==(const iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::iterator::operator!=(const iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::iterator ValueMap::begin() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } + else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} +ValueMap::iterator ValueMap::end() { + ValueMap::iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + +ValueMap::const_iterator ValueMap::const_iterator::operator++() { + ptr_->node = (ptr_->node == ptr_->last) ? NodeRef() : ptr_->node.next_sibling(); + return *this; +} +ValueMap::const_iterator ValueMap::const_iterator::operator++(int junk) { return operator++();} +const ValueMap &ValueMap::const_iterator::operator*() {ptr_->vm.doc_->node = ptr_->node; return ptr_->vm;} +const ValueMap *ValueMap::const_iterator::operator->() {ptr_->vm.doc_->node = ptr_->node; return &ptr_->vm;} +bool ValueMap::const_iterator::operator==(const const_iterator &rhs) const { return ptr_->node == rhs.ptr_->node; } +bool ValueMap::const_iterator::operator!=(const const_iterator &rhs) const { return ptr_->node != rhs.ptr_->node; } + +ValueMap::const_iterator ValueMap::cbegin() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + if (doc_->node.has_children()) { + itr.ptr_->node = doc_->node.first_child(); + itr.ptr_->last = doc_->node.last_child(); + } else + itr.ptr_->node = NodeRef(); // the end() + return itr; +} + +ValueMap::const_iterator ValueMap::cend() const { + ValueMap::const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->node = NodeRef(); + return itr; +} + + +std::vector ValueMap::getKeys() const { + std::vector v; + NodeRef root = doc_->t.rootref(); + for (NodeRef n : root.children()) { + if (n.has_key()) + v.push_back(S_(n.key())); + } + return v; +} + + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } + } +} + + +static void to_json(std::ostream& f, const htm::ValueMap& vm) { + bool first = true; + switch (vm.getCategory()) { + case ValueMap::Empty: + return; + case ValueMap::Scalar: + f << '"'; + escape_json(f, vm.as()); + f << '"'; + break; + case ValueMap::Sequence: + f << "["; + for (size_t i = 0; i < vm.size(); i++) { + if (!first) + f << ", "; + first = false; + to_json(f, vm[i]); + } + f << "]"; + break; + case ValueMap::Map: + f << "{"; + for (auto it = vm.cbegin(); it != vm.cend(); ++it) { + if (!first) + f << ", "; + first = false; + f << it->key() << ": "; + to_json(f, *it); + } + f << "}"; + break; + } +} + +std::string ValueMap::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); +} + +std::string ValueMap::to_yaml() const { + std::stringstream ss; + ss << doc_->t; + return ss.str(); +} + + +std::ostream &operator<<(std::ostream &f, const htm::ValueMap &vm) { + f << vm.to_json(); + return f; +} + +void ValueMap::handleErrors(const std::string &msg) +{ + throw htm::LoggingException(__FILE__, __LINE__) << "Yaml parse error: " + msg; +} + +#endif // YAML_PARSER_libyaml diff --git a/src/htm/ntypes/Value-yamlcpp.cpp b/src/htm/ntypes/Value-yamlcpp.cpp new file mode 100644 index 0000000000..e946853cf2 --- /dev/null +++ b/src/htm/ntypes/Value-yamlcpp.cpp @@ -0,0 +1,308 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2019, Numenta, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ +#ifdef YAML_PARSER_yamlcpp + +#include +#include +#include + +#include +#include +#include +#include + +using namespace htm; + +// This is just so yaml-cpp/yaml.h declarations are hidden from our .hpp +struct htm::Value::OpaqueTree { + YAML::Node node; +}; +struct htm::Value::iterator::OpaqueIterator { + YAML::iterator it; + YAML::iterator end; +}; +struct htm::Value::const_iterator::OpaqueConstIterator { + YAML::const_iterator it; + YAML::const_iterator end; +}; + +// Constructor +Value::Value() { + doc_ = std::make_shared(); // creates an empty Tree +} + +// Parse YAML or JSON string document into the tree. +Value &Value::parse(const std::string &yaml_string) { + doc_->node = YAML::Load(yaml_string); + return *this; +} + +// checking content of a parameter +// enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category Value::getCategory() const { + if (doc_->node.IsMap()) + return ValueMap::Category::Map; + if (doc_->node.IsSequence()) + return ValueMap::Category::Sequence; + if (doc_->node.IsScalar()) + return ValueMap::Category::Scalar; + return ValueMap::Category::Empty; +} + +bool Value::contains(const std::string &key) const { return (doc_->node[key]); } + +bool Value::isScalar() const { return doc_->node.IsScalar(); } +bool Value::isSequence() const { return doc_->node.IsSequence(); } +bool Value::isMap() const { return doc_->node.IsMap(); } +bool Value::isEmpty() const { return getCategory() == Value::Category::Empty; } + +size_t Value::size() const { return doc_->node.size(); } + +// Accessing members of a map +Value Value::operator[](const std::string &key) { + NTA_CHECK(isMap()) << "This is not a map."; + Value v; + if (doc_->node[key]) { + v.doc_->node = doc_->node[key]; + return v; + } else { + NTA_THROW << "No value found for key '" + key + "'."; + } +} +const Value Value::operator[](const std::string &key) const { + NTA_CHECK(isMap()) << "This is not a map."; + Value v; + if (doc_->node[key]) { + v.doc_->node = doc_->node[key]; + return v; + } else { + NTA_THROW << "No value found for key '" + key + "'."; + } +} + + +// accessing members of a sequence +Value Value::operator[](size_t index) { + NTA_CHECK(isSequence()) << "This is not a sequence."; + Value v; + if (doc_->node.size() > index) { + v.doc_->node = doc_->node[index]; + return v; + } else { + NTA_THROW << "Index '" + std::to_string(index) + "' out of range."; + } +} +const Value Value::operator[](size_t index) const { + NTA_CHECK(isSequence()) << "This is not a sequence."; + Value v; + if (doc_->node.size() > index) { + v.doc_->node = doc_->node[index]; + return v; + } else { + NTA_THROW << "Index '" + std::to_string(index) + "' out of range."; + } +} + +std::string Value::str() const { + NTA_CHECK(isScalar()) << "This is not a scalar."; + return doc_->node.as(); +} + +// Return a value converted to the specified type T. +template T Value::as() const { + NTA_CHECK(isScalar()) << "This is not a scalar."; + return doc_->node.as(); +} +// explicit instantiation. Can only be used with these types. +template int8_t Value::as() const; +template int16_t Value::as() const; +template uint16_t Value::as() const; +template int32_t Value::as() const; +template uint32_t Value::as() const; +template int64_t Value::as() const; +template uint64_t Value::as() const; +template bool Value::as() const; +template float Value::as() const; +template double Value::as() const; +template std::string Value::as() const; + +// Iterator +Value::iterator Value::begin() { + Value::iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.begin(); + itr.ptr_->end = doc_->node.end(); + return itr; +} +Value::iterator Value::end() { + Value::iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.end(); + return itr; +} +Value::iterator Value::iterator::operator++() { + ptr_->it++; + return *this; +} +Value::iterator Value::iterator::operator++(int junk) { return operator++(); } + +Value &Value::iterator::operator*() { + second.doc_->node = ptr_->it->second; + return second; +} +Value::iterator *Value::iterator::operator->() { + first = ptr_->it->first.as(); + second.doc_->node = ptr_->it->second; + return this; +} +bool Value::iterator::operator==(const Value::iterator &rhs) const { return ptr_->it == rhs.ptr_->it; } +bool Value::iterator::operator!=(const Value::iterator &rhs) const { return ptr_->it != rhs.ptr_->it; } + + +// Const Iterator +Value::const_iterator Value::cbegin() const { + Value::const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.begin(); + itr.ptr_->end = doc_->node.end(); + return itr; +} + +Value_const::iterator Value::cend() const { + Value::const_iterator itr; + itr.ptr_ = std::make_shared(); + itr.ptr_->it = doc_->node.end(); + return itr; +} + +Value::const_iterator Value::const_iterator::operator++() { + ptr_->it++; + return *this; +} +Value::const_iterator Value::const_iterator::operator++(int junk) { return operator++(); } + +const Value &Value::const_iterator::operator*() { + second.doc_->node = ptr_->it->second; + return second; +} +Value::const_iterator *Value::const_iterator::operator->() { + first = ptr_->it->first.as(); + second.doc_->node = ptr_->it->second; + return this; +} +bool Value_const::iterator::operator==(const Value::const_iterator &rhs) const { return ptr_->it == rhs.ptr_->it; } +bool Value_const::iterator::operator!=(const Value::const_iterator &rhs) const { return ptr_->it != rhs.ptr_->it; } + +std::vector Value::getKeys() const { + NTA_CHECK(isMap()) << "This is not a map."; + std::vector v; + for (auto it = doc_->node.begin(); it != doc_->node.end(); it++) { + v.push_back(it->first.as()); + } + return v; +} + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } + } +} + +static void to_json(std::ostream &f, const htm::Value &v) { + bool first = true; + switch (v.getCategory()) { + case Value::Empty: + return; + case Value::Scalar: + f << '"'; + escape_json(f, v.as()); + f << '"'; + break; + case Value::Sequence: + f << "["; + for (size_t i = 0; i < v.size(); i++) { + if (!first) + f << ", "; + first = false; + to_json(f, v[i]); + } + f << "]"; + break; + case Value::Map: + f << "{"; + for (auto it = v.cbegin(); it != v.cend(); ++it) { + if (!first) + f << ", "; + first = false; + f << it->first << ": "; + to_json(f, *it); + } + f << "}"; + break; + } +} + +std::string Value::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); +} + +std::string Value::to_yaml() const { + std::stringstream ss; + ss << doc_->node; + return ss.str(); +} + +std::ostream &operator<<(std::ostream &f, const htm::Value &v) { + f << v.to_json(); + return f; +} + + +#endif // YAML_PARSER_yamlcpp diff --git a/src/htm/ntypes/Value.cpp b/src/htm/ntypes/Value.cpp deleted file mode 100644 index 9975c3da88..0000000000 --- a/src/htm/ntypes/Value.cpp +++ /dev/null @@ -1,285 +0,0 @@ -/* --------------------------------------------------------------------- - * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero Public License version 3 as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU Affero Public License for more details. - * - * You should have received a copy of the GNU Affero Public License - * along with this program. If not, see http://www.gnu.org/licenses. - * --------------------------------------------------------------------- */ - -/** @file - * Implementation of the Value class - */ - -#include -#include - -using namespace htm; - -Value::Value(std::shared_ptr &s) { - category_ = scalarCategory; - scalar_ = s; -} - -Value::Value(std::shared_ptr &a) { - category_ = arrayCategory; - array_ = a; -} - -Value::Value(const std::string& s) { - category_ = stringCategory; - string_ = s; -} - -bool Value::isScalar() const { return category_ == scalarCategory; } - -bool Value::isArray() const { return category_ == arrayCategory; } - -bool Value::isString() const { return category_ == stringCategory; } - -NTA_BasicType Value::getType() const { - switch (category_) { - case scalarCategory: - return scalar_->getType(); - break; - case arrayCategory: - return array_->getType(); - break; - default: - // string - return NTA_BasicType_Byte; - break; - } -} - -std::shared_ptr Value::getScalar() const { - NTA_CHECK(category_ == scalarCategory); - return scalar_; -} - -std::shared_ptr Value::getArray() const { - NTA_CHECK(category_ == arrayCategory); - return array_; -} - -std::string Value::getString() const { - NTA_CHECK(category_ == stringCategory); - return string_; -} - -template T Value::getScalarT() const { - NTA_CHECK(category_ == scalarCategory); - if (BasicType::getType() != scalar_->getType()) { - NTA_THROW << "Attempt to access scalar of type " - << BasicType::getName(scalar_->getType()) << " as type " - << BasicType::getName(); - } - return scalar_->getValue(); -} - -const std::string Value::getDescription() const { - switch (category_) { - case stringCategory: - return std::string("string") + " (" + string_ + ")"; - break; - case scalarCategory: - return std::string("Scalar of type ") + BasicType::getName(scalar_->getType()); - break; - case arrayCategory: - return std::string("Array of type ") + BasicType::getName(array_->getType()); - break; - } - return "NOT REACHED"; -} - -void ValueMap::add(const std::string &key, const Value &value) { - if (map_.find(key) != map_.end()) { - NTA_THROW << "Key '" << key << "' specified twice"; - } - auto vp = new Value(value); - - map_.insert(std::make_pair(key, vp)); -} - -Value::Category Value::getCategory() const { return category_; } - -ValueMap::const_iterator ValueMap::begin() const { return map_.begin(); } - -ValueMap::const_iterator ValueMap::end() const { return map_.end(); } - -// specializations of getValue() -// gcc 4.2 complains if they are not inside the namespace declaration -namespace htm { -template Byte Value::getScalarT() const; -template Int16 Value::getScalarT() const; -template Int32 Value::getScalarT() const; -template Int64 Value::getScalarT() const; -template UInt16 Value::getScalarT() const; -template UInt32 Value::getScalarT() const; -template UInt64 Value::getScalarT() const; -template Real32 Value::getScalarT() const; -template Real64 Value::getScalarT() const; -template Handle Value::getScalarT() const; -template bool Value::getScalarT() const; -} // namespace htm - -ValueMap::ValueMap(){}; - -ValueMap::~ValueMap() { - for (auto &elem : map_) { - delete elem.second; - elem.second = nullptr; - } - map_.clear(); -} - -ValueMap::ValueMap(const ValueMap &rhs) { - for (auto &elem : map_) { - delete elem.second; - elem.second = nullptr; - } - map_.clear(); - - for (const auto &rh : rhs) { - auto vp = new Value(*(rh.second)); - - map_.insert(std::make_pair(rh.first, vp)); - } -} - -void ValueMap::dump() const { - NTA_DEBUG << "===== Value Map:"; - for (const auto &elem : map_) { - std::string key = elem.first; - Value *value = elem.second; - NTA_DEBUG << "key: " << key - << " datatype: " << BasicType::getName(value->getType()) - << " category: " << value->getCategory(); - } - NTA_DEBUG << "===== End of Value Map"; -} - -bool ValueMap::contains(const std::string &key) const { - return (map_.find(key) != map_.end()); -} - -Value &ValueMap::getValue(const std::string &key) const { - auto item = map_.find(key); - if (item == map_.end()) { - NTA_THROW << "No value '" << key << "' found in Value Map"; - } - return *(item->second); -} - -template -T ValueMap::getScalarT(const std::string &key, T defaultValue) const { - auto item = map_.find(key); - if (item == map_.end()) { - return defaultValue; - } else { - return getScalarT(key); - } -} - -template T ValueMap::getScalarT(const std::string &key) const { - std::shared_ptr s = getScalar(key); - if (s->getType() != BasicType::getType()) { - NTA_THROW << "Invalid attempt to access parameter '" << key - << "' as type a " << BasicType::getName() - << " but the Spec defines it as type " << BasicType::getName(s->getType()); - } - - return s->getValue(); -} - -std::shared_ptr ValueMap::getArray(const std::string &key) const { - Value &v = getValue(key); - if (!v.isArray()) { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as an array but it is a '" - << v.getDescription(); - } - return v.getArray(); -} - -std::shared_ptr ValueMap::getScalar(const std::string &key) const { - Value &v = getValue(key); - if (!v.isScalar()) { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as an array but it is a '" - << v.getDescription(); - } - return v.getScalar(); -} - -std::string ValueMap::getString(const std::string& key) const { - Value& v = getValue(key); - if (! v.isString()) - { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as a string but it is a '" - << v.getDescription(); - } - return v.getString(); -} -std::string ValueMap::getString(const std::string &key, const std::string defaultValue) const { - auto item = map_.find(key); - if (item == map_.end()) { - return defaultValue; - } else { - Value &v = getValue(key); - if (!v.isString()) { - NTA_THROW << "Attempt to access element '" << key - << "' of value map as a string but it is a '" - << v.getDescription(); - } - return v.getString(); - } -} - - -// explicit instantiations of getScalarT -namespace htm { -template Byte ValueMap::getScalarT(const std::string &key, - Byte defaultValue) const; -template UInt16 ValueMap::getScalarT(const std::string &key, - UInt16 defaultValue) const; -template Int16 ValueMap::getScalarT(const std::string &key, - Int16 defaultValue) const; -template UInt32 ValueMap::getScalarT(const std::string &key, - UInt32 defaultValue) const; -template Int32 ValueMap::getScalarT(const std::string &key, - Int32 defaultValue) const; -template UInt64 ValueMap::getScalarT(const std::string &key, - UInt64 defaultValue) const; -template Int64 ValueMap::getScalarT(const std::string &key, - Int64 defaultValue) const; -template Real32 ValueMap::getScalarT(const std::string &key, - Real32 defaultValue) const; -template Real64 ValueMap::getScalarT(const std::string &key, - Real64 defaultValue) const; -template Handle ValueMap::getScalarT(const std::string &key, - Handle defaultValue) const; -template bool ValueMap::getScalarT(const std::string &key, - bool defaultValue) const; - -template Byte ValueMap::getScalarT(const std::string &key) const; -template UInt16 ValueMap::getScalarT(const std::string &key) const; -template Int16 ValueMap::getScalarT(const std::string &key) const; -template UInt32 ValueMap::getScalarT(const std::string &key) const; -template Int32 ValueMap::getScalarT(const std::string &key) const; -template UInt64 ValueMap::getScalarT(const std::string &key) const; -template Int64 ValueMap::getScalarT(const std::string &key) const; -template Real32 ValueMap::getScalarT(const std::string &key) const; -template Real64 ValueMap::getScalarT(const std::string &key) const; -template Handle ValueMap::getScalarT(const std::string &key) const; -template bool ValueMap::getScalarT(const std::string &key) const; -} // namespace htm diff --git a/src/htm/ntypes/Value.hpp b/src/htm/ntypes/Value.hpp index 08ca996376..478268a784 100644 --- a/src/htm/ntypes/Value.hpp +++ b/src/htm/ntypes/Value.hpp @@ -1,6 +1,8 @@ /* --------------------------------------------------------------------- * HTM Community Edition of NuPIC - * Copyright (C) 2013, Numenta, Inc. + * Copyright (C) 2019, Numenta, Inc. + * + * David Keeney dkeeney@gmail.com * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero Public License version 3 as @@ -14,122 +16,166 @@ * You should have received a copy of the GNU Affero Public License * along with this program. If not, see http://www.gnu.org/licenses. * --------------------------------------------------------------------- */ - -/** @file - * Definitions for the Value class - * - * A Value object holds a Scalar or an Array - * A ValueMap is essentially a map - * It is used internally in the conversion of YAML strings to C++ objects. - * The API and implementation are geared towards clarify rather than - * performance, since it is expected to be used only during network construction. - */ - #ifndef NTA_VALUE_HPP #define NTA_VALUE_HPP - -#include +#include #include - -#include - -#include -#include +#include +#include +#include +#include +#include namespace htm { -/** - * The Value class is used to store construction parameters - * for regions and links. A YAML string specified by the user - * is parsed and converted into a set of Values. - * - * A Value is essentially a union of Scalar/Array/string. - * In turn, a Scalar is a union of NTA_BasicType types, - * and an Array is an array of such types. - * - * A string is similar to an Array of NTA_BasicType_Byte, but - * is handled differently, so it is separated in the API. - * - * The Value API uses std::shared_ptr instead of directly - * using the underlying objects, to avoid copying, and because - * Array may not be copied. - */ +class Value_iterator; +class Value_const_iterator; + class Value { public: - Value(std::shared_ptr &s); - Value(std::shared_ptr &a); - Value(const std::string &s); + Value(); + Value& parse(const std::string &yaml_string); - enum Category { scalarCategory, arrayCategory, stringCategory }; + bool contains(const std::string &key)const; + size_t size() const; - bool isArray() const; - bool isString() const; - bool isScalar() const; + // type of value in the Value + enum Category { Empty = 0, Scalar, Sequence, Map }; Category getCategory() const; + bool isScalar() const; + bool isSequence() const; + bool isMap() const; + bool isEmpty() const; + + // Access + Value operator[](const std::string &key); + Value operator[](size_t index); + const Value operator[](const std::string &key) const; + const Value operator[](size_t index) const; + template T as() const; + std::string str() const; + std::vector getKeys() const; + + + class iterator { + public: + using value_type = Value; + using reference = value_type &; + using pointer = iterator *; + using difference_type = std::ptrdiff_t; + using iterator_category = std::forward_iterator_tag; + iterator operator++(); + iterator operator++(int junk); + reference operator*(); + pointer operator->(); + bool operator==(const iterator &rhs) const; + bool operator!=(const iterator &rhs) const; + + struct OpaqueIterator; + std::shared_ptr ptr_; + std::string first; + htm::Value second; + template T as() { return second->as(); } + }; + + class const_iterator { + public: + using value_type = const Value; + using reference = value_type &; + using pointer = const_iterator *; + using difference_type = std::ptrdiff_t; + using iterator_category = std::forward_iterator_tag; + const_iterator operator++(); + const_iterator operator++(int junk); + reference operator*(); + pointer operator->(); + bool operator==(const const_iterator &rhs) const; + bool operator!=(const const_iterator &rhs) const; + + struct OpaqueConstIterator; + std::shared_ptr ptr_; + std::string first; + const htm::Value second; + template T as() { return second->as(); } + }; + + iterator begin(); + iterator end(); + const_iterator cbegin(); + const_iterator cend(); + + + + + template std::vector asVector() const { + std::vector v; + if (!isSequence()) + NTA_THROW << "Not a sequence node."; + for (auto iter = cbegin(); iter != cend(); iter++) { // iterate through the children of this node. + const Value& n = *iter; + try { + if (n.isScalar()) { + v.push_back(n.as()); + } + } catch (std::exception e) { + NTA_THROW << "Invalid vector element; " << e.what(); + } + } + return v; + } + template std::map asMap() const { + std::map v; + if (!isSequence()) + NTA_THROW << "Not a dictionary node."; + for (auto iter = cbegin(); iter != cend(); iter++) { // iterate through the children of this node. + const Value& n = *iter; + try { + if (n.isScalar() && n.hasKey()) { + v[n.key()] = n.as(); + } + } catch (std::exception e) { + NTA_THROW << "Invalid map element; " << e.what; + } + } + } + + // serializing routines + std::string to_yaml() const; + std::string to_json() const; - NTA_BasicType getType() const; - - std::shared_ptr getScalar() const; - - std::shared_ptr getArray() const; - - std::string getString() const; - - template T getScalarT() const; - - const std::string getDescription() const; private: - // Default constructor would not be useful - Value(); - Category category_; - std::shared_ptr scalar_; - std::shared_ptr array_; - std::string string_; -}; - -class ValueMap { -public: - ValueMap(); - ValueMap(const ValueMap &rhs); - ~ValueMap(); - void add(const std::string &key, const Value &value); + struct OpaqueTree; + std::shared_ptr doc_; // This is an opaque pointer to a ryml Tree object. - // map.find(key) != map.end() - bool contains(const std::string &key) const; - - // map.find(key) + exception if not found - Value &getValue(const std::string &key) const; - - // Method below are for convenience, bypassing the Value - std::shared_ptr getArray(const std::string &key) const; - std::shared_ptr getScalar(const std::string &key) const; - std::string getString(const std::string &key) const; - std::string getString(const std::string &key, const std::string defaultValue) const; - - // More convenience methods, bypassing the Value and the contained Scalar - - // use default value if not specified in map - template - T getScalarT(const std::string &key, T defaultValue) const; +}; - // raise exception if value is not specified in map - template T getScalarT(const std::string &key) const; - void dump() const; - typedef std::map::const_iterator const_iterator; - const_iterator begin() const; - const_iterator end() const; -private: - // must be a Value* since Value doesn't have a default constructor - // We own all the items in the map and must delete them in our destructor - typedef std::map::iterator iterator; - std::map map_; +class ValueMap : public Value { +public: + // Access for backward compatability + template T getScalarT(const std::string &key) const { // throws + return (*this)[key].as(); + } + template T getScalarT(const std::string &key, T defaultValue) const { // with default + if (contains(key)) + return (*this)[key].as(); + else + return defaultValue; + } + std::string getString(const std::string &key, const std::string& defaultValue) const { + if (contains(key)) + return (*this)[key].str(); + else + return defaultValue; + } }; } // namespace htm -#endif // NTA_VALUE_HPP +std::ostream &operator<<(std::ostream &f, const htm::ValueMap &vm); + +#endif // NTA_VALUEMAP_HPP diff --git a/src/htm/regions/SPRegion.cpp b/src/htm/regions/SPRegion.cpp index aa5db33e5f..a9d890478e 100644 --- a/src/htm/regions/SPRegion.cpp +++ b/src/htm/regions/SPRegion.cpp @@ -30,7 +30,6 @@ #include #include #include -#include #include #include diff --git a/src/htm/regions/SPRegion.hpp b/src/htm/regions/SPRegion.hpp index 07633bc64b..021e24d08f 100644 --- a/src/htm/regions/SPRegion.hpp +++ b/src/htm/regions/SPRegion.hpp @@ -30,6 +30,7 @@ #include #include +#include //---------------------------------------------------------------------- diff --git a/src/htm/regions/ScalarSensor.cpp b/src/htm/regions/ScalarSensor.cpp index 0d391d3877..41bc22f1dd 100644 --- a/src/htm/regions/ScalarSensor.cpp +++ b/src/htm/regions/ScalarSensor.cpp @@ -32,19 +32,19 @@ namespace htm { ScalarSensor::ScalarSensor(const ValueMap ¶ms, Region *region) : RegionImpl(region) { - params_.size = params.getScalarT("n"); - params_.activeBits = params.getScalarT("w"); - params_.resolution = params.getScalarT("resolution"); - params_.radius = params.getScalarT("radius"); - params_.minimum = params.getScalarT("minValue"); - params_.maximum = params.getScalarT("maxValue"); - params_.periodic = params.getScalarT("periodic"); - params_.clipInput = params.getScalarT("clipInput"); + params_.size = params.getScalarT("n", 0); + params_.activeBits = params.getScalarT("w", 0); + params_.resolution = params.getScalarT("resolution", 0.0); + params_.radius = params.getScalarT("radius", 0.0); + params_.minimum = params.getScalarT("minValue", -1.0); + params_.maximum = params.getScalarT("maxValue", +1.0); + params_.periodic = params.getScalarT("periodic", false); + params_.clipInput = params.getScalarT("clipInput", false); encoder_ = std::make_shared( params_ ); - sensedValue_ = params.getScalarT("sensedValue"); + sensedValue_ = params.getScalarT("sensedValue", -1.0); } ScalarSensor::ScalarSensor(ArWrapper &wrapper, Region *region):RegionImpl(region) { diff --git a/src/htm/regions/TestNode.cpp b/src/htm/regions/TestNode.cpp index 9ec534c2cf..1d42978ec6 100644 --- a/src/htm/regions/TestNode.cpp +++ b/src/htm/regions/TestNode.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include diff --git a/src/htm/regions/VectorFileEffector.cpp b/src/htm/regions/VectorFileEffector.cpp index 21bb80a531..00da1dec04 100644 --- a/src/htm/regions/VectorFileEffector.cpp +++ b/src/htm/regions/VectorFileEffector.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include diff --git a/src/htm/regions/VectorFileEffector.hpp b/src/htm/regions/VectorFileEffector.hpp index 30f571599d..d1bb8f9792 100644 --- a/src/htm/regions/VectorFileEffector.hpp +++ b/src/htm/regions/VectorFileEffector.hpp @@ -30,10 +30,10 @@ #include #include #include +#include namespace htm { -class ValueMap; /** * VectorFileEffector is a node that takes its input vectors and diff --git a/src/htm/regions/VectorFileSensor.cpp b/src/htm/regions/VectorFileSensor.cpp index 7f947c3023..131cdf2167 100644 --- a/src/htm/regions/VectorFileSensor.cpp +++ b/src/htm/regions/VectorFileSensor.cpp @@ -32,7 +32,6 @@ #include #include #include -#include using namespace std; namespace htm { diff --git a/src/htm/regions/VectorFileSensor.hpp b/src/htm/regions/VectorFileSensor.hpp index 2c8cfc2386..f0522c75ca 100644 --- a/src/htm/regions/VectorFileSensor.hpp +++ b/src/htm/regions/VectorFileSensor.hpp @@ -33,6 +33,7 @@ #include #include #include +#include namespace htm { class ValueMap; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index e4bd4258b1..455d006ab3 100755 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -57,7 +57,6 @@ set(engine_tests unit/engine/InputTest.cpp unit/engine/LinkTest.cpp unit/engine/NetworkTest.cpp - unit/engine/YAMLUtilsTest.cpp unit/engine/WatcherTest.cpp ) @@ -71,7 +70,6 @@ set(ntypes_tests unit/ntypes/BasicTypeTest.cpp unit/ntypes/CollectionTest.cpp unit/ntypes/DimensionsTest.cpp - unit/ntypes/ScalarTest.cpp unit/ntypes/ValueTest.cpp ) @@ -86,8 +84,8 @@ set(regions_tests unit/regions/RegionTestUtilities.cpp unit/regions/RegionTestUtilities.hpp unit/regions/SPRegionTest.cpp - unit/regions/TMRegionTest.cpp - unit/regions/VectorFileTest.cpp + unit/regions/TMRegionTest.cpp + unit/regions/VectorFileTest.cpp ) diff --git a/src/test/unit/ntypes/ValueMapTest.cpp b/src/test/unit/ntypes/ValueMapTest.cpp new file mode 100644 index 0000000000..84a358f636 --- /dev/null +++ b/src/test/unit/ntypes/ValueMapTest.cpp @@ -0,0 +1,116 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2013, Numenta, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ + +/** @file + * Implementation of YAML tests + */ + +#include "gtest/gtest.h" +#include +#include +#include + +namespace testing { + +using namespace htm; + +TEST(ValueMapTest, IntScalerTest) { + ValueMap vm; + + const char *s1 = "10"; + vm.parse(s1); + EXPECT_TRUE(vm.isScalar()); + UInt32 u = vm.as(); + EXPECT_EQ(10u, u); + + vm.parse("-1"); + EXPECT_ANY_THROW(UInt32 x = vm.as()); // Unsigned must be positive. + Int32 i = vm.as(); + EXPECT_EQ(-1, i); + + vm.parse("- 1"); + EXPECT_TRUE(vm.isSequence()); + u = vm[0].as(); + EXPECT_EQ(1, u); + + vm.parse("[123]"); // explicit sequence with one element. + EXPECT_TRUE(vm.isSequence()); + i = vm[0].as(); + EXPECT_EQ(123, i); + + EXPECT_ANY_THROW(Int32 i = vm.parse("999999999999999999999999999").as()); + EXPECT_ANY_THROW(Int32 i = vm.parse("-1").as()); + EXPECT_ANY_THROW(Int32 i = vm.parse("abc").as()); + EXPECT_ANY_THROW(Int32 i = vm.parse("").as()); +} + +TEST(ValueMapTest, toValueTestReal32) { + ValueMap vm; + vm.parse("10.1"); + Real32 x = vm.getScalarT(""); + EXPECT_NEAR(10.1f, x, 0.000001); +} + +TEST(ValueMapTest, toValueTestString) { + ValueMap vm; + std::string s1 = "\"this is a string\""; + vm.parse(s1); + std::string s = vm.getScalarT(""); + EXPECT_TRUE(s == s1); + + std::string s2 = "this is a string"; + vm.parse(s1); + s = vm.getScalarT(""); + EXPECT_TRUE(s == s2); + + s = vm.str(); + EXPECT_TRUE(s == s2); +} + +TEST(ValueMapTest, toValueTestBool) { + ValueMap vm; + EXPECT_TRUE(vm.parse("true").getScalarT("")); + EXPECT_TRUE(vm.parse("1").getScalarT("")); + EXPECT_FALSE(vm.parse("false").getScalarT("")); + EXPECT_FALSE(vm.parse("0").getScalarT("")); + EXPECT_ANY_THROW(vm.parse("1234").getScalarT("")); +} + +TEST(ValueMapTest, asArray) { + ValueMap vm; + std::string json = "[1,2,3,4,5]"; + vm.parse(json); + + EXPECT_EQ(ValueMap::Sequence, vm.getCategory()); + EXPECT_TRUE(vm.isSequence()); + ASSERT_TRUE(!vm.isScalar()); + ASSERT_TRUE(!vm.isScalar()); + ASSERT_TRUE(!vm.isEmpty()); + + std::vector s1 = vm.asVector(); + std::vector s2 = {1, 2, 3, 4, 5}; + ASSERT_TRUE(s1 == s2); + + ASSERT_EQ(vm[0].as(), 1); + ASSERT_EQ(vm[0].str(), "1"); + + ASSERT_ANY_THROW(vm.as()); // not a scaler + ASSERT_ANY_THROW(vm["foobar"]); // not a map +} + + +} // namespace testing \ No newline at end of file diff --git a/src/test/unit/regions/SPRegionTest.cpp b/src/test/unit/regions/SPRegionTest.cpp index 2c9b182485..47a1bdb11c 100644 --- a/src/test/unit/regions/SPRegionTest.cpp +++ b/src/test/unit/regions/SPRegionTest.cpp @@ -50,7 +50,6 @@ #include #include #include -#include #include @@ -64,7 +63,6 @@ #include -#include "yaml-cpp/yaml.h" #include "gtest/gtest.h" #include "RegionTestUtilities.hpp" diff --git a/src/test/unit/regions/TMRegionTest.cpp b/src/test/unit/regions/TMRegionTest.cpp index 7d83942608..69cfb304c5 100644 --- a/src/test/unit/regions/TMRegionTest.cpp +++ b/src/test/unit/regions/TMRegionTest.cpp @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include @@ -66,7 +65,6 @@ #include #include "RegionTestUtilities.hpp" -#include "yaml-cpp/yaml.h" #include "gtest/gtest.h" #define VERBOSE if (verbose) std::cerr << "[ ] " diff --git a/src/test/unit/regions/VectorFileTest.cpp b/src/test/unit/regions/VectorFileTest.cpp index 23d4721571..54db00f477 100644 --- a/src/test/unit/regions/VectorFileTest.cpp +++ b/src/test/unit/regions/VectorFileTest.cpp @@ -49,7 +49,6 @@ #include #include #include -#include #include #include @@ -67,7 +66,6 @@ -#include "yaml-cpp/yaml.h" #include "gtest/gtest.h" #include "RegionTestUtilities.hpp" From 404a28e1e72041b43c6981c7451cfd6a4c9c0a7f Mon Sep 17 00:00:00 2001 From: David Keeney Date: Sun, 29 Sep 2019 16:44:51 -0700 Subject: [PATCH 09/10] binding is compiling. passing this PR back to windows. --- VERSION | 2 +- bindings/py/cpp_src/plugin/PyBindRegion.cpp | 105 ++++++++++-------- .../cpp_src/plugin/RegisteredRegionImplPy.hpp | 2 +- src/CMakeLists.txt | 2 +- src/htm/ntypes/Value-yamlcpp.cpp | 9 +- src/htm/ntypes/Value.hpp | 23 ++-- src/test/unit/ntypes/ValueTest.cpp | 44 ++++++-- 7 files changed, 116 insertions(+), 71 deletions(-) diff --git a/VERSION b/VERSION index 0797f736d0..3a3b26eac4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.0.11 \ No newline at end of file +v2.0.16 \ No newline at end of file diff --git a/bindings/py/cpp_src/plugin/PyBindRegion.cpp b/bindings/py/cpp_src/plugin/PyBindRegion.cpp index a227937d20..d9d836ddd9 100644 --- a/bindings/py/cpp_src/plugin/PyBindRegion.cpp +++ b/bindings/py/cpp_src/plugin/PyBindRegion.cpp @@ -67,56 +67,65 @@ namespace py = pybind11; } // switch } - static void prepareCreationParams(const ValueMap & vm, py::kwargs& kwargs) - { - ValueMap::const_iterator it; - for (it = vm.begin(); it != vm.end(); ++it) - { - try + // recurseive helper for prepareCreationParams + static py::object make_args(const Value& vm) { + if (vm.isScalar()) { + return py::str(vm.as()); + } + if (vm.isMap()) { + py::kwargs kw; + ValueMap::map_const_iterator it; + for (it = vm.begin(); it != vm.end(); ++it) { - auto key = it->first.c_str(); + std::string key = it->first.c_str(); + Value v = it->second; + kw[key.c_str()] = make_args(v); + } + return kw; + } + if (vm.isSequence()) { + auto a = py::list(); + for(size_t i = 0; i < vm.size(); i++) { + Value v = vm[i]; + a.append(make_args(v)); + } + return a; + } + throw Exception(__FILE__, __LINE__, "Not implemented."); + } - auto value = it->second; - if (value->isScalar()) - { - auto s = value->getScalar(); - switch (s->getType()) - { - case NTA_BasicType_Bool: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Byte: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Int16: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_UInt16: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Int32: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_UInt32: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Int64: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_UInt64: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Real32: { kwargs[key] = s->getValue(); break; } - case NTA_BasicType_Real64: { kwargs[key] = s->getValue(); break; } - - default: - NTA_THROW << "Invalid type: " << s->getType(); - } - } - else if(value->isString()) - { - kwargs[key] = value->getString(); - } - else if (value->isArray()) - { - auto a = value->getArray(); - kwargs[key] = create_numpy_view(*a.get()); + // make kwargs from ValueMap. + static void prepareCreationParams(const ValueMap & vm, py::kwargs& kwargs) + { + if (vm.isMap()) { + ValueMap::map_const_iterator it; + for (it = vm.begin(); it != vm.end(); ++it) { + std::string key = it->first.c_str(); + try { + Value value = it->second; + kwargs[key.c_str()] = make_args(value); } - else - { - throw Exception(__FILE__, __LINE__, "Not implemented."); + catch (Exception& e) { + NTA_THROW << "Unable to create a Python object for parameter '" + << key << ": " << e.what(); } } + } + else if (vm.isScalar()) { + kwargs["arg"] = vm.as(); + } + else if (vm.isSequence()) { + try { + kwargs["array"] = make_args(vm); + } catch (Exception& e) { - NTA_THROW << "Unable to create a Python object for parameter '" - << it->first << ": " << e.what(); + NTA_THROW << "Unable to create a Python object for parameter 'array:' " << e.what(); } } + else { + NTA_THROW << "Unable to create a Python object for parameters."; + } }; PyBindRegion::PyBindRegion(const char * module, const ValueMap & nodeParams, Region * region, const char* className) @@ -184,10 +193,10 @@ namespace py = pybind11; args = py::make_tuple(node_, f, 2); // use type 2 protocol pickle.attr("dump")(*args); pickle.attr("close")(); - + // copy the pickle into the out string std::ifstream pfile(tmp_pickle.c_str(), std::ios::binary); - std::string content((std::istreambuf_iterator(pfile)), + std::string content((std::istreambuf_iterator(pfile)), std::istreambuf_iterator()); pfile.close(); Path::remove(tmp_pickle); @@ -206,7 +215,7 @@ namespace py = pybind11; // copy the extra data into the extra string std::ifstream efile(tmp_extra.c_str(), std::ios::binary); - std::string extra((std::istreambuf_iterator(efile)), + std::string extra((std::istreambuf_iterator(efile)), std::istreambuf_iterator()); efile.close(); Path::remove(tmp_extra); @@ -221,11 +230,11 @@ namespace py = pybind11; std::ofstream des; std::string tmp_pickle = "pickle.tmp"; - + std::ofstream pfile(tmp_pickle.c_str(), std::ios::binary); pfile.write(p.c_str(), p.size()); pfile.close(); - + // Tell Python to un-pickle using what is now in the pickle.tmp file. py::args args = py::make_tuple(tmp_pickle, "rb"); @@ -534,7 +543,7 @@ namespace py = pybind11; auto count = input["count"].cast(); bool required = false; - if (input.contains("required")) + if (input.contains("required")) { required = input["required"].cast(); } @@ -609,7 +618,7 @@ namespace py = pybind11; regionLevel = output["regionLevel"].cast(); } bool isDefaultOutput = false; - if (output.contains("isDefaultOutput")) + if (output.contains("isDefaultOutput")) { isDefaultOutput = output["isDefaultOutput"].cast(); } diff --git a/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp b/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp index 9fe9c658b5..0b03f2bc3d 100644 --- a/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp +++ b/bindings/py/cpp_src/plugin/RegisteredRegionImplPy.hpp @@ -65,6 +65,7 @@ #include #include +#include #include namespace py = pybind11; @@ -79,7 +80,6 @@ namespace htm class Spec; class PyRegionImpl; class Region; - class ValueMap; class RegisteredRegionImplPy: public RegisteredRegionImpl { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 765ca5dba7..1410b6839e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -126,7 +126,7 @@ set(ntypes_files htm/ntypes/Collection.hpp htm/ntypes/Dimensions.hpp htm/ntypes/Value-yamlcpp.cpp - htm/ntypes/Value-rapidyaml.cpp + htm/ntypes/Value-RapidYaml.cpp htm/ntypes/Value-libyaml.cpp htm/ntypes/Value.hpp ) diff --git a/src/htm/ntypes/Value-yamlcpp.cpp b/src/htm/ntypes/Value-yamlcpp.cpp index a9dd926f43..35b8339484 100644 --- a/src/htm/ntypes/Value-yamlcpp.cpp +++ b/src/htm/ntypes/Value-yamlcpp.cpp @@ -123,7 +123,7 @@ void Value::operator=(float val) { doc_->node = val; } void Value::operator=(double val) { doc_->node = val; } void Value::operator=(std::vector val) { doc_->node = val; } -// Compare two nodes recursively to see if content is same. +// Compare two nodes recursively to see if content is same. // yaml-cpp does equals by compairing pointers so we have to do our own. static bool equals(const YAML::Node &n1, const YAML::Node &n2) { if (n1.IsScalar() && n2.IsScalar() && n1.as() == n2.as()) @@ -162,7 +162,7 @@ template float Value::as() const; template double Value::as() const; template std::string Value::as() const; template <> bool Value::as() const { std::string val = doc_->node.as(); - transform(val.begin(), val.end(), val.begin(), ::tolower); + transform(val.begin(), val.end(), val.begin(), ::tolower); if (val == "true" || val == "on" || val == "1") return true; if (val == "false" || val == "off" || val == "0") @@ -365,10 +365,11 @@ std::string Value::to_yaml() const { return ss.str(); } -std::ostream &htm::operator<<(std::ostream &f, const htm::Value &v) { +namespace htm { +std::ostream &operator<<(std::ostream &f, const htm::Value &v) { f << v.to_json(); return f; } - +} // htm namespace #endif // YAML_PARSER_yamlcpp diff --git a/src/htm/ntypes/Value.hpp b/src/htm/ntypes/Value.hpp index f6dac094d6..e3132f9174 100644 --- a/src/htm/ntypes/Value.hpp +++ b/src/htm/ntypes/Value.hpp @@ -26,6 +26,7 @@ #include #include #include +#include // transform namespace htm { @@ -89,7 +90,7 @@ class Value { // return false if node is empty: if (vm) { do something } explicit operator bool() const { return !isEmpty(); } - + // extract a Vector template std::vector asVector() const { std::vector v; @@ -101,7 +102,7 @@ class Value { if (n.isScalar()) { v.push_back(n.as()); } - } catch (std::exception e) { + } catch (std::exception& e) { NTA_THROW << "Invalid vector element; " << e.what(); } } @@ -114,15 +115,21 @@ class Value { if (!isSequence()) NTA_THROW << "Not a Map node."; for (auto iter = cbegin(); iter != cend(); iter++) { // iterate through the children of this node. - const Value n = *iter; + const std::string key = iter->first; + const Value n = iter->second; try { - if (n.isScalar() && n.hasKey()) { - v[n.key()] = n.as(); + if (n.isScalar()) { + v[key] = n.as(); } - } catch (std::exception e) { - NTA_THROW << "Invalid map element; " << e.what; + else { + // non-scalar field. Ignore + } + } catch (std::exception& e) { + // probably bad conversion of scalar to requested type. + NTA_THROW << "Invalid map element[" << key << "] " << e.what(); } } + return v; } @@ -193,7 +200,7 @@ class Value { private: - struct OpaqueTree; + struct OpaqueTree; std::shared_ptr doc_; // This is an opaque pointer to implementation Tree node object. }; diff --git a/src/test/unit/ntypes/ValueTest.cpp b/src/test/unit/ntypes/ValueTest.cpp index ba55fe1f1d..2a1f17f1ae 100644 --- a/src/test/unit/ntypes/ValueTest.cpp +++ b/src/test/unit/ntypes/ValueTest.cpp @@ -22,6 +22,10 @@ #include #include +#include +#include +#include + namespace testing { using namespace htm; @@ -43,8 +47,9 @@ TEST(ValueTest, toValueNumber) { vm.parse("- 1"); // "- " means a sequence element in YAML EXPECT_TRUE(vm.isSequence()); - u = vm[0].as(); - EXPECT_EQ(1, u); + UInt32 u1 = vm[0].as(); + UInt32 u2 = 1u; + EXPECT_EQ(u1, u2); vm.parse("[123]"); // explicit sequence with one element. EXPECT_TRUE(vm.isSequence()); @@ -115,7 +120,7 @@ TEST(ValueTest, toValueBool) { TEST(ValueTest, asArray) { ValueMap vm; Value v; - std::vector s2 = {10, 20, 30, 40, 50}; + std::vector s2 = {10u, 20u, 30u, 40u, 50u}; std::string json = "[10,20,30,40,50]"; vm.parse(json); @@ -132,17 +137,40 @@ TEST(ValueTest, asArray) { EXPECT_EQ(vm[0].as(), 10u); EXPECT_STREQ(vm[0].str().c_str(), "10"); - std::vector s3 = {100, 200, 300, 400, 500}; + std::vector s3 = {100u, 200u, 300u, 400u, 500u}; vm[5] = s3; // assign an array to the 6th element. - EXPECT_EQ(vm[0].as(), 10); + EXPECT_EQ(vm[0].as(), 10u); EXPECT_TRUE(vm[5].isSequence()); EXPECT_TRUE(vm[5][4].isScalar()); - EXPECT_EQ(vm[5][4].as(), 500); + EXPECT_EQ(vm[5][4].as(), 500u); EXPECT_ANY_THROW(vm.as()); // not a scaler EXPECT_ANY_THROW(vm[5].as()); // not a sequence } + +TEST(ValueTest, asMap) { + ValueMap vm; + std::string src = "{scalar: \"456\", array: [\"1\", \"2\", \"3\", \"4\"], string: \"true\"}"; + vm.parse(src); + + std::map m; + m = vm.asMap(); + std::stringstream ss; + ss << "{"; + bool first = true; + for (auto itr = m.begin(); itr != m.end(); itr++) { + if (!first) ss << ", "; + first = false; + ss << itr->first << ": " << itr->second; + } + ss << "}"; + std::string result = ss.str(); + std::cout << result << "\n"; + EXPECT_STREQ(result.c_str(), "{scalar: \"456\", string: \"true\"}"); +} + + TEST(ValueTest, String) { std::string s("hello world"); Value v; @@ -166,7 +194,7 @@ TEST(ValueTest, ValueMap) { vm["scalar"] = 123; vm["scalar"] = 456; // should replace vm["array"] = a; - vm["string"] = "str"; + vm["string"] = std::string("str"); EXPECT_TRUE(vm.isMap()); EXPECT_TRUE(vm.contains("scalar")); @@ -209,7 +237,7 @@ TEST(ValueTest, Iterations) { std::string data = R"( scalar: 123.45 -array: +array: - 1 - 2 - 3 From 39f983c0d8e1cf7693be89bc486cb9317dba785b Mon Sep 17 00:00:00 2001 From: David Keeney Date: Fri, 11 Oct 2019 07:35:28 -0700 Subject: [PATCH 10/10] Cleaned up new yaml-cpp interface --- src/CMakeLists.txt | 4 +- src/htm/engine/RegionImpl.cpp | 31 ++ src/htm/engine/RegionImpl.hpp | 5 +- src/htm/ntypes/Value.cpp | 510 ++++++++++++++++++ src/htm/ntypes/Value.hpp | 188 ++++--- src/htm/regions/SPRegion.cpp | 4 +- src/htm/regions/ScalarSensor.cpp | 23 +- src/htm/regions/ScalarSensor.hpp | 1 + src/htm/regions/TMRegion.cpp | 2 +- src/htm/regions/TestNode.cpp | 2 +- src/test/unit/engine/CppRegionTest.cpp | 12 +- src/test/unit/engine/WatcherTest.cpp | 5 +- src/test/unit/ntypes/ValueTest.cpp | 124 ++++- src/test/unit/regions/RegionTestUtilities.cpp | 5 - src/test/unit/regions/TMRegionTest.cpp | 12 +- 15 files changed, 794 insertions(+), 134 deletions(-) create mode 100644 src/htm/ntypes/Value.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1410b6839e..733c7d12ae 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -125,9 +125,7 @@ set(ntypes_files htm/ntypes/BasicType.hpp htm/ntypes/Collection.hpp htm/ntypes/Dimensions.hpp - htm/ntypes/Value-yamlcpp.cpp - htm/ntypes/Value-RapidYaml.cpp - htm/ntypes/Value-libyaml.cpp + htm/ntypes/Value.cpp htm/ntypes/Value.hpp ) diff --git a/src/htm/engine/RegionImpl.cpp b/src/htm/engine/RegionImpl.cpp index ec81e3708d..ff890b402c 100644 --- a/src/htm/engine/RegionImpl.cpp +++ b/src/htm/engine/RegionImpl.cpp @@ -252,5 +252,36 @@ Dimensions RegionImpl::getOutputDimensions(const std::string &name) const { return region_->getOutputDimensions(name); } +/** + * Checks the parameters in the ValueMap and gives an error if it + * is not consistant with the Spec. If a field in the Spec is not given + * in the ValueMap, insert it with its default value. + */ +void RegionImpl::ValidateParameters(ValueMap &vm) { + std::shared_ptr ns = region_->getSpec(); + + // Look for parameters that don't belong + for (auto p: vm) { + std::string key = p.first; + Value v = p.second; + if (key == "dim") + continue; + if (!ns->parameters.contains(key)) + NTA_THROW << "Parameter '" << key << "' is not expected for this Region."; + } + + // Look for missing parameters and apply their default value. + for (auto p : ns->parameters) { + std::string key = p.first; + ParameterSpec &ps = p.second; + if (vm.getString(key, "").length() == 0) { + // a missing or empty parameter. + vm[key] = ps.defaultValue; + } + } + +} + + } // namespace htm diff --git a/src/htm/engine/RegionImpl.hpp b/src/htm/engine/RegionImpl.hpp index 4ed7446b58..6fd1b17d64 100644 --- a/src/htm/engine/RegionImpl.hpp +++ b/src/htm/engine/RegionImpl.hpp @@ -140,6 +140,8 @@ #include #include #include +#include +#include namespace htm { @@ -294,8 +296,9 @@ class RegionImpl virtual void setDimensions(Dimensions dim) { dim_ = std::move(dim); } virtual Dimensions getDimensions() const { return dim_; } + virtual void ValidateParameters(ValueMap &vm); -protected: + protected: // A pointer to the Region object. This is the portion visible // to the applications. This class and it's subclasses are the // hidden implementations behind the Region class. diff --git a/src/htm/ntypes/Value.cpp b/src/htm/ntypes/Value.cpp new file mode 100644 index 0000000000..0fb5d522f0 --- /dev/null +++ b/src/htm/ntypes/Value.cpp @@ -0,0 +1,510 @@ +/* --------------------------------------------------------------------- + * HTM Community Edition of NuPIC + * Copyright (C) 2019, Numenta, Inc. + * + * Author: David Keeney, 10/2019 + * dkeeney@gmail.com + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU Affero Public License for more details. + * + * You should have received a copy of the GNU Affero Public License + * along with this program. If not, see http://www.gnu.org/licenses. + * --------------------------------------------------------------------- */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace htm; + +#define ZOMBIE_MAP ((size_t)-1) // Means the key of zombie was a map key +#define ZOMBIE_SEQ 0 // Means the key of zombie was a seq key + +////////////////////////////////////////////////////////////// +#ifdef YAML_PARSER_yamlcpp +#include + +// Parse YAML or JSON string document into the tree. +Value &Value::parse(const std::string &yaml_string) { + // If this Value node is being re-used (like in unit tests) + // we need to clear variables. + vec_.clear(); + map_.clear(); + scalar_ = ""; + type_ = Value::Category::Empty; + parent_ = nullptr; + + YAML::Node node = YAML::Load(yaml_string); + // walk the tree and copy data into our structure + + setNode(&node); + return *this; +} + +void Value::setNode(void *n) { + YAML::Node &node = *((YAML::Node *)n); + std::pair::iterator, bool> ret; + bool first = true; + if (node.IsScalar()) { + type_ = Value::Scalar; + scalar_ = node.as(); + } else if (node.IsSequence()) { + type_ = Value::Sequence; + for (size_t i = 0; i < node.size(); i++) { + Value itm; + itm.parent_ = this; + itm.index_ = vec_.size(); + itm.key_ = std::to_string(i); + // insert empty sequence into parent. (makes copy of itm) + ret = map_.insert(std::pair(itm.key_, itm)); + vec_.push_back(ret.first); + + // Now populate the new node. + Value *new_location = &ret.first->second; + new_location->setNode(&node[i]); + } + } else if (node.IsMap()) { + type_ = Value::Map; + for (auto it = node.begin(); it != node.end(); it++) { + Value itm; + itm.parent_ = this; + itm.index_ = vec_.size(); + itm.key_ = it->first.as(); // get the key + // insert empty map into parent. (makes copy of itm). + ret = map_.insert(std::pair(itm.key_, itm)); + vec_.push_back(ret.first); + + // Now populate the new node. + Value *new_location = &ret.first->second; + new_location->setNode(&it->second); + } + } +} + +#endif // YAML_PARSER_yamlcpp + +///////////////////////////////////////////////////////////////////////////////////////// + +// Constructor +Value::Value() { + type_ = Value::Category::Empty; + parent_ = nullptr; + zombie_ = nullptr; + assigned_ = nullptr; + index_ = ZOMBIE_MAP; +} + + +// checking content of a parameter +// enum ValueMap::Category { Empty = 0, Scalar, Sequence, Map }; +ValueMap::Category Value::getCategory() const { return type_; } + +bool Value::contains(const std::string& key) const { return (map_.find(key) != map_.end()); } + +bool Value::isScalar() const { return type_ == Value::Category::Scalar; } +bool Value::isSequence() const { return type_ == Value::Category::Sequence; } +bool Value::isMap() const { return type_ == Value::Category::Map; } +bool Value::isEmpty() const { return type_ == Value::Category::Empty; } + +size_t Value::size() const { + NTA_CHECK(map_.size() == vec_.size()) << "Detected Corruption of ValueMap structure"; + return map_.size(); +} + +// Accessing members of a map +// If not found, a Zombie Value object is returned. +// An error will be displayed when you try to access the value in the Zombie Value object. +// If you assign something to the Zombie Value, it will insert it into the tree with the saved key. +Value& Value::operator[](const std::string& key) { + if (assigned_) + return (*assigned_)[key]; + auto it = map_.find(key); + if (it == map_.end()) { + // not found. Create a zombie in case we will later assign something to this key. + // Its type is Value::Category::Empty. + // NOTE: only one zombie per parent can exist at a time. + zombie_.reset(new Value()); + zombie_->parent_ = this; + zombie_->scalar_ = key; + zombie_->key_ = key; + zombie_->index_ = ZOMBIE_MAP; + return *zombie_; + } + else + return it->second; +} +const Value& Value::operator[](const std::string& key) const { + if (assigned_) + return (*assigned_)[key]; + auto it = map_.find(key); + if (it == map_.end()) { + // not found. Create a (const) zombie which signals if found or not but + // cannot be used with an assignment. Its type is Value::Category::Empty. + static Value const_zombie; // This is a constant + return const_zombie; + } + else + return it->second; +} + +// accessing members of a sequence +Value& Value::operator[](size_t index) { + if (assigned_) + return (*assigned_)[index]; + if (index < vec_.size()) + return vec_[index]->second; + else if (index == vec_.size()) { + // Not found, create a zombie in case we later assign it. + // Note that the index can ONLY be the size-of-vector. + // Make sure the key is uneque. append '-'s until it is. + std::string key = std::to_string(index); + while (true) { + if (map_.find(key) == map_.end()) + break; + key += "-"; + } + zombie_.reset(new Value()); + zombie_->parent_ = this; + zombie_->key_ = key; + zombie_->index_ = ZOMBIE_SEQ; + return *zombie_; + } + NTA_THROW << "Index out of range; " << index; +} +const Value& Value::operator[](size_t index) const { + if (assigned_) + return (*assigned_)[index]; + if (index < vec_.size()) + return vec_[index]->second; + NTA_THROW << "Index out of range; " << index; // is const so cannot make a zombie +} + +std::string Value::str() const { + NTA_CHECK(type_ == Value::Category::Scalar); + return scalar_; +} +const char* Value::c_str() const { + NTA_CHECK(type_ == Value::Category::Scalar); + return scalar_.c_str(); +} + +std::string Value::key() const { return key_; } + +std::vector Value::getKeys() const { + NTA_CHECK(isMap()) << "This is not a map."; + std::vector v; + for (auto it = begin(); it != end(); it++) { + v.push_back(it->first); + } + return v; +} + +// Insert this node into the parent. +// Requires that there was a key (either string or index) unless it is root +// and if it was a string key, its index will be ZOMBIE_MAP. +void Value::addToParent() { + std::pair::iterator, bool> ret; + if (parent_ == nullptr) + return; // This is the root + NTA_CHECK(!key_.empty()) << "No key was provided. Use node[key] = value."; + if (parent_->type_ == Value::Category::Empty) { + parent_->addToParent(); + if (parent_->assigned_) + parent_ = parent_->assigned_; + } + bool map_key = (index_ == ZOMBIE_MAP); + + // Add the node to the parent. + index_ = parent_->vec_.size(); + ret = parent_->map_.insert(std::pair(key_, *this)); + parent_->vec_.push_back(ret.first); + assigned_ = &ret.first->second; + + NTA_CHECK(parent_->map_.size() == parent_->vec_.size()) << "Detected Corruption of ValueMap structure"; + + if (map_key) + parent_->type_ = Value::Category::Map; + else if (parent_->type_ == Value::Category::Empty) + parent_->type_ = Value::Category::Sequence; +} + +void Value::assign(std::string val) { + std::pair::iterator, bool> ret; + if (type_ == Value::Category::Empty) { // previous search was false + // This is a zombie node. By assigning a value we add it to the tree. + // The key was already set in the operator[]. + + // Add to parent. + // If its parent is also a zombie, add it to the tree as well. + addToParent(); + assigned_->scalar_ = val; + assigned_->type_ = Value::Category::Scalar; + } else { + // Must be a value already in the tree. Do a replace. + if (type_ != Value::Category::Scalar) { + map_.clear(); + vec_.clear(); + type_ = Value::Category::Scalar; + } + scalar_ = val; + } +} +// Assign a value converted from a specified type T. +void Value::operator=(char *val) { assign(val); } +void Value::operator=(const std::string &val) { assign(val); } +void Value::operator=(int8_t val) { assign(std::to_string(val)); } +void Value::operator=(int16_t val) { assign(std::to_string(val)); } +void Value::operator=(uint16_t val) { assign(std::to_string(val)); } +void Value::operator=(int32_t val) { assign(std::to_string(val)); } +void Value::operator=(uint32_t val) { assign(std::to_string(val)); } +void Value::operator=(int64_t val) { assign(std::to_string(val)); } +void Value::operator=(uint64_t val) { assign(std::to_string(val)); } +void Value::operator=(bool val) { assign((val) ? "true" : "false"); } +void Value::operator=(float val) { assign(std::to_string(val)); } +void Value::operator=(double val) { assign(std::to_string(val)); } +void Value::operator=(std::vector val) { + // Insert the contents of the vector into this node. + map_.clear(); + vec_.clear(); + for (size_t i = 0; i < val.size(); i++) { + operator[](i) = std::to_string(val[i]); + } +} + + + +void Value::remove() { + Value *node = (assigned_) ? assigned_ : this; + NTA_CHECK(!node->isEmpty()) << "Item not found."; // current node is a zombie. + if (node->parent_ == nullptr) { + // This is root. Just clear the map. + node->vec_.clear(); + node->map_.clear(); + node->type_ = Value::Category::Empty; + return; + } + NTA_CHECK(node->parent_->vec_[index_]->second == *node); + if (node->parent_->vec_.size() == 1) { + // Last node in parent, remove parent. + node->parent_->remove(); + return; + } + std::string key = node->parent_->vec_[index_]->first; + auto itr = node->parent_->map_.find(key); + + // adjust the index on all following items. + // We have to do it here because as soon as we erase the map item + // it will delete 'this'. + for (size_t i = index_+1; i < node->parent_->vec_.size(); i++) { + node->parent_->vec_[i]->second.index_ = i-1; + } + + node->parent_->vec_.erase(node->parent_->vec_.begin() + index_); + node->parent_->map_.erase(itr); + // The node object is deleted. Do no try to access it. +} + +// Compare two nodes recursively to see if content is same. +static bool equals(const Value &n1, const Value &n2) { + if (n1.getCategory() != n2.getCategory()) + return false; + if (n1.isSequence()) { + if (n1.size() != n2.size()) + return false; + for (size_t i = 0; i < n1.size(); i++) + if (!equals(n1[i], n2[i])) + return false; + return true; + } + if (n1.isMap()) { + if (n1.size() != n2.size()) + return false; + for (auto it : n1) { + if (!n2[it.first]) + return false; + if (!equals(it.second, n2[it.first])) + return false; + } + return true; + } + if (n1.isScalar()) { + if (n1.str() == n2.str()) + return true; + } + return false; +} +bool Value::operator==(const Value &v) const { return equals(*this, v); } + +// Explicit instantiations for as() +template int8_t Value::as() const; +template int16_t Value::as() const; +template uint16_t Value::as() const; +template int32_t Value::as() const; +template uint32_t Value::as() const; +template int64_t Value::as() const; +template uint64_t Value::as() const; +template float Value::as() const; +template double Value::as() const; +template std::string Value::as() const; +template bool Value::as() const; + +/** + * a local function to apply escapes for a JSON string. + */ +static void escape_json(std::ostream &o, const std::string &s) { + for (auto c = s.cbegin(); c != s.cend(); c++) { + switch (*c) { + case '"': + o << "\\\""; + break; + case '\\': + o << "\\\\"; + break; + case '\b': + o << "\\b"; + break; + case '\f': + o << "\\f"; + break; + case '\n': + o << "\\n"; + break; + case '\r': + o << "\\r"; + break; + case '\t': + o << "\\t"; + break; + default: + if ('\x00' <= *c && *c <= '\x1f') { + o << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int)*c; + } else { + o << *c; + } + } + } +} + +static void to_json(std::ostream &f, const htm::Value &v) { + bool first = true; + std::string s; + switch (v.getCategory()) { + case Value::Empty: + return; + case Value::Scalar: + s = v.str(); + if (std::regex_match(s, std::regex("^[-+]?[0-9]+([.][0-9]+)?$"))) { + escape_json(f, s); + } else { + f << '"'; + escape_json(f, s); + f << '"'; + } + break; + case Value::Sequence: + f << "["; + for (size_t i = 0; i < v.size(); i++) { + if (!first) + f << ", "; + first = false; + const Value &n = v[i]; + to_json(f, n); + } + f << "]"; + break; + case Value::Map: + f << "{"; + for (size_t i = 0; i < v.size(); i++) { + if (!first) + f << ", "; + first = false; + const Value &n = v[i]; + f << n.key() << ": "; + to_json(f, n); + } + f << "}"; + break; + } +} + +std::string Value::to_json() const { + std::stringstream f; + ::to_json(f, *this); + return f.str(); +} + +static void escape_yaml(std::ostream &o, const std::string &s, const std::string &indent) { + if (std::strchr(s.c_str(), '\n')) { + // contains newlines + o << " |"; // all blanks are significant + const char *from = s.c_str(); + const char *to = from; + while ((to = std::strchr(to, '\n')) != NULL) { + std::string line(from, to); + o << "\n" + indent + line; + ++to; + from = to; + } + o << "\n" + indent + from; + } else { + o << s; + } +} + +static void to_yaml(std::ostream & f, const htm::Value &v, std::string indent) { + bool first = true; + std::string s; + switch (v.getCategory()) { + case Value::Empty: + return; + case Value::Scalar: + s = v.str(); + escape_yaml(f, s, indent); + f << "\n"; + break; + case Value::Sequence: + for (size_t i = 0; i < v.size(); i++) { + const Value &n = v[i]; + f << indent << "- "; + if (n.isMap() || n.isSequence()) + f << "\n"; + to_yaml(f, n, indent + " "); + } + break; + case Value::Map: + for (size_t i = 0; i < v.size(); i++) { + const Value &n = v[i]; + f << indent << n.key() << ": "; + if (n.isMap() || n.isSequence()) + f << "\n"; + to_yaml(f, n, indent + " "); + } + break; + } +} + +std::string Value::to_yaml() const { + std::stringstream f; + ::to_yaml(f, *this, ""); + return f.str(); +} + +namespace htm { +std::ostream &operator<<(std::ostream &f, const htm::Value &v) { + f << v.to_json(); + return f; +} +} // namespace htm diff --git a/src/htm/ntypes/Value.hpp b/src/htm/ntypes/Value.hpp index e3132f9174..441ec9eb4a 100644 --- a/src/htm/ntypes/Value.hpp +++ b/src/htm/ntypes/Value.hpp @@ -25,8 +25,10 @@ #include #include #include -#include +#include #include // transform +#include +#include // std::strerror(errno) namespace htm { @@ -37,10 +39,9 @@ class Value { Value(); // Parse a Yaml or JSON string an assign it to this node in the tree. - Value& parse(const std::string &yaml_string); + Value &parse(const std::string &yaml_string); - - bool contains(const std::string &key)const; + bool contains(const std::string &key) const; size_t size() const; // type of value in the Value node @@ -49,21 +50,76 @@ class Value { bool isScalar() const; bool isSequence() const; bool isMap() const; - bool isEmpty() const; // false if current node has a value,sequence,or map - // true if operator[] did not find a value or was not assigned to. + bool isEmpty() const; // false if current node has a value,sequence,or map + // true if operator[] did not find a value or was not assigned to. // Access - Value operator[](const std::string &key); - Value operator[](size_t index); - const Value operator[](const std::string &key) const; - const Value operator[](size_t index) const; - template T as() const; - template T as(T default_value) const; - std::string str() const; + std::string str() const; // return copy of raw value as a string + std::string key() const; // return copy of raw key as a string + const char *c_str() const; // return a const pointer to raw value std::vector getKeys() const; + Value &operator[](const std::string &key); + Value &operator[](size_t index); + const Value &operator[](const std::string &key) const; + const Value &operator[](size_t index) const; + + template T as() const { + if (type_ != Category::Scalar) + NTA_THROW << "not found. '" << scalar_ << "'"; + T result; + std::stringstream ss(scalar_); + ss >> result; + return result; + } +#define NTA_CONVERT(T, I) \ + if (type_ != Value::Category::Scalar) \ + NTA_THROW << "value not found."; \ + errno = 0; \ + char *end; \ + T val = (T)I; \ + if (errno) \ + NTA_THROW << "In '" << scalar_ << "' numeric conversion error: " << std::strerror(errno); \ + if (*end != '\0') \ + NTA_THROW << "In '" << scalar_ << "' numeric conversion error: invalid char."; \ + return val; + + // Return a value converted to the specified type T. + // explicit specializations. By default it uses streams to convert. + template <> int8_t as() const { NTA_CONVERT(int8_t, std::strtol(scalar_.c_str(), &end, 0)); } + template <> int16_t as() const { NTA_CONVERT(int16_t, std::strtol(scalar_.c_str(), &end, 0)); } + template <> uint16_t as() const { NTA_CONVERT(uint16_t, std::strtoul(scalar_.c_str(), &end, 0)); } + template <> int32_t as() const { NTA_CONVERT(int32_t, std::strtol(scalar_.c_str(), &end, 0)); } + template <> uint32_t as() const { NTA_CONVERT(uint32_t, std::strtoul(scalar_.c_str(), &end, 0)); } + template <> int64_t as() const { NTA_CONVERT(int64_t, std::strtoll(scalar_.c_str(), &end, 0)); } + template <> uint64_t as() const { NTA_CONVERT(uint64_t, std::strtoull(scalar_.c_str(), &end, 0)); } + template <> float as() const { NTA_CONVERT(float, std::strtof(scalar_.c_str(), &end)); } + template <> double as() const { NTA_CONVERT(double, std::strtod(scalar_.c_str(), &end)); } + template <> std::string as() const { + if (type_ != Value::Category::Scalar) + NTA_THROW << "value not found."; + return scalar_; + } + template <> bool as() const { + if (type_ != Value::Category::Scalar) + NTA_THROW << "value not found. " << scalar_; + std::string val = str(); + transform(val.begin(), val.end(), val.begin(), ::tolower); + if (val == "true" || val == "on" || val == "1") + return true; + if (val == "false" || val == "off" || val == "0") + return false; + NTA_THROW << "Invalid value for a boolean. " << val; + } + + template T as(T d) const { + if (type_ != Value::Category::Scalar) + return d; + return as(); + } + /** Assign a value to a Value node in the tree. - * If a previous operator[] did find a match, this does a replace. + * If a previous operator[] found a match, this does a replace. * If a previous operator[] did not find a match in the tree * the current Value is a Zombie and not attached to the tree. * But in this case its requested key is remembered. A subsequent @@ -71,7 +127,7 @@ class Value { * The parent will become a map if it is not already. */ void operator=(char *val); - void operator=(const std::string& val); + void operator=(const std::string &val); void operator=(int8_t val); void operator=(int16_t val); void operator=(uint16_t val); @@ -84,25 +140,31 @@ class Value { void operator=(double val); void operator=(std::vector); + /** + * Remove a node from the tree. (an element of map or sequence) + * It is assumed that this is a low use function because + * it will not be very efficent. + */ + void remove(); + // compare two nodes in the tree. - bool operator==(const Value &v); + bool operator==(const Value &v) const; // return false if node is empty: if (vm) { do something } explicit operator bool() const { return !isEmpty(); } - // extract a Vector template std::vector asVector() const { std::vector v; if (!isSequence()) NTA_THROW << "Not a sequence node."; for (size_t i = 0; i < size(); i++) { // iterate through the children of this node. - const Value n = (*this)[i]; + const Value &n = (*this)[i]; try { if (n.isScalar()) { v.push_back(n.as()); } - } catch (std::exception& e) { + } catch (std::exception &e) { NTA_THROW << "Invalid vector element; " << e.what(); } } @@ -112,33 +174,27 @@ class Value { // extract a map. Key is always a string. template std::map asMap() const { std::map v; - if (!isSequence()) - NTA_THROW << "Not a Map node."; for (auto iter = cbegin(); iter != cend(); iter++) { // iterate through the children of this node. - const std::string key = iter->first; - const Value n = iter->second; + const Value &n = iter->second; try { if (n.isScalar()) { - v[key] = n.as(); - } - else { + v[n.key_] = n.as(); + } else { // non-scalar field. Ignore } - } catch (std::exception& e) { + } catch (std::exception &e) { // probably bad conversion of scalar to requested type. - NTA_THROW << "Invalid map element[" << key << "] " << e.what(); + NTA_THROW << "Invalid map element[\"" << n.key_ << "\"] " << e.what(); } } return v; } - // serializing routines std::string to_yaml() const; std::string to_json() const; - - // Access for backward compatability + // Access for backward compatability template T getScalarT(const std::string &key) const { // throws return (*this)[key].as(); } @@ -151,60 +207,30 @@ class Value { friend std::ostream &operator<<(std::ostream &f, const htm::Value &vm); - // Iterator as a map - class map_iterator { - public: - using value_type = Value; - using reference = value_type &; - using pointer = std::pair *; - using difference_type = std::ptrdiff_t; - using iterator_category = std::forward_iterator_tag; - - map_iterator operator++(); - map_iterator operator++(int junk); - reference operator*(); - pointer operator->(); - bool operator==(const map_iterator &rhs) const; - bool operator!=(const map_iterator &rhs) const; - - struct OpaqueIterator; - std::shared_ptr ptr_; - }; - - class map_const_iterator { - public: - using value_type = const Value; - using reference = value_type &; - using pointer = const std::pair *; - using difference_type = std::ptrdiff_t; - using iterator_category = std::forward_iterator_tag; - - map_const_iterator operator++(); - map_const_iterator operator++(int junk); - reference operator*(); - pointer operator->(); - bool operator==(const map_const_iterator &rhs) const; - bool operator!=(const map_const_iterator &rhs) const; - - struct OpaqueConstIterator; - std::shared_ptr ptr_; - }; - - - map_iterator begin(); - map_iterator end(); - map_const_iterator begin() const { return cbegin(); } - map_const_iterator end() const { return cend(); } - map_const_iterator cbegin() const; - map_const_iterator cend() const; - + std::map::iterator begin() { return map_.begin(); } + std::map::iterator end() { return map_.end(); } + std::map::const_iterator begin() const { return map_.begin(); } + std::map::const_iterator end() const { return map_.end(); } + std::map::const_iterator cbegin() const { return map_.cbegin(); } + std::map::const_iterator cend() const { return map_.cend(); } private: - struct OpaqueTree; - std::shared_ptr doc_; // This is an opaque pointer to implementation Tree node object. - + void setNode(void *node); // add opaque node. + void assign(std::string val); // add a scalar + void addToParent(); // Assign to the map/vector in the parent + + enum Category type_; // type of node + std::map map_; + std::vector::iterator> vec_; + Value *parent_; // a pointer to the parent node (do not delete) + std::string scalar_; // scalar value. Not used on Map or Sequence nodes + std::string key_; // The key for this node. + size_t index_; // initial index Also, ZOMBIE_MAP in zombie nodes. + std::shared_ptr zombie_; // pointer to a zombie child Value node. Returned when operator[] not found. + Value *assigned_; // For a zombie, a pointer to assigned node in tree. }; + using ValueMap = Value; } // namespace htm diff --git a/src/htm/regions/SPRegion.cpp b/src/htm/regions/SPRegion.cpp index a9d890478e..985837d861 100644 --- a/src/htm/regions/SPRegion.cpp +++ b/src/htm/regions/SPRegion.cpp @@ -274,7 +274,7 @@ Spec *SPRegion::createSpec() { NTA_BasicType_Bool, // type 1, // elementCount "bool", // constraints - "false", // defaultValue + "true", // defaultValue ParameterSpec::ReadWriteAccess)); // access ns->parameters.add( @@ -425,7 +425,7 @@ Spec *SPRegion::createSpec() { NTA_BasicType_Int32, // type 1, // elementCount "", // constraints - "-1", // defaultValue + "1", // defaultValue ParameterSpec::CreateAccess)); // access ns->parameters.add( diff --git a/src/htm/regions/ScalarSensor.cpp b/src/htm/regions/ScalarSensor.cpp index 41bc22f1dd..b40bd0e04b 100644 --- a/src/htm/regions/ScalarSensor.cpp +++ b/src/htm/regions/ScalarSensor.cpp @@ -203,21 +203,40 @@ ScalarSensor::~ScalarSensor() {} Real64 ScalarSensor::getParameterReal64(const std::string &name, Int64 index) { if (name == "sensedValue") { return sensedValue_; - } + } else if (name == "resolution") return encoder_->parameters.resolution; + else if (name == "radius") + return encoder_->parameters.radius; + else if (name == "minValue") + return encoder_->parameters.minimum; + else if (name == "maxValue") + return encoder_->parameters.maximum; else { return RegionImpl::getParameterReal64(name, index); } } +bool ScalarSensor::getParameterBool(const std::string& name, Int64 index) { + if (name == "periodic") + return encoder_->parameters.periodic; + if (name == "clipInput") + return encoder_->parameters.clipInput; + else { + return RegionImpl::getParameterBool(name, index); + } +} + UInt32 ScalarSensor::getParameterUInt32(const std::string &name, Int64 index) { if (name == "n") { return (UInt32)encoder_->size; } - else { + else if (name == "w") { + return encoder_->parameters.activeBits; + } else { return RegionImpl::getParameterUInt32(name, index); } } + void ScalarSensor::setParameterReal64(const std::string &name, Int64 index, Real64 value) { if (name == "sensedValue") { sensedValue_ = value; diff --git a/src/htm/regions/ScalarSensor.hpp b/src/htm/regions/ScalarSensor.hpp index afa53f51fa..ed1491108f 100644 --- a/src/htm/regions/ScalarSensor.hpp +++ b/src/htm/regions/ScalarSensor.hpp @@ -51,6 +51,7 @@ class ScalarSensor : public RegionImpl, Serializable { virtual Real64 getParameterReal64(const std::string &name, Int64 index = -1) override; virtual UInt32 getParameterUInt32(const std::string &name, Int64 index = -1) override; + virtual bool getParameterBool(const std::string &name, Int64 index = -1) override; virtual void setParameterReal64(const std::string &name, Int64 index, Real64 value) override; virtual void initialize() override; diff --git a/src/htm/regions/TMRegion.cpp b/src/htm/regions/TMRegion.cpp index d08fdbb620..75eeebbed6 100644 --- a/src/htm/regions/TMRegion.cpp +++ b/src/htm/regions/TMRegion.cpp @@ -338,7 +338,7 @@ Spec *TMRegion::createSpec() { NTA_BasicType_UInt32, // type 1, // elementCount "", // constraints - "8", // defaultValue + "10", // defaultValue ParameterSpec::CreateAccess)); // access ns->parameters.add( diff --git a/src/htm/regions/TestNode.cpp b/src/htm/regions/TestNode.cpp index 7775313bb9..0d8ce81106 100644 --- a/src/htm/regions/TestNode.cpp +++ b/src/htm/regions/TestNode.cpp @@ -51,7 +51,7 @@ TestNode::TestNode(const ValueMap ¶ms, Region *region) shouldCloneParam_ = params.getScalarT("shouldCloneParam", 1) != 0; - stringParam_ = params.getString("stringParam", ""); + stringParam_ = params.getString("stringParam", "nodespec value"); real32ArrayParam_.resize(8); for (size_t i = 0; i < 8; i++) { diff --git a/src/test/unit/engine/CppRegionTest.cpp b/src/test/unit/engine/CppRegionTest.cpp index c57c42af34..c6966a7c08 100644 --- a/src/test/unit/engine/CppRegionTest.cpp +++ b/src/test/unit/engine/CppRegionTest.cpp @@ -118,6 +118,9 @@ TEST(CppRegionTest, testCppLinkingSDR) { net.initialize(); + EXPECT_EQ(region1->getParameterUInt32("n"), 6u); + EXPECT_EQ(region1->getParameterUInt32("w"), 2u); + region1->setParameterReal64("sensedValue", 0.8); //Note: default range setting is -1.0 to +1.0 const Dimensions r1dims = region1->getOutput("encoded")->getDimensions(); EXPECT_EQ(r1dims.size(), 2u) << " actual dims: " << r1dims.toString(); @@ -153,22 +156,17 @@ TEST(CppRegionTest, testCppLinkingSDR) { << "Expected dimensions on the output to match dimensions on the buffer."; VERBOSE << r2OutputArray << "\n"; SDR exp({20u, 3u}); - exp.setSparse(SDR_sparse_t{ - 4, 21, 32, 46 - }); - EXPECT_EQ(r2OutputArray, exp.getDense()) << "got " << r2OutputArray; + exp.setSparse(SDR_sparse_t{10, 38, 57}); + EXPECT_EQ(r2OutputArray, exp.getDense()) << "expected " << exp << " got " << r2OutputArray; } TEST(CppRegionTest, testYAML) { const char *params = "{count: 42, int32Param: 1234, real64Param: 23.1}"; - // badparams contains a non-existent parameter - const char *badparams = "{int32Param: 1234, real64Param: 23.1, badParam: 4}"; Network net; std::shared_ptr level1; - EXPECT_THROW(net.addRegion("level1", "TestNode", badparams), exception); EXPECT_NO_THROW({level1 = net.addRegion("level1", "TestNode", params);}); diff --git a/src/test/unit/engine/WatcherTest.cpp b/src/test/unit/engine/WatcherTest.cpp index e6688eb47e..2502eab37d 100644 --- a/src/test/unit/engine/WatcherTest.cpp +++ b/src/test/unit/engine/WatcherTest.cpp @@ -36,7 +36,7 @@ namespace testing { -static bool verbose = false; +static bool verbose = true; #define VERBOSE \ if (verbose) \ std::cerr << "[ ] " @@ -125,8 +125,7 @@ TEST(WatcherTest, FileTest1) { std::string tempString; if (inStream.is_open()) { getline(inStream, tempString); - ASSERT_EQ("Info: watchID, regionName, nodeType, nodeIndex, varName", - tempString); + ASSERT_EQ("Info: watchID, regionName, nodeType, nodeIndex, varName", tempString); getline(inStream, tempString); ASSERT_EQ("1, level1, TestNode, -1, uint32Param", tempString); getline(inStream, tempString); diff --git a/src/test/unit/ntypes/ValueTest.cpp b/src/test/unit/ntypes/ValueTest.cpp index 2a1f17f1ae..41406fb136 100644 --- a/src/test/unit/ntypes/ValueTest.cpp +++ b/src/test/unit/ntypes/ValueTest.cpp @@ -29,7 +29,6 @@ namespace testing { using namespace htm; - TEST(ValueTest, toValueNumber) { ValueMap vm; @@ -56,9 +55,9 @@ TEST(ValueTest, toValueNumber) { i = vm[0].as(); EXPECT_EQ(123, i); - EXPECT_ANY_THROW(Int32 i = vm.parse("999999999999999999999999999").as()); - EXPECT_ANY_THROW(Int32 i = vm.parse("abc").as()); - EXPECT_ANY_THROW(Int32 i = vm.parse("").as()); + EXPECT_ANY_THROW(vm.parse("999999999999999999999999999").as()); + EXPECT_ANY_THROW(vm.parse("abc").as()); + EXPECT_ANY_THROW(vm.parse("").as()); vm.parse("A: 1"); EXPECT_EQ(1, vm["A"].as(123)); // with a default value but found @@ -114,12 +113,11 @@ TEST(ValueTest, toValueBool) { vm.parse("B: false"); EXPECT_FALSE(vm.getScalarT("B", true)); EXPECT_FALSE(vm.parse("B: 0").getScalarT("B", true)); - EXPECT_ANY_THROW(bool result = vm.parse("B: 1234").getScalarT("B", false)); + EXPECT_ANY_THROW(vm.parse("B: 1234").getScalarT("B", false)); } TEST(ValueTest, asArray) { ValueMap vm; - Value v; std::vector s2 = {10u, 20u, 30u, 40u, 50u}; std::string json = "[10,20,30,40,50]"; @@ -140,6 +138,9 @@ TEST(ValueTest, asArray) { std::vector s3 = {100u, 200u, 300u, 400u, 500u}; vm[5] = s3; // assign an array to the 6th element. + std::string t = vm.to_json(); + EXPECT_STREQ(t.c_str(), "[10, 20, 30, 40, 50, [100, 200, 300, 400, 500]]"); + EXPECT_EQ(vm[0].as(), 10u); EXPECT_TRUE(vm[5].isSequence()); EXPECT_TRUE(vm[5][4].isScalar()); @@ -151,9 +152,11 @@ TEST(ValueTest, asArray) { TEST(ValueTest, asMap) { ValueMap vm; - std::string src = "{scalar: \"456\", array: [\"1\", \"2\", \"3\", \"4\"], string: \"true\"}"; + std::string src = "{scalar: 456, array: [1, 2, 3, 4], string: \"true\"}"; vm.parse(src); + //std::cout << vm << "\n"; + std::map m; m = vm.asMap(); std::stringstream ss; @@ -167,7 +170,7 @@ TEST(ValueTest, asMap) { ss << "}"; std::string result = ss.str(); std::cout << result << "\n"; - EXPECT_STREQ(result.c_str(), "{scalar: \"456\", string: \"true\"}"); + EXPECT_STREQ(result.c_str(), "{scalar: 456, string: true}"); } @@ -188,7 +191,26 @@ TEST(ValueTest, String) { EXPECT_STREQ("\"hello world\"", v.to_json().c_str()); } -TEST(ValueTest, ValueMap) { + + +TEST(ValueTest, inserts) { + ValueMap vm; + vm[0][0][0] = 1; + //std::cout << "inserts1: " << vm << "\n"; + EXPECT_STREQ("[[[1]]]", vm.to_json().c_str()); + + Value v = vm[1]; // Create a zombie node + for (int i = 0; i < 3; i++) { + v[i] = i*100+100; // assign to an array on a zombie which should add it to the tree. + } + //std::cout << "inserts2: " << vm << "\n"; + EXPECT_STREQ("[[[1]], [100, 200, 300]]", vm.to_json().c_str()); + + EXPECT_ANY_THROW(vm[3]["hello"] = "world"); +} + + +TEST(ValueTest, ValueMapTest) { std::vector a = {1, 2, 3, 4}; ValueMap vm; vm["scalar"] = 123; @@ -213,7 +235,7 @@ TEST(ValueTest, ValueMap) { Int32 x = vm.getScalarT("scalar2", (Int32)20); EXPECT_EQ((Int32)20, x); - std::string expected = "{scalar: \"456\", array: [\"1\", \"2\", \"3\", \"4\"], string: \"true\"}"; + std::string expected = "{scalar: 456, array: [1, 2, 3, 4], string: \"str\"}"; std::string result; std::stringstream ss; ss << vm; @@ -225,8 +247,8 @@ TEST(ValueTest, ValueMap) { ValueMap vm2; vm2.parse(result); - std::cout << "vm=" << vm << "\n"; - std::cout << "vm2" << vm2 << "\n"; + //std::cout << "vm=" << vm << "\n"; + //std::cout << "vm2" << vm2 << "\n"; EXPECT_TRUE(vm == vm2); } @@ -235,18 +257,17 @@ TEST(ValueTest, Iterations) { std::vector results; int cnt = 0; - std::string data = R"( -scalar: 123.45 -array: + std::string data = R"(scalar: 123.45 +array: - 1 - 2 - 3 - 4 -string: "this is a string" +string: this is a string )"; vm.parse(data); - std::cout << vm << "\n"; + //std::cout << vm << "\n"; for (auto itr = vm.begin(); itr != vm.end(); itr++) { std::string key = itr->first; if (key == "scalar") @@ -263,14 +284,77 @@ string: "this is a string" // iterate with for range cnt = 0; - for (Value& itm : vm) { - if (itm.isScalar()) + for (auto itm : vm) { + if (itm.second.isScalar()) cnt++; - if (itm.isSequence()) + if (itm.second.isSequence()) cnt++; } EXPECT_EQ(cnt, 3); + std::string result = vm.to_yaml(); + //std::cout << result << "\n"; + EXPECT_STREQ(result.c_str(), data.c_str()); + +} + +TEST(ValueTest, deletes) { + ValueMap vm; + std::string src = "{scalar: 456, array: [1, 2, 3, 4], string: \"true\"}"; + vm.parse(src); + + EXPECT_EQ(vm.size(), 3); + EXPECT_EQ(vm["array"].size(), 4); + + vm["scalar"].remove(); + EXPECT_EQ(vm.size(), 2); + EXPECT_ANY_THROW(vm["scalar"].str()); + EXPECT_TRUE(vm[0].isSequence()); + EXPECT_TRUE(vm[0][0].isScalar()); + EXPECT_EQ(vm[0][0].as(), 1); + EXPECT_EQ(vm[0][3].as(), 4); + EXPECT_EQ(vm[0].size(), 4); + + vm[0][0].remove(); + EXPECT_EQ(vm[0].size(), 3); + EXPECT_TRUE(vm[0][0].isScalar()); + EXPECT_EQ(vm[0][0].as(), 2); + EXPECT_EQ(vm[0][2].as(), 4); + EXPECT_ANY_THROW(vm[0][3].as()); + + vm[0][2].remove(); + EXPECT_EQ(vm[0].size(), 2); + EXPECT_TRUE(vm[0][1].isScalar()); + EXPECT_EQ(vm[0][0].as(), 2); + EXPECT_EQ(vm[0][1].as(), 3); + EXPECT_ANY_THROW(vm[0][2].as()); + + vm[0][2] = 6; + EXPECT_EQ(vm[0].size(), 3); + EXPECT_TRUE(vm[0][1].isScalar()); + EXPECT_EQ(vm[0][0].as(), 2); + EXPECT_EQ(vm[0][1].as(), 3); + EXPECT_ANY_THROW(vm[0][3].as()); + std::vector v = vm[0].asVector(); + + std::vector expected({2, 3, 6}); + for (size_t i = 0; i < 3; i++) { + EXPECT_EQ(expected[i], v[i]); + } + + vm[0][2].remove(); + EXPECT_EQ(vm[0].size(), 2); + for (size_t i = 0; i < 2; i++) { + EXPECT_EQ(expected[i], v[i]); + } + + vm[0].remove(); + EXPECT_EQ(vm.size(), 1); + EXPECT_TRUE(vm[0].isScalar()); + EXPECT_TRUE(vm.contains("string")); + + vm.remove(); + EXPECT_TRUE(vm.isEmpty()); } } // namespace testing diff --git a/src/test/unit/regions/RegionTestUtilities.cpp b/src/test/unit/regions/RegionTestUtilities.cpp index 3405fad352..088957990d 100644 --- a/src/test/unit/regions/RegionTestUtilities.cpp +++ b/src/test/unit/regions/RegionTestUtilities.cpp @@ -68,11 +68,6 @@ void checkGetSetAgainstSpec(std::shared_ptr region1, switch (p.second.dataType) { case NTA_BasicType_UInt32: { - if (negativeCheck) { - negativeCheck = false; - VERBOSE << "negative check..." << std::endl; - EXPECT_THROW(region1->getParameterInt32("bad_parameter"), htm::Exception); - } VERBOSE << "Parameter \"" << name << "\" type: " << BasicType::getName(p.second.dataType) << std::endl; // check the getter. diff --git a/src/test/unit/regions/TMRegionTest.cpp b/src/test/unit/regions/TMRegionTest.cpp index 69cfb304c5..7201ada382 100644 --- a/src/test/unit/regions/TMRegionTest.cpp +++ b/src/test/unit/regions/TMRegionTest.cpp @@ -214,7 +214,7 @@ TEST(TMRegionTest, testLinking) { std::shared_ptr region1 = net.addRegion("region1", "VectorFileSensor",parameters); std::shared_ptr region2 = net.addRegion("region2", "SPRegion", "{dim: [2,10]}"); std::shared_ptr region3 = net.addRegion("region3", "TMRegion", - "{activationThreshold: 9, cellsPerColumn: 5}"); + "{activationThreshold: 12, cellsPerColumn: 5}"); std::shared_ptr region4 = net.addRegion("region4", "VectorFileEffector", "{outputFile: '" + test_output_file + "'}"); @@ -271,7 +271,7 @@ TEST(TMRegionTest, testLinking) { VERBOSE << " " << r3InputArray << "\n"; std::vector expected3in = VectorHelpers::sparseToBinary( { - 7 + 19 }, (UInt32)r3InputArray.getCount()); EXPECT_EQ(r3InputArray, expected3in) << r3InputArray; @@ -288,9 +288,7 @@ TEST(TMRegionTest, testLinking) { EXPECT_TRUE(r3OutputArray.getType() == NTA_BasicType_SDR); VERBOSE << " " << r3OutputArray << "\n"; std::vector expected3out = VectorHelpers::sparseToBinary( //TODO replace with SDR - { - 35, 36, 37, 38, 39 - }, (UInt32)r3OutputArray.getCount()); + {95, 96, 97, 98, 99 }, (UInt32)r3OutputArray.getCount()); EXPECT_EQ(r3OutputArray, expected3out) << r3OutputArray; EXPECT_EQ(r3OutputArray.getSDR().getSparse().size(), 5u); @@ -307,9 +305,7 @@ TEST(TMRegionTest, testLinking) { << numberOfCols << " * " << cellsPerColumn; VERBOSE << " " << r3OutputArray << ")\n"; std::vector expected3outa = VectorHelpers::sparseToBinary( - { - 95, 96, 97, 98, 99 - }, (UInt32)r3OutputArray.getCount()); + {60, 61, 62, 63, 64 }, (UInt32)r3OutputArray.getCount()); EXPECT_EQ(r3OutputArray, expected3outa) << r3OutputArray;