From f60e82bac760ae86f750855cbdbffcf508c6c25c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 10:13:23 +0100 Subject: [PATCH 1/9] FIX improve double2string function to cope with very large float numbers --- src/lib/common/string.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/lib/common/string.cpp b/src/lib/common/string.cpp index 2bfc72e99a..7da83064e9 100644 --- a/src/lib/common/string.cpp +++ b/src/lib/common/string.cpp @@ -22,6 +22,8 @@ * * Author: Ken Zangelin */ + +#include // numeric_limits #include #include #include @@ -949,8 +951,14 @@ bool str2double(const char* s, double* dP) return true; } +std::string double2string(double value) +{ + std::ostringstream oss; + oss << std::setprecision(std::numeric_limits::max_digits10) << value; + return oss.str(); +} - +#if 0 /* **************************************************************************** * * double2string @@ -1006,7 +1014,7 @@ std::string double2string(double f) return std::string(buf); } - +#endif /***************************************************************************** From c843fd735c01ce37710f5c793d87649c91de7e8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 10:47:36 +0100 Subject: [PATCH 2/9] FIX double2string in C style --- src/lib/common/string.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/common/string.cpp b/src/lib/common/string.cpp index 7da83064e9..ffe976b5e4 100644 --- a/src/lib/common/string.cpp +++ b/src/lib/common/string.cpp @@ -953,9 +953,9 @@ bool str2double(const char* s, double* dP) std::string double2string(double value) { - std::ostringstream oss; - oss << std::setprecision(std::numeric_limits::max_digits10) << value; - return oss.str(); + char buffer[32]; + snprintf(buffer, sizeof(buffer), "%.17g", value); + return std::string(buffer); } #if 0 From 381c4af4a3706250744ce06bed75471bc2b4e643 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 12:33:53 +0100 Subject: [PATCH 3/9] FIX improve existing double2string --- src/lib/common/string.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/lib/common/string.cpp b/src/lib/common/string.cpp index ffe976b5e4..95bdf39b95 100644 --- a/src/lib/common/string.cpp +++ b/src/lib/common/string.cpp @@ -951,14 +951,16 @@ bool str2double(const char* s, double* dP) return true; } +#if 0 std::string double2string(double value) { char buffer[32]; snprintf(buffer, sizeof(buffer), "%.17g", value); return std::string(buffer); } +#endif + -#if 0 /* **************************************************************************** * * double2string @@ -976,7 +978,12 @@ std::string double2string(double f) // abs value for 'diff' diff = (diff < 0)? -diff : diff; - if (diff > 0.9999999998) + if (diff > 1) + { + // this is an overflow situation, pe. f = 1.79e308 + isInteger = false; + } + else if (diff > 0.9999999998) { intPart += 1; isInteger = true; @@ -992,7 +999,8 @@ std::string double2string(double f) } else { - snprintf(buf, bufSize, "%.9f", f); + //snprintf(buf, bufSize, "%.9f", f); + snprintf(buf, bufSize, "%.17g", f); // Clear out any unwanted trailing zeroes char* last = &buf[strlen(buf) - 1]; @@ -1014,7 +1022,7 @@ std::string double2string(double f) return std::string(buf); } -#endif + /***************************************************************************** From a4be4dbb751cd54e5e7131c7471e6f0dc21e4a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 14:08:21 +0100 Subject: [PATCH 4/9] FIX improve existing double2string plug ftest for large numbers --- src/lib/common/string.cpp | 13 ++- .../large_float_numbers.test | 91 +++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test diff --git a/src/lib/common/string.cpp b/src/lib/common/string.cpp index 95bdf39b95..261d9488f4 100644 --- a/src/lib/common/string.cpp +++ b/src/lib/common/string.cpp @@ -974,6 +974,7 @@ std::string double2string(double f) long long intPart = (long long) f; double diff = f - intPart; bool isInteger = false; + bool large = false; // abs value for 'diff' diff = (diff < 0)? -diff : diff; @@ -981,7 +982,7 @@ std::string double2string(double f) if (diff > 1) { // this is an overflow situation, pe. f = 1.79e308 - isInteger = false; + large = true; } else if (diff > 0.9999999998) { @@ -999,8 +1000,14 @@ std::string double2string(double f) } else { - //snprintf(buf, bufSize, "%.9f", f); - snprintf(buf, bufSize, "%.17g", f); + if (large) + { + snprintf(buf, bufSize, "%.17g", f); + } + else + { + snprintf(buf, bufSize, "%.9f", f); + } // Clear out any unwanted trailing zeroes char* last = &buf[strlen(buf) - 1]; diff --git a/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test new file mode 100644 index 0000000000..d17436c0b8 --- /dev/null +++ b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test @@ -0,0 +1,91 @@ +# Copyright 2025 Telefonica Investigacion y Desarrollo, S.A.U +# +# This file is part of Orion Context Broker. +# +# Orion Context Broker is free software: you can redistribute it and/or +# modify it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# Orion Context Broker is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. +# +# For those usages not covered by this license please contact with +# iot_support at tid dot es + +# VALGRIND_READY - to mark the test ready for valgrindTestSuite.sh + +--NAME-- +Large float numbers in attributes + +--SHELL-INIT-- +dbInit CB +brokerStart CB 0 IPV4 -relogAlarms + +--SHELL-- + +# +# 01. Create entity with large float number +# 02. Retrieve entity with large float numbers +# + +echo "01. 01. Create entity with large float number" +echo "=============================================" +payload='{ + "id": "E", + "type": "T", + "A": { + "value": 1.79e308, + "type": "Number" + } +}' +orionCurl --url '/v2/entities' --payload "$payload" +echo +echo + + +echo "02. Retrieve entity with large float numbers" +echo "============================================" +orionCurl --url '/v2/entities/E' +echo +echo + + +--REGEXPECT-- +01. 01. Create entity with large float number +============================================= +HTTP/1.1 201 Created +Date: Fri, 24 Jan 2025 13:07:04 GMT +Fiware-Correlator: 1b79d82c-da54-11ef-9ba0-080027cd35f1 +Location: /v2/entities/E?type=T +Content-Length: 0 + + + +02. Retrieve entity with large float numbers +============================================ +HTTP/1.1 200 OK +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) +Content-Type: application/json +Content-Length: 75 + +{ + "A": { + "metadata": {}, + "type": "Number", + "value": 1.79e+308 + }, + "id": "E", + "type": "T" +} + + +--TEARDOWN-- +brokerStop CB +#dbDrop CB From ac694cd4f90f4c21818df0b48fc67a0c281ea606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 14:20:43 +0100 Subject: [PATCH 5/9] FIX 4642 ftest --- .../large_float_numbers.test | 57 ++++++++++++++++--- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test index d17436c0b8..8d7f7a9f65 100644 --- a/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test +++ b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test @@ -29,19 +29,40 @@ brokerStart CB 0 IPV4 -relogAlarms --SHELL-- +# We are using the the limits included in the IEEE754 standard. However, note that +# min non zero cases are rounded to 0 due to how double2string() function is implemented +# (ten digits precision). We could remove that rounding, but consecuences may not be7 +# backward compatible (eg. places where we have now 1.7 could become in 1.666666666666666666669) +# # # 01. Create entity with large float number # 02. Retrieve entity with large float numbers # -echo "01. 01. Create entity with large float number" -echo "=============================================" +echo "01. Create entity with large float number" +echo "=========================================" payload='{ "id": "E", "type": "T", "A": { "value": 1.79e308, "type": "Number" + }, + "maxPositive": { + "value": 1.7976931348623157e308, + "type": "Number" + }, + "minNonZeroPositive": { + "value": 4.9406564584124654e-324, + "type": "Number" + }, + "maxNegative": { + "value": -1.7976931348623157e308, + "type": "Number" + }, + "minNonZeroNegative": { + "value": -4.9406564584124654e-324, + "type": "Number" } }' orionCurl --url '/v2/entities' --payload "$payload" @@ -57,11 +78,11 @@ echo --REGEXPECT-- -01. 01. Create entity with large float number -============================================= +01. Create entity with large float number +========================================= HTTP/1.1 201 Created -Date: Fri, 24 Jan 2025 13:07:04 GMT -Fiware-Correlator: 1b79d82c-da54-11ef-9ba0-080027cd35f1 +Date: REGEX(.*) +Fiware-Correlator: REGEX([0-9a-f\-]{36}) Location: /v2/entities/E?type=T Content-Length: 0 @@ -73,7 +94,7 @@ HTTP/1.1 200 OK Date: REGEX(.*) Fiware-Correlator: REGEX([0-9a-f\-]{36}) Content-Type: application/json -Content-Length: 75 +Content-Length: 358 { "A": { @@ -82,10 +103,30 @@ Content-Length: 75 "value": 1.79e+308 }, "id": "E", + "maxNegative": { + "metadata": {}, + "type": "Number", + "value": -1.7976931348623157e+308 + }, + "maxPositive": { + "metadata": {}, + "type": "Number", + "value": 1.7976931348623157e+308 + }, + "minNonZeroNegative": { + "metadata": {}, + "type": "Number", + "value": 0 + }, + "minNonZeroPositive": { + "metadata": {}, + "type": "Number", + "value": 0 + }, "type": "T" } --TEARDOWN-- brokerStop CB -#dbDrop CB +dbDrop CB From 8c4b5530c3e8fc8d01d60d16bd9eaba7da615cf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 14:27:04 +0100 Subject: [PATCH 6/9] ADD CNR entry --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 241c815fcb..495e1a2e42 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,4 @@ +- Fix: correct rendering of large float numbers (according IEEE754 standard) (#4642) - Fix: improve attribute and metadata invalid format dates for DateTime types in logs (#4616) - Hardening: upgrade microhttpd dependency from 0.9.76 to 1.0.1 - Hardening: upgrade libmosquitto dependency from 2.0.15 to 2.0.20 From db04827ea51f67ce19e739c4230671497673885c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 16:04:08 +0100 Subject: [PATCH 7/9] FIX cleanup --- src/lib/common/string.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/lib/common/string.cpp b/src/lib/common/string.cpp index 261d9488f4..2657793f9f 100644 --- a/src/lib/common/string.cpp +++ b/src/lib/common/string.cpp @@ -23,7 +23,6 @@ * Author: Ken Zangelin */ -#include // numeric_limits #include #include #include @@ -951,14 +950,6 @@ bool str2double(const char* s, double* dP) return true; } -#if 0 -std::string double2string(double value) -{ - char buffer[32]; - snprintf(buffer, sizeof(buffer), "%.17g", value); - return std::string(buffer); -} -#endif /* **************************************************************************** From 0b74e2af46193c206d97e3fbadd92b46d7372485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 16:05:10 +0100 Subject: [PATCH 8/9] FIX simplify ftest --- .../cases/4642_large_float_numbers/large_float_numbers.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test index 8d7f7a9f65..30aa829f91 100644 --- a/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test +++ b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test @@ -25,7 +25,7 @@ Large float numbers in attributes --SHELL-INIT-- dbInit CB -brokerStart CB 0 IPV4 -relogAlarms +brokerStart CB --SHELL-- From 4c908d158b4ee8338fa2ac3751a866b4f890160d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Fri, 24 Jan 2025 17:02:06 +0100 Subject: [PATCH 9/9] Apply suggestions from code review --- CHANGES_NEXT_RELEASE | 2 +- .../cases/4642_large_float_numbers/large_float_numbers.test | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 495e1a2e42..084629fd8f 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,4 +1,4 @@ -- Fix: correct rendering of large float numbers (according IEEE754 standard) (#4642) +- Fix: rendering of large float numbers (according IEEE754 standard) (#4642) - Fix: improve attribute and metadata invalid format dates for DateTime types in logs (#4616) - Hardening: upgrade microhttpd dependency from 0.9.76 to 1.0.1 - Hardening: upgrade libmosquitto dependency from 2.0.15 to 2.0.20 diff --git a/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test index 30aa829f91..881b5b3661 100644 --- a/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test +++ b/test/functionalTest/cases/4642_large_float_numbers/large_float_numbers.test @@ -31,9 +31,8 @@ brokerStart CB # We are using the the limits included in the IEEE754 standard. However, note that # min non zero cases are rounded to 0 due to how double2string() function is implemented -# (ten digits precision). We could remove that rounding, but consecuences may not be7 +# (ten digits precision). We could remove that rounding, but consequences may not be # backward compatible (eg. places where we have now 1.7 could become in 1.666666666666666666669) -# # # 01. Create entity with large float number # 02. Retrieve entity with large float numbers