update up to 7.3.49 (#138)

CMakeLists.txt

@@ -24,7 +24,7 @@ if(POLICY CMP0135)
 endif()
 
 project(foundationdb
-  VERSION 7.3.43
+  VERSION 7.3.49
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

bindings/go/src/fdb/tenant.go

@@ -22,7 +22,7 @@
 
 package fdb
 
-// #define FDB_API_VERSION 710
+// #define FDB_API_VERSION 730
 // #include <foundationdb/fdb_c.h>
 // #include <stdlib.h>
 import "C"

cmake/ConfigureCompiler.cmake

@@ -40,6 +40,7 @@ if(USE_SANITIZER OR WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") OR APPLE)
   set(jemalloc_default OFF)
 endif()
 env_set(USE_JEMALLOC ${jemalloc_default} BOOL "Link with jemalloc")
+env_set(USE_CUSTOM_JEMALLOC OFF BOOL "Manually download and build jemalloc")
 
 if(USE_LIBCXX AND STATIC_LINK_LIBCXX AND NOT USE_LD STREQUAL "LLD")
   message(FATAL_ERROR "Unsupported configuration: STATIC_LINK_LIBCXX with libc++ only works if USE_LD=LLD")

cmake/FDBComponents.cmake

@@ -5,7 +5,11 @@ set(FORCE_ALL_COMPONENTS OFF CACHE BOOL "Fails cmake if not all dependencies are
 ################################################################################
 
 if(USE_JEMALLOC)
-  find_package(jemalloc 5.3.0 REQUIRED)
+  if(NOT USE_CUSTOM_JEMALLOC)
+    find_package(jemalloc 5.3.0 REQUIRED)
+  else()
+    include(Jemalloc)
+  endif()
 endif()
 
 ################################################################################

cmake/InstallLayout.cmake

@@ -260,7 +260,7 @@ set(CPACK_RPM_PACKAGE_NAME                                 "${package_name}")
 set(CPACK_RPM_COMPRESSION_TYPE                             "gzip")
 set(CPACK_RPM_CLIENTS-EL7_PACKAGE_NAME                     "${clients_package_name}")
 set(CPACK_RPM_CLIENTS-EL7_FILE_NAME                        "${CPACK_RPM_CLIENTS-EL7_PACKAGE_NAME}-${rpm_filename_suffix}")
-set(CPACK_RPM_CLIENTS-EL7_DEBUGINFO_FILE_NAME              "${CPACK_RPM_CLIENTS-EL7_PACKAGE_NAME}-debuginfo--${rpm_filename_suffix}")
+set(CPACK_RPM_CLIENTS-EL7_DEBUGINFO_FILE_NAME              "${CPACK_RPM_CLIENTS-EL7_PACKAGE_NAME}-debuginfo-${rpm_filename_suffix}")
 set(CPACK_RPM_CLIENTS-EL7_PRE_INSTALL_SCRIPT_FILE          ${CMAKE_SOURCE_DIR}/packaging/rpm/scripts/preclients.sh)
 set(CPACK_RPM_CLIENTS-EL7_POST_INSTALL_SCRIPT_FILE         ${CMAKE_SOURCE_DIR}/packaging/rpm/scripts/postclients.sh)
 set(CPACK_RPM_CLIENTS-EL7_USER_FILELIST                    "%dir /etc/foundationdb")

cmake/Jemalloc.cmake

@@ -4,28 +4,25 @@ if(NOT USE_JEMALLOC)
   return()
 endif()
 
-add_library(im_jemalloc_pic STATIC IMPORTED)
-add_library(im_jemalloc STATIC IMPORTED)
-if(EXISTS /opt/jemalloc_5.3.0)
-  set(JEMALLOC_DIR /opt/jemalloc_5.3.0)
-else()
-  include(ExternalProject)
-  set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc")
-  ExternalProject_add(Jemalloc_project
-    URL "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
-    URL_HASH SHA256=2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa
-    BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
-    "${JEMALLOC_DIR}/lib/libjemalloc.a"
-    "${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
-    CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof --disable-initial-exec-tls
-    BUILD_IN_SOURCE ON
-    BUILD_COMMAND make
-    INSTALL_DIR "${JEMALLOC_DIR}"
-    INSTALL_COMMAND make install)
-endif()
-add_dependencies(im_jemalloc Jemalloc_project)
-add_dependencies(im_jemalloc_pic Jemalloc_project)
-set_target_properties(im_jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a")
-set_target_properties(im_jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a")
-target_include_directories(jemalloc INTERFACE "${JEMALLOC_DIR}/include")
-target_link_libraries(jemalloc INTERFACE im_jemalloc_pic im_jemalloc)
+add_library(jemalloc::jemalloc STATIC IMPORTED)
+add_library(jemalloc_pic::jemalloc_pic STATIC IMPORTED)
+
+include(ExternalProject)
+set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/jemalloc")
+ExternalProject_add(Jemalloc_project
+  URL "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
+  URL_HASH SHA256=2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa
+  BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
+  "${JEMALLOC_DIR}/lib/libjemalloc.a"
+  "${JEMALLOC_DIR}/lib/libjemalloc_pic.a"
+  CONFIGURE_COMMAND CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} ./configure --prefix=${JEMALLOC_DIR} --enable-static --disable-cxx --enable-prof
+  BUILD_IN_SOURCE ON
+  BUILD_COMMAND make
+  INSTALL_DIR "${JEMALLOC_DIR}"
+  INSTALL_COMMAND make install)
+
+add_dependencies(jemalloc::jemalloc Jemalloc_project)
+add_dependencies(jemalloc_pic::jemalloc_pic Jemalloc_project)
+
+set_target_properties(jemalloc::jemalloc PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.a")
+set_target_properties(jemalloc_pic::jemalloc_pic PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc_pic.a")

documentation/sphinx/source/consistency-check-urgent.rst

@@ -37,7 +37,7 @@ How to use?
 To run the ConsistencyCheckerUrgent, you need 1 checker and N testers. The process is as follows:
 
 * If you want to specify ranges to do consistency checker, set ranges via knobs: CONSISTENCY_CHECK_URGENT_RANGE_BEGIN_* and CONSISTENCY_CHECK_URGENT_RANGE_END_*. The custom range's start and end points must be represented in hexadecimal ASCII format, strictly adhering to the "\\\\x" prefix (including two escape chars, followed by 'x'). By default, the knob is set to check the entire key space (i.e., " " ~ "\\xff\\xff").
-* Start N testers (i.e. fdbserver --class test).
+* Start N testers (i.e. fdbserver --class test --consistency-check-urgent-mode). When setting consistency-check-urgent-mode, the tester server will check whether the received workload is a "ConsistencyCheckUrgent" workload. If yes, accept the workload. Otherwise, drop the workload with a SevError trace event: StartingTesterServerCoreUnexpectedWorkload.
 * Start the checker (i.e. fdbserver -r consistencycheckurgent --num-testers={num-testers}), which initiates the consistency checking automatically.
 * Once the checking is complete, the checker exits automatically, leaving the testers alive but idle.
 * If you need to rerun the checking, simply restart the checker process.

documentation/sphinx/source/release-notes/release-notes-710.rst

@@ -15,7 +15,7 @@ Release Notes
 * Increase visibility of CommitProxyTerminated events for failed_to_progress errors. `(PR #11316) <https://github.com/apple/foundationdb/pull/11316>`_
 * Fixed an infinite retry of GRV request bug. `(PR #11352) <https://github.com/apple/foundationdb/pull/11352>`_
 * Improved distributed consistency checker to continuously run by default and visibility of recruitment errors. `(PR #11351) <https://github.com/apple/foundationdb/pull/11351>`_
-* Fix an assertion failure of cluster controller when waiting for recovery. `(PR #11398) <https://github.com/apple/foundationdb/pull/11398>`_
+* Fixed an assertion failure of cluster controller when waiting for recovery. `(PR #11398) <https://github.com/apple/foundationdb/pull/11398>`_
 
 7.1.59
 ======

documentation/sphinx/source/release-notes/release-notes-730.rst

@@ -4,6 +4,48 @@
 Release Notes
 #############
 
+7.3.43
+======
+* Same as 7.3.42 release with AVX enabled.
+
+7.3.42
+======
+* Fixed a segfault when tlog encounters a platform_error. `(PR #11406) <https://github.com/apple/foundationdb/pull/11406>`_
+* Fixed an assertion failure of cluster controller when waiting for recovery. `(PR #11402) <https://github.com/apple/foundationdb/pull/11402>`_
+* Fixed a global config bug that causes client latency metrics to be missing after upgrade. `(PR #11400) <https://github.com/apple/foundationdb/pull/11400>`_
+* Added a knob for shard merge parallelism and added logs for max shard size. `(PR #11389) <https://github.com/apple/foundationdb/pull/11389>`_
+* Updated the RocksDB memtable max range deletions knob. `(PR #11387) <https://github.com/apple/foundationdb/pull/11387>`_
+* Improved visibility when Sharded RocksDB takes a long time to commit. `(PR #11362) <https://github.com/apple/foundationdb/pull/11362>`_
+* Fixed data move trigger for rebalancing storage queue. `(PR #11375) <https://github.com/apple/foundationdb/pull/11375>`_
+* Fixed an infinite retry of GRV request bug. `(PR #11353) <https://github.com/apple/foundationdb/pull/11353>`_
+* Improved distributed consistency checker to continuously run by default and visibility of recruitment errors. `(PR #11349) <https://github.com/apple/foundationdb/pull/11349>`_
+* Fixed add-prefix and remove-prefix for fdbrestore. `(PR #11344) <https://github.com/apple/foundationdb/pull/11344>`_
+* Fixed a crash of data distributor when taking a snapshot. `(PR #11341) <https://github.com/apple/foundationdb/pull/11341>`_
+* Increased visibility of gray failure actions. `(PR #11324) <https://github.com/apple/foundationdb/pull/11324>`_
+* Increased visibility of CommitProxyTerminated events for failed_to_progress errors. `(PR #11315) <https://github.com/apple/foundationdb/pull/11315>`_
+
+7.3.41
+======
+* Same as 7.3.40 release with AVX enabled.
+
+7.3.40
+======
+* Upgraded RocksDB to version 8.11.4. `(PR #11327) <https://github.com/apple/foundationdb/pull/11327>`_
+
+7.3.39
+======
+* Same as 7.3.38 release with AVX enabled.
+
+7.3.38
+======
+* Fixed the detection of private mutations in version vector. `(PR #11279) <https://github.com/apple/foundationdb/pull/11279>`_
+* Added accumulative checksum feature. `(PR #11281) <https://github.com/apple/foundationdb/pull/11281>`_ and `(PR #11289) <https://github.com/apple/foundationdb/pull/11289>`_
+* Added Go tenanting support. `(PR #11299) <https://github.com/apple/foundationdb/pull/11299>`_
+* Added RocksDB caching knobs. `(PR #11312) <https://github.com/apple/foundationdb/pull/11312>`_
+* Added RocksDB metrics in status json. `(PR #11320) <https://github.com/apple/foundationdb/pull/11320>`_
+* Various Sharded RocksDB improvements. `(PR #11332) <https://github.com/apple/foundationdb/pull/11332>`_
+
+
 7.3.37
 ======
 * Same as 7.3.36 release with AVX enabled.
@@ -84,9 +126,30 @@ Release Notes
 7.3.24
 ======
 * Released with AVX disabled.
-* Added support for large shard. `(PR#10965) <https://github.com/apple/foundationdb/pull/10965>`_
-* Fixed perpetual wiggle locality match regex. `(PR#10973) <https://github.com/apple/foundationdb/pull/10973>`_
-* Added a knob to throttle perpetual wiggle data move. `(PR#10957) <https://github.com/apple/foundationdb/pull/10957>`_
+* Added support for large shard. `(PR #10965) <https://github.com/apple/foundationdb/pull/10965>`_
+* Fixed perpetual wiggle locality match regex. `(PR #10973) <https://github.com/apple/foundationdb/pull/10973>`_
+* Added a knob to throttle perpetual wiggle data move. `(PR #10957) <https://github.com/apple/foundationdb/pull/10957>`_
+
+7.3.23
+======
+* Same as 7.3.22 release with AVX enabled.
+
+7.3.22
+======
+* No code change, only version bumped.
+
+7.3.21
+======
+* Same as 7.3.20 release with AVX enabled.
+
+7.3.20
+======
+* Added data move throttling for perpetual wiggle. `(PR #10957) <https://github.com/apple/foundationdb/pull/10957>`_
+* Fixed AuditStorage to check all DC replicas. `(PR #10966) <https://github.com/apple/foundationdb/pull/10966>`_
+* Added large shards support. `(PR #10965) <https://github.com/apple/foundationdb/pull/10965>`_
+* Fixed bugs for locality-based exclusion. `(PR #10946) <https://github.com/apple/foundationdb/pull/10946>`_
+* Fixed various memory-related bugs. `(PR #10952) <https://github.com/apple/foundationdb/pull/10952>`_ and `(PR #10969) <https://github.com/apple/foundationdb/pull/10969>`_
+* Fixed perpetual wiggling locality match regex. `(PR #10972) <https://github.com/apple/foundationdb/pull/10972>`_
 
 7.3.19
 ======

fdbcli/DebugCommands.actor.cpp

@@ -166,6 +166,24 @@ ACTOR Future<bool> getallCommandActor(Database cx, std::vector<StringRef> tokens
 // hidden commands, no help text for now
 CommandFactory getallCommandFactory("getall");
 
+std::string printStorageServerMachineInfo(const StorageServerInterface& server) {
+	std::string serverIp = server.address().toString();
+	std::string serverLocality = server.locality.toString();
+	return serverLocality + " " + serverIp;
+}
+
+std::string printAllStorageServerMachineInfo(const std::vector<StorageServerInterface>& servers) {
+	std::string res;
+	for (int i = 0; i < servers.size(); i++) {
+		if (i == 0) {
+			res = printStorageServerMachineInfo(servers[i]);
+		} else {
+			res = res + "; " + printStorageServerMachineInfo(servers[i]);
+		}
+	}
+	return res;
+}
+
 // check that all replies are the same. Update begin to the next key to check
 // checkResults keeps invariants:
 // (1) hasMore = true if any server has more data not read yet
@@ -184,6 +202,11 @@ bool checkResults(Version version,
 	for (int j = 0; j < replies.size(); j++) {
 		if (firstValidServer == -1) {
 			firstValidServer = j;
+			// Print full list of comparing servers and the reference server
+			// Used to check server info which does not produce an inconsistency log
+			printf("CheckResult: servers: %s, reference server: %s\n",
+			       printAllStorageServerMachineInfo(servers).c_str(),
+			       printStorageServerMachineInfo(servers[firstValidServer]).c_str());
 			continue; // always select the first server as reference
 		}
 		// compare reference and current
@@ -207,8 +230,8 @@ bool checkResults(Version version,
 				// have the key
 				printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key "
 				       "%s\n",
-				       servers[firstValidServer].address().toString().c_str(),
-				       servers[j].address().toString().c_str(),
+				       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
+				       printStorageServerMachineInfo(servers[j]).c_str(),
 				       currentI,
 				       referenceI,
 				       version,
@@ -217,8 +240,8 @@ bool checkResults(Version version,
 			} else if (referenceI >= reference.data.size()) {
 				printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, Key "
 				       "%s\n",
-				       servers[j].address().toString().c_str(),
-				       servers[firstValidServer].address().toString().c_str(),
+				       printStorageServerMachineInfo(servers[j]).c_str(),
+				       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
 				       currentI,
 				       referenceI,
 				       version,
@@ -232,8 +255,8 @@ bool checkResults(Version version,
 						printf("Inconsistency: MismatchValue, %s(1), %s(1), CurrentIndex %lu, ReferenceIndex %lu, "
 						       "Version %ld, "
 						       "Key %s\n",
-						       servers[firstValidServer].address().toString().c_str(),
-						       servers[j].address().toString().c_str(),
+						       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
+						       printStorageServerMachineInfo(servers[j]).c_str(),
 						       currentI,
 						       referenceI,
 						       version,
@@ -244,8 +267,8 @@ bool checkResults(Version version,
 				} else if (currentKV.key < referenceKV.key) {
 					printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
 					       "Key %s\n",
-					       servers[j].address().toString().c_str(),
-					       servers[firstValidServer].address().toString().c_str(),
+					       printStorageServerMachineInfo(servers[j]).c_str(),
+					       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
 					       currentI,
 					       referenceI,
 					       version,
@@ -254,8 +277,8 @@ bool checkResults(Version version,
 				} else {
 					printf("Inconsistency: UniqueKey, %s(1), %s(0), CurrentIndex %lu, ReferenceIndex %lu, Version %ld, "
 					       "Key %s\n",
-					       servers[firstValidServer].address().toString().c_str(),
-					       servers[j].address().toString().c_str(),
+					       printStorageServerMachineInfo(servers[firstValidServer]).c_str(),
+					       printStorageServerMachineInfo(servers[j]).c_str(),
 					       currentI,
 					       referenceI,
 					       version,
@@ -265,6 +288,7 @@ bool checkResults(Version version,
 			}
 		}
 	}
+
 	return allSame;
 }
 

fdbcli/ExcludeCommand.actor.cpp

@@ -42,6 +42,7 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 	state Reference<ITransaction> tr = db->createTransaction();
 	loop {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		try {
 			if (force && servers.size())
 				tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey,
@@ -78,6 +79,7 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 				            : "Type `exclude FORCE failed <ADDRESS...>' to exclude without performing safety checks.");
 				return false;
 			}
+			TraceEvent(SevWarn, "ExcludeServersAndLocalitiesError").error(err);
 			wait(safeThreadFutureToFuture(tr->onError(err)));
 		}
 	}
@@ -99,6 +101,7 @@ ACTOR Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> d
 			}
 			return exclusions;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetExcludedServersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -164,6 +167,7 @@ ACTOR Future<std::vector<std::string>> getFailedLocalities(Reference<IDatabase>
 			}
 			return excludedLocalities;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -213,6 +217,7 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatab
 
 			wait(delayJittered(1.0)); // SOMEDAY: watches!
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "CheckForExcludingServersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -233,6 +238,7 @@ ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<Addres
 			coordinatorList = NetworkAddress::parseList(coordinators.get().toString());
 			break;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "CheckForCoordinatorsError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}

fdbcli/Util.actor.cpp

@@ -148,6 +148,7 @@ ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>*
 
 			return true;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetWorkersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -174,6 +175,7 @@ ACTOR Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
 			}
 			return Void();
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetStorageServerInterfacesError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}