#2509 [YSQL] Analyze core dump of PostgreSQL processes in tests

Summary:
Our test environment already analyzes core dumps of most processes such
as yb-master and yb-tserver, and prints their symbolized stack traces
into the log.  To analyze a core dump of any child process, the test
environment needs to know the child process id and the full path to its
executable file, and we already have this information for masters and
tablet servers.

But sometimes a PostgreSQL process can also crash with a core dump.  In
this diff, we are adding a new test flag, process_info_dir, and when it
is specified, each PostgreSQL process (either the postmaster or a
backend) will create a file named as its pid in this directory, and the
file would contain its executable path. This allows us to properly
symbolize core file stack traces for PostgreSQL processes.

Test Plan: Jenkins

Reviewers: mikhail

Reviewed By: mikhail

Subscribers: yql

Differential Revision: https://phabricator.dev.yugabyte.com/D7495

java/yb-client/src/test/java/org/yb/minicluster/MiniYBCluster.java

@@ -32,6 +32,7 @@
 import com.google.common.collect.Lists;
 import com.google.common.net.HostAndPort;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.math.NumberUtils;
 import org.yb.AssertionWrappers;
 import org.yb.client.BaseYBClientTest;
 import org.yb.client.TestUtils;
@@ -51,6 +52,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListSet;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.yb.AssertionWrappers.assertTrue;
@@ -553,7 +555,8 @@ public void startTServer(List<String> tserverArgs, String tserverBindAddress,
         "--cql_proxy_webserver_port=" + cqlWebPort,
         "--pgsql_proxy_webserver_port=" + pgsqlWebPort,
         "--yb_client_admin_operation_timeout_sec=" + YB_CLIENT_ADMIN_OPERATION_TIMEOUT_SEC,
-        "--callhome_enabled=false");
+        "--callhome_enabled=false",
+        "--process_info_dir=" + getProcessInfoDir());
     addFlagsFromEnv(tsCmdLine, "YB_EXTRA_TSERVER_FLAGS");
 
     if (startPgSqlProxy) {
@@ -606,7 +609,8 @@ private List<String> getCommonMasterCmdLine(String flagsPath, String dataDirPath
       "--catalog_manager_bg_task_wait_ms=" + CATALOG_MANAGER_BG_TASK_WAIT_MS,
       "--rpc_slow_query_threshold_ms=" + RPC_SLOW_QUERY_THRESHOLD,
       "--webserver_port=" + masterWebPort,
-      "--callhome_enabled=false");
+      "--callhome_enabled=false",
+      "--process_info_dir=" + getProcessInfoDir());
     addFlagsFromEnv(masterCmdLine, "YB_EXTRA_MASTER_FLAGS");
     return masterCmdLine;
   }
@@ -947,6 +951,9 @@ private List<Process> destroyDaemons(Collection<MiniYBDaemon> daemons) throws Ex
   public void shutdown() throws Exception {
     LOG.info("Shutting down mini cluster");
     shutdownDaemons();
+    String processInfoDir = getProcessInfoDir();
+    processCoreFiles(processInfoDir);
+    pathsToDelete.add(processInfoDir);
     for (String path : pathsToDelete) {
       try {
         File f = new File(path);
@@ -963,6 +970,28 @@ public void shutdown() throws Exception {
     LOG.info("Mini cluster shutdown finished");
   }
 
+  private void processCoreFiles(String folder) {
+    File[] files = (new File(folder)).listFiles();
+    for (File file : files == null ? new File[]{} : files) {
+      String fileName = file.getAbsolutePath();
+      try {
+        String exeFile = new String(Files.readAllBytes(Paths.get(fileName)));
+        int pid = Integer.parseInt(file.getName());
+        CoreFileUtil.processCoreFile(
+            pid, exeFile, exeFile, null /* coreFileDir */,
+            CoreFileUtil.CoreFileMatchMode.EXACT_PID);
+      } catch (Exception e) {
+        LOG.warn("Failed to analyze PID from '{}' file", fileName, e);
+      }
+    }
+  }
+
+  private String getProcessInfoDir() {
+    Path path = Paths.get(TestUtils.getBaseTmpDir()).resolve("process_info");
+    path.toFile().mkdirs();
+    return path.toAbsolutePath().toString();
+  }
+
   private void shutdownDaemons() throws Exception {
     List<Process> processes = new ArrayList<>();
     List<MiniYBDaemon> allDaemons = new ArrayList<>();

src/yb/common/ybc_util.cc

@@ -13,22 +13,30 @@
 #include "yb/common/ybc_util.h"
 
 #include <stdarg.h>
+#include <fstream>
 
 #include "yb/common/pgsql_error.h"
 #include "yb/common/pgsql_protocol.pb.h"
 #include "yb/common/transaction_error.h"
 #include "yb/common/ybc-internal.h"
 
-#include "yb/util/logging.h"
-#include "yb/util/init.h"
-#include "yb/util/version_info.h"
-#include "yb/util/status.h"
-#include "yb/util/debug-util.h"
 #include "yb/util/bytes_formatter.h"
+#include "yb/util/debug-util.h"
+#include "yb/util/env.h"
+#include "yb/util/flag_tags.h"
+#include "yb/util/init.h"
+#include "yb/util/logging.h"
 #include "yb/util/scope_exit.h"
+#include "yb/util/status.h"
+#include "yb/util/version_info.h"
+
 #include "yb/gutil/stringprintf.h"
 
 using std::string;
+DEFINE_test_flag(string,
+                 process_info_dir,
+                 string(),
+                 "Directory where all postgres process will writes their PIDs and executable name");
 
 namespace yb {
 
@@ -42,6 +50,21 @@ void ChangeWorkingDir(const char* dir) {
   }
 }
 
+void WriteCurrentProcessInfo(const string& destination_dir) {
+  string executable_path;
+  if (Env::Default()->GetExecutablePath(&executable_path).ok()) {
+    const auto destination_file = Format("$0/$1", destination_dir, getpid());
+    std::ofstream out(destination_file, std::ios_base::out);
+    out << executable_path;
+    if (out) {
+      LOG(INFO) << "Process info is written to " << destination_file;
+      return;
+    }
+  }
+  LOG(WARNING) << "Unable to write process info to "
+               << destination_dir << " dir: error " << errno << " " << std::strerror(errno);
+}
+
 Status InitInternal(const char* argv0) {
   // Change current working directory from postgres data dir (as set by postmaster)
   // to the one from yb-tserver so that relative paths in gflags would be resolved in the same way.
@@ -201,7 +224,11 @@ YBCStatus YBCInit(const char* argv0,
                   YBCCStringToTextWithLenFn cstring_to_text_with_len_fn) {
   YBCSetPAllocFn(palloc_fn);
   YBCSetCStringToTextWithLenFn(cstring_to_text_with_len_fn);
-  return ToYBCStatus(yb::InitInternal(argv0));
+  auto status = yb::InitInternal(argv0);
+  if (status.ok() && !FLAGS_process_info_dir.empty()) {
+    WriteCurrentProcessInfo(FLAGS_process_info_dir);
+  }
+  return ToYBCStatus(status);
 }
 
 void YBCLogImpl(