[YSQL] #5459 #6088 #2415 Include ldap and uuid libs in default builds

Summary:
Include support in all default builds for:

- LDAP Authentication (by including ldap lib in the build)
- UUID (by including uuid-ossp extension with e2fs implementation)

Additional change:
Add `ysql_pg_conf_csv`/`ysql_hba_conf_csv` flags as substitutions for deprecated `ysql_pg_conf` and `ysql_hba_conf`.
Example how to start the cluster with LDAP authentication

```
./bin/yb-ctl start --ysql_hba_conf_csv='host all yugabyte 127.0.0.1/0 password,"host   all         all      0.0.0.0/0  ldap ldapserver=ldap.forumsys.com ldapprefix=""uid="" ldapsuffix="", dc=example, dc=com"" ldapport=389"'
```

Test Plan:
Jenkins:
Existing tests, additional tests in:

TestPgRegressExtension (yb_extensions),

Reviewers: sanketh, mihnea

Reviewed By: mihnea

Subscribers: mikhail, rskannan, yql

Differential Revision: https://phabricator.dev.yugabyte.com/D9692

build-support/thirdparty_url_centos.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20200829090443-f431681041/yugabyte-db-thirdparty-v20200829090443-f431681041-centos.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20201023135146-67b1057678/yugabyte-db-thirdparty-v20201023135146-67b1057678-centos.tar.gz

build-support/thirdparty_url_mac.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20200829090320-f431681041/yugabyte-db-thirdparty-v20200829090320-f431681041-macos.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20201023023256-67b1057678/yugabyte-db-thirdparty-v20201023023256-67b1057678-macos.tar.gz

build-support/thirdparty_url_ubuntu.txt

@@ -1 +1 @@
-https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20200829090458-f431681041/yugabyte-db-thirdparty-v20200829090458-f431681041-ubuntu.tar.gz
+https://github.com/yugabyte/yugabyte-db-thirdparty/releases/download/v20201023162109-67b1057678/yugabyte-db-thirdparty-v20201023162109-67b1057678-ubuntu.tar.gz

python/yb/build_postgres.py

@@ -397,7 +397,11 @@ def configure_postgres(self):
                 '--prefix', self.pg_prefix,
                 '--with-extra-version=-YB-' + self.get_yb_version(),
                 '--enable-depend',
+                '--with-ldap',
                 '--with-openssl',
+                # Options are ossp (original/old implementation), bsd (BSD) and e2fs
+                # (libuuid-based for Unix/Mac).
+                '--with-uuid=e2fs',
                 '--with-libedit-preferred',
                 '--with-includes=' + self.openssl_include_dir,
                 '--with-libraries=' + self.openssl_lib_dir,

python/yb/library_packager.py

@@ -350,6 +350,16 @@ def package_binaries(self):
         linuxbrew_dest_dir = os.path.join(self.dest_dir, 'linuxbrew')
         linuxbrew_lib_dest_dir = os.path.join(linuxbrew_dest_dir, 'lib')
 
+        # Add libresolv and libnss_* libs explicitly because they are loaded by glibc at runtime.
+        additional_libs = set()
+        for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
+            additional_libs.update(lib_path for lib_path in
+                                   glob.glob(os.path.join(linuxbrew_home.cellar_glibc_dir,
+                                                          '*',
+                                                          'lib',
+                                                          additional_lib_name_glob))
+                                   if not lib_path.endswith('.a'))
+
         for category, deps_in_category in sorted_grouped_by(all_deps,
                                                             lambda dep: dep.get_category()):
             logging.info("Found {} dependencies in category '{}':".format(
@@ -367,30 +377,26 @@ def package_binaries(self):
             mkdir_p(category_dest_dir)
 
             for dep in deps_in_category:
+                additional_libs.discard(dep.target)
                 self.install_dyn_linked_binary(dep.target, category_dest_dir)
-                if os.path.basename(dep.target) != dep.name:
-                    symlink(os.path.basename(dep.target),
-                            os.path.join(category_dest_dir, dep.name))
-
-        # Add libresolv and libnss_* libraries explicitly because they are loaded by glibc at
-        # runtime and will not be discovered automatically using ldd.
-        for additional_lib_name_glob in ADDITIONAL_LIB_NAME_GLOBS:
-            for lib_path in glob.glob(os.path.join(linuxbrew_home.cellar_glibc_dir, '*', 'lib',
-                                                   additional_lib_name_glob)):
-                lib_basename = os.path.basename(lib_path)
-                if lib_basename.endswith('.a'):
-                    continue
-                if os.path.isfile(lib_path):
-                    self.install_dyn_linked_binary(lib_path, linuxbrew_lib_dest_dir)
-                    logging.info("Installed additional lib: " + lib_path)
-                elif os.path.islink(lib_path):
-                    link_target_basename = os.path.basename(os.readlink(lib_path))
-                    logging.info("Installed additional symlink: " + lib_path)
-                    symlink(link_target_basename,
-                            os.path.join(linuxbrew_lib_dest_dir, lib_basename))
-                else:
-                    raise RuntimeError(
-                        "Expected '{}' to be a file or a symlink".format(lib_path))
+                target_name = os.path.basename(dep.target)
+                if target_name != dep.name:
+                    target_src = os.path.join(os.path.dirname(dep.target), dep.name)
+                    additional_libs.discard(target_src)
+                    symlink(target_name, os.path.join(category_dest_dir, dep.name))
+
+        for lib_path in additional_libs:
+            if os.path.isfile(lib_path):
+                self.install_dyn_linked_binary(lib_path, linuxbrew_lib_dest_dir)
+                logging.info("Installed additional lib: " + lib_path)
+            elif os.path.islink(lib_path):
+                link_target_basename = os.path.basename(os.readlink(lib_path))
+                logging.info("Installed additional symlink: " + lib_path)
+                symlink(link_target_basename,
+                        os.path.join(linuxbrew_lib_dest_dir, lib_basename))
+            else:
+                raise RuntimeError(
+                    "Expected '{}' to be a file or a symlink".format(lib_path))
 
         for installed_binary in self.installed_dyn_linked_binaries:
             # Sometimes files that we copy from other locations are not even writable by user!

scripts/installation/bin/yb-ctl

@@ -569,6 +569,8 @@ class ClusterOptions:
         self.placement_info = []
         self.verbose_level = 0
         self.use_cassandra_authentication = False
+        self.ysql_hba_conf_csv = None
+        self.ysql_pg_conf_csv = None
         self.node_type = DAEMON_TYPE_TSERVER
         self.is_shell_master = False
         self.is_startup_command = False
@@ -645,6 +647,12 @@ class ClusterOptions:
         if hasattr(args, "use_cassandra_authentication"):
             self.use_cassandra_authentication = args.use_cassandra_authentication
 
+        if hasattr(args, "ysql_hba_conf_csv"):
+            self.ysql_hba_conf_csv = args.ysql_hba_conf_csv
+
+        if hasattr(args, "ysql_pg_conf_csv"):
+            self.ysql_pg_conf_csv = args.ysql_pg_conf_csv
+
         if hasattr(args, "ip_start"):
             self.ip_start = args.ip_start
 
@@ -1105,6 +1113,12 @@ class ClusterControl:
                                          help="Disable YugaByte SQL API.",
                                          default=None)
 
+            subparsers[cmd].add_argument("--ysql_hba_conf_csv",
+                                         help="Set HBA configuration for YSQL API in CSV format")
+
+            subparsers[cmd].add_argument("--ysql_pg_conf_csv",
+                                         help="Set configuration for YSQL API in CSV format")
+
             subparsers[cmd].add_argument(
                     "--ysql_port",
                     help="YSQL (PostgreSQL-compatible) API port. Default: %d." % YSQL_DEFAULT_PORT,
@@ -1332,6 +1346,12 @@ class ClusterControl:
                 "--pgsql_proxy_bind_address=" +
                 self.options.get_client_listen_host_port(daemon_id, 'ysql')
             ]
+            if (self.options.ysql_hba_conf_csv is not None):
+                command_list.append(
+                    "--ysql_hba_conf_csv='{}'".format(self.options.ysql_hba_conf_csv))
+            if (self.options.ysql_pg_conf_csv is not None):
+                command_list.append(
+                    "--ysql_pg_conf_csv='{}'".format(self.options.ysql_pg_conf_csv))
         else:
             command_list += ["--enable_ysql=false"]
         return self.customize_flags(command_list, self.options.tserver_flags)

src/postgres/src/test/regress/expected/yb_extensions.out

@@ -84,3 +84,17 @@ select userid,dbid,queryid,query,calls,rows,shared_blks_hit,shared_blks_read,sha
 (4 rows)
 
 drop table test;
+-- Testing uuid-ossp
+create extension "uuid-ossp";
+select uuid_generate_v5('00000000-0000-0000-0000-000000000000', 'yugabyte');
+           uuid_generate_v5
+--------------------------------------
+ dc72413f-8ae5-51c0-8aa2-c7d117077c5e
+(1 row)
+
+-- generated values are random, so to ensure deterministic output ignore the actual values.
+select uuid_generate_v1() != uuid_generate_v4();
+ ?column?
+----------
+ t
+(1 row)

src/postgres/src/test/regress/sql/yb_extensions.sql

@@ -37,3 +37,11 @@ select userid,dbid,queryid,query,calls,rows,shared_blks_hit,shared_blks_read,sha
 insert into test(a,b) values (15,20);
 select userid,dbid,queryid,query,calls,rows,shared_blks_hit,shared_blks_read,shared_blks_dirtied,shared_blks_written,local_blks_hit,local_blks_read,local_blks_dirtied,local_blks_written,temp_blks_read,temp_blks_written,blk_read_time from pg_stat_statements;
 drop table test;
+
+-- Testing uuid-ossp
+create extension "uuid-ossp";
+
+select uuid_generate_v5('00000000-0000-0000-0000-000000000000', 'yugabyte');
+
+-- generated values are random, so to ensure deterministic output ignore the actual values.
+select uuid_generate_v1() != uuid_generate_v4();

src/yb/yql/pgwrapper/pg_wrapper.cc

@@ -18,6 +18,7 @@
 #include <string>
 #include <random>
 #include <fstream>
+#include <regex>
 
 #include <gflags/gflags.h>
 #include <boost/algorithm/string.hpp>
@@ -68,9 +69,16 @@ DEFINE_string(ysql_log_min_duration_statement, "",
 
 
 // Catch-all postgres configuration flags.
+DEFINE_string(ysql_pg_conf_csv, "",
+              "CSV formatted line represented list of postgres setting assignments");
+DEFINE_string(ysql_hba_conf_csv, "",
+              "CSV formatted line represented list of postgres hba rules (in order)");
+
 DEFINE_string(ysql_pg_conf, "",
+              "Deprecated, use the `ysql_pg_conf_csv` flag instead. " \
               "Comma separated list of postgres setting assignments");
 DEFINE_string(ysql_hba_conf, "",
+              "Deprecated, use `ysql_hba_conf_csv` flag instead. " \
               "Comma separated list of postgres hba rules (in order)");
 
 using std::vector;
@@ -105,10 +113,44 @@ Status WriteConfigFile(const string& path, const vector<string>& lines) {
   return Status::OK();
 }
 
-void ReadCSVConfigValues(const string& csv, vector<string>* lines) {
-  vector<string> csv_lines;
-  boost::split(csv_lines, csv, boost::is_any_of(","));
-  lines->insert(lines->end(), csv_lines.begin(), csv_lines.end());
+void ReadCommaSeparatedValues(const string& src, vector<string>* lines) {
+  vector<string> new_lines;
+  boost::split(new_lines, src, boost::is_any_of(","));
+  lines->insert(lines->end(), new_lines.begin(), new_lines.end());
+}
+
+CHECKED_STATUS ReadCSVValues(const string& csv, vector<string>* lines) {
+  // Function reads CSV string in the following format:
+  // - fields are divided with comma (,)
+  // - fields with comma (,) or double-quote (") are quoted with double-quote (")
+  // - pair of double-quote ("") in quoted field represents single double-quote (")
+  //
+  // Examples:
+  // 1,"two, 2","three ""3""", four , -> ['1', 'two, 2', 'three "3"', ' four ', '']
+  // 1,"two                           -> Malformed CSV (quoted field 'two' is not closed)
+  // 1, "two"                         -> Malformed CSV (quoted field 'two' has leading spaces)
+  // 1,two "2"                        -> Malformed CSV (field with " must be quoted)
+  // 1,"tw"o"                         -> Malformed CSV (no separator after quoted field 'tw')
+
+  const std::regex exp(R"(^(?:([^,"]+)|(?:"((?:[^"]|(?:""))*)\"))(?:(?:,)|(?:$)))");
+  auto i = csv.begin();
+  const auto end = csv.end();
+  std::smatch match;
+  while (i != end && std::regex_search(i, end, match, exp)) {
+    // Replace pair of double-quote ("") with single double-quote (") in quoted field.
+    if (match[2].length() > 0) {
+      lines->emplace_back(match[2].first, match[2].second);
+      boost::algorithm::replace_all(lines->back(), "\"\"", "\"");
+    } else {
+      lines->emplace_back(match[1].first, match[1].second);
+    }
+    i += match.length();
+  }
+  SCHECK(i == end, InvalidArgument, Format("Malformed CSV '$0'", csv));
+  if (!csv.empty() && csv.back() == ',') {
+    lines->emplace_back();
+  }
+  return Status::OK();
 }
 
 Result<string> WritePostgresConfig(const PgProcessConf& conf) {
@@ -131,8 +173,10 @@ Result<string> WritePostgresConfig(const PgProcessConf& conf) {
     lines.push_back(line);
   }
 
-  if (!FLAGS_ysql_pg_conf.empty()) {
-    ReadCSVConfigValues(FLAGS_ysql_pg_conf, &lines);
+  if (!FLAGS_ysql_pg_conf_csv.empty()) {
+    RETURN_NOT_OK(ReadCSVValues(FLAGS_ysql_pg_conf_csv, &lines));
+  } else if (!FLAGS_ysql_pg_conf.empty()) {
+    ReadCommaSeparatedValues(FLAGS_ysql_pg_conf, &lines);
   }
 
   if (conf.enable_tls) {
@@ -192,8 +236,10 @@ Result<string> WritePgHbaConfig(const PgProcessConf& conf) {
     }
   }
 
-  if (!FLAGS_ysql_hba_conf.empty()) {
-    ReadCSVConfigValues(FLAGS_ysql_hba_conf, &lines);
+  if (!FLAGS_ysql_hba_conf_csv.empty()) {
+    RETURN_NOT_OK(ReadCSVValues(FLAGS_ysql_hba_conf_csv, &lines));
+  } else if (!FLAGS_ysql_hba_conf.empty()) {
+    ReadCommaSeparatedValues(FLAGS_ysql_hba_conf, &lines);
   }
 
   // Enforce a default hba configuration, so users don't lock themselves out.
@@ -202,7 +248,7 @@ Result<string> WritePgHbaConfig(const PgProcessConf& conf) {
     lines.push_back("host all all ::0/0 trust");
   }
 
-  string conf_path = JoinPathSegments(conf.data_dir, "ysql_hba.conf");
+  const auto conf_path = JoinPathSegments(conf.data_dir, "ysql_hba.conf");
   RETURN_NOT_OK(WriteConfigFile(conf_path, lines));
   return "hba_file=" + conf_path;
 }