From 0c3f6313e8ffdea489f930ec921976eed0b0ef06 Mon Sep 17 00:00:00 2001 From: Yury Shchetinin Date: Fri, 10 Feb 2023 11:27:52 +0300 Subject: [PATCH 01/81] [PLAT-5028] Support separate gflags for read-replicas Summary: 1) Added new flag in userIntent to allow having separate gflags for read replica 2) Added ability to upgrade gflags separately for RR cluster if it has diverged gflags Test Plan: 1) sbt test 2) create RR -> verify gflags are taken from primary cluster 3) run gflags upgrade - verify RR is upgraded too 4) create RR with gflagsDiverged=true -> verify gflags are not taken from primary cluster 5) run gflags upgrade -> verify RR is not changed 6) run gflags upgrade for RR (by API) -> verify applied Examples of requests: This will set gflags for primary in new manner, RR inherits gflags from primary (default behavior) ``` { "sleepAfterMasterRestartMillis": 180000, "sleepAfterTServerRestartMillis": 180000, "clusters": [ { "uuid": "3afcdd4b-63f0-4300-898a-f1dffd777eac", "clusterType": "PRIMARY", "userIntent": { ... "specificGFlags": { "inheritFromPrimary": false, "perProcessFlags": { "value": { "MASTER": {}, "TSERVER": { "tserver-flag": "t1" } } }, "perAZ": {} } }, ... }, { "uuid": "b85eaf40-451a-40db-ac7e-4b05fe464548", "clusterType": "ASYNC", "userIntent": { ... "specificGFlags": { "inheritFromPrimary": true, "perAZ": {} } }, ... } ], "currentClusterType": "PRIMARY", ... } ``` This is for overriding gflags for RR: ``` { "sleepAfterMasterRestartMillis": 180000, "sleepAfterTServerRestartMillis": 180000, "clusters": [ { "uuid": "3afcdd4b-63f0-4300-898a-f1dffd777eac", "clusterType": "PRIMARY", "userIntent": { ... "specificGFlags": { "inheritFromPrimary": false, "perProcessFlags": { "value": { "MASTER": {}, "TSERVER": { "tserver-flag": "t1" } } }, "perAZ": {} } }, ... }, { "uuid": "b85eaf40-451a-40db-ac7e-4b05fe464548", "clusterType": "ASYNC", "userIntent": { ... "specificGFlags": { "inheritFromPrimary": false, "perProcessFlags": { "value": { "MASTER": { "master-flag": "flag 2" }, "TSERVER": { "tserver-flag": "different flag" } } }, "perAZ": {} } }, ... } ], "currentClusterType": "PRIMARY", ... } ``` This is using specific gflags per AZ: ``` { "sleepAfterMasterRestartMillis": 180000, "sleepAfterTServerRestartMillis": 180000, "clusters": [ { "uuid": "3afcdd4b-63f0-4300-898a-f1dffd777eac", "clusterType": "PRIMARY", "userIntent": { ... "specificGFlags": { "inheritFromPrimary": false, "perProcessFlags": { "value": { "MASTER": { "master-flag": "m1" }, "TSERVER": { "tserver-flag": "t1" } } }, "perAZ": { "4345d489-22d3-435a-85b8-984f6979f3ed": { "value": { "MASTER": { "master-flag2": "m3" }, "TSERVER": { "tserver-flag2": "t3" } } }, "1bfc8fed-1521-4dfd-97a2-a66a8bab6052": { "value": { "MASTER": { "master-flag": "m2" }, "TSERVER": { "tserver-flag": "t2" } } } } } }, ... } ], ... } ``` Reviewers: arnav, nsingh, nbhatia, vbansal, sanketh Reviewed By: sanketh Subscribers: shagarwal, jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D20106 --- .../yw/commissioner/HealthChecker.java | 9 +- .../yw/commissioner/UpgradeTaskBase.java | 30 +- .../tasks/UniverseDefinitionTaskBase.java | 61 ++- .../commissioner/tasks/UniverseTaskBase.java | 16 +- .../tasks/XClusterConfigTaskBase.java | 21 +- .../subtasks/KubernetesCommandExecutor.java | 25 +- .../subtasks/UpdateAndPersistGFlags.java | 28 +- .../upgrade/GFlagsKubernetesUpgrade.java | 5 +- .../tasks/upgrade/GFlagsUpgrade.java | 250 +++++++++--- .../tasks/upgrade/ResizeNode.java | 2 + .../tasks/upgrade/SoftwareUpgrade.java | 22 +- .../com/yugabyte/yw/common/NodeManager.java | 8 + .../yugabyte/yw/common/gflags/GFlagsUtil.java | 78 +++- .../yw/common/gflags/SpecificGFlags.java | 127 ++++++ .../handlers/UniverseCRUDHandler.java | 12 +- .../handlers/UpgradeUniverseHandler.java | 19 +- .../yw/forms/GFlagsUpgradeParams.java | 107 +++++ .../forms/UniverseDefinitionTaskParams.java | 22 +- managed/src/main/resources/swagger.json | 36 ++ .../tasks/upgrade/GFlagsUpgradeTest.java | 377 +++++++++++++++++- .../UpgradeUniverseControllerTest.java | 1 - 21 files changed, 1089 insertions(+), 167 deletions(-) create mode 100644 managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/HealthChecker.java b/managed/src/main/java/com/yugabyte/yw/commissioner/HealthChecker.java index 560de5689192..aadf7a4ff8c1 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/HealthChecker.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/HealthChecker.java @@ -31,6 +31,7 @@ import com.google.inject.Singleton; import com.typesafe.config.Config; import com.yugabyte.yw.commissioner.Common.CloudType; +import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase; import com.yugabyte.yw.commissioner.tasks.KubernetesTaskBase; import com.yugabyte.yw.common.EmailHelper; import com.yugabyte.yw.common.NodeUniverseManager; @@ -43,6 +44,7 @@ import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.config.RuntimeConfigFactory; import com.yugabyte.yw.common.config.UniverseConfKeys; +import com.yugabyte.yw.common.gflags.GFlagsUtil; import com.yugabyte.yw.common.metrics.MetricService; import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.forms.AlertingData; @@ -714,8 +716,11 @@ public void checkSingleUniverse(CheckSingleUniverseParams params) { if (providerCode.equals(Common.CloudType.kubernetes.toString())) { nodeInfo.setK8s(true); } - if (userIntent.tserverGFlags.containsKey("ssl_protocols")) { - nodeInfo.setSslProtocol(cluster.userIntent.tserverGFlags.get("ssl_protocols")); + Map tserverGflags = + GFlagsUtil.getGFlagsForNode( + nodeDetails, UniverseTaskBase.ServerType.TSERVER, cluster, details.clusters); + if (tserverGflags.containsKey("ssl_protocols")) { + nodeInfo.setSslProtocol(tserverGflags.get("ssl_protocols")); } if (nodeInfo.enableYSQL && nodeDetails.isYsqlServer) { nodeInfo.setYsqlPort(nodeDetails.ysqlServerRpcPort); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/UpgradeTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/UpgradeTaskBase.java index c7c229f00f21..4097dfee3ea4 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/UpgradeTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/UpgradeTaskBase.java @@ -578,9 +578,20 @@ protected void createServerConfFileUpdateTasks( SubTaskGroupType.UpdatingGFlags, taskParams().nodePrefix); SubTaskGroup subTaskGroup = getTaskExecutor().createSubTaskGroup(subGroupDescription, executor); for (NodeDetails node : nodes) { + ServerType processType = getSingle(processTypes); + Map oldGflags; + Map newGflags; + if (processType == ServerType.MASTER) { + newGflags = masterGflags; + oldGflags = getUserIntent().masterGFlags; + } else if (processType == ServerType.TSERVER) { + newGflags = tserverGflags; + oldGflags = getUserIntent().tserverGFlags; + } else { + throw new IllegalStateException("Unknown process type for updating gflags " + processType); + } subTaskGroup.addSubTask( - getAnsibleConfigureServerTask( - userIntent, node, getSingle(processTypes), masterGflags, tserverGflags)); + getAnsibleConfigureServerTask(userIntent, node, processType, oldGflags, newGflags)); } subTaskGroup.setSubTaskGroupType(SubTaskGroupType.UpdatingGFlags); getRunnableTask().addSubTaskGroup(subTaskGroup); @@ -590,20 +601,13 @@ protected AnsibleConfigureServers getAnsibleConfigureServerTask( UniverseDefinitionTaskParams.UserIntent userIntent, NodeDetails node, ServerType processType, - Map masterGflags, - Map tserverGflags) { + Map oldGflags, + Map newGflags) { AnsibleConfigureServers.Params params = getAnsibleConfigureServerParams( userIntent, node, processType, UpgradeTaskType.GFlags, UpgradeTaskSubType.None); - if (processType.equals(ServerType.MASTER)) { - params.gflags = masterGflags; - params.gflagsToRemove = - GFlagsUtil.getDeletedGFlags(getUserIntent().masterGFlags, masterGflags); - } else { - params.gflags = tserverGflags; - params.gflagsToRemove = - GFlagsUtil.getDeletedGFlags(getUserIntent().tserverGFlags, tserverGflags); - } + params.gflags = newGflags; + params.gflagsToRemove = GFlagsUtil.getDeletedGFlags(oldGflags, newGflags); AnsibleConfigureServers task = createTask(AnsibleConfigureServers.class); task.initialize(params); task.setUserTaskUUID(userTaskUUID); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java index 5316ab868e9d..b5de8528c83f 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseDefinitionTaskBase.java @@ -28,6 +28,7 @@ import com.yugabyte.yw.common.PlacementInfoUtil.SelectMastersResult; import com.yugabyte.yw.common.Util; import com.yugabyte.yw.common.certmgmt.EncryptionInTransitUtil; +import com.yugabyte.yw.common.gflags.GFlagsUtil; import com.yugabyte.yw.common.helm.HelmUtils; import com.yugabyte.yw.common.password.RedactingService; import com.yugabyte.yw.forms.CertsRotateParams; @@ -566,16 +567,6 @@ public void selectNumMastersAZ(PlacementInfo pi) { PlacementInfoUtil.selectNumMastersAZ(pi, numTotalMasters); } - /** - * Return map of primary cluster gflags based on serverType - * - * @param taskType - */ - public Map getPrimaryClusterGFlags(ServerType taskType, Universe universe) { - UserIntent userIntent = universe.getUniverseDetails().getPrimaryCluster().userIntent; - return taskType.equals(ServerType.MASTER) ? userIntent.masterGFlags : userIntent.tserverGFlags; - } - // Utility method so that the same tasks can be executed in StopNodeInUniverse.java // part of the automatic restart process of a master, if applicable, as well as in // StartMasterOnNode.java for any user-specified master starts. @@ -664,24 +655,10 @@ public void createGFlagsOverrideTasks( getTaskExecutor().createSubTaskGroup("AnsibleConfigureServersGFlags", executor); Universe universe = Universe.getOrBadRequest(taskParams().universeUUID); - // Read gflags from taskPrams primary cluster. If it is null, read it from the universe. - Map gflags; - Cluster primaryClusterInTaskParams = taskParams().getPrimaryCluster(); - if (primaryClusterInTaskParams != null) { - UserIntent userIntent = primaryClusterInTaskParams.userIntent; - gflags = - serverType.equals(ServerType.MASTER) ? userIntent.masterGFlags : userIntent.tserverGFlags; - log.debug( - "gflags in taskParams: {}, gflags in universeDetails: {}", - gflags, - getPrimaryClusterGFlags(serverType, universe)); - } else { - gflags = getPrimaryClusterGFlags(serverType, universe); - log.debug("gflags gathered from the UniverseDetails : {}", gflags); - } - for (NodeDetails node : nodes) { - UserIntent userIntent = taskParams().getClusterByUuid(node.placementUuid).userIntent; + Cluster cluster = taskParams().getClusterByUuid(node.placementUuid); + UserIntent userIntent = cluster.userIntent; + AnsibleConfigureServers.Params params = new AnsibleConfigureServers.Params(); // Set the device information (numVolumes, volumeSize, etc.) params.deviceInfo = userIntent.getDeviceInfoForNode(node); @@ -724,7 +701,9 @@ public void createGFlagsOverrideTasks( // Add task type params.type = UpgradeTaskParams.UpgradeTaskType.GFlags; params.setProperty("processType", serverType.toString()); - params.gflags = gflags; + params.gflags = + GFlagsUtil.getGFlagsForNode( + node, serverType, cluster, universe.getUniverseDetails().clusters); params.useSystemd = userIntent.useSystemd; paramsCustomizer.accept(params); AnsibleConfigureServers task = createTask(AnsibleConfigureServers.class); @@ -1065,7 +1044,8 @@ public SubTaskGroup createConfigureServerTasks( SubTaskGroup subTaskGroup = getTaskExecutor().createSubTaskGroup("AnsibleConfigureServers", executor); for (NodeDetails node : nodes) { - UserIntent userIntent = taskParams().getClusterByUuid(node.placementUuid).userIntent; + Cluster cluster = taskParams().getClusterByUuid(node.placementUuid); + UserIntent userIntent = cluster.userIntent; AnsibleConfigureServers.Params params = new AnsibleConfigureServers.Params(); // Set the device information (numVolumes, volumeSize, etc.) params.deviceInfo = userIntent.getDeviceInfoForNode(node); @@ -1113,10 +1093,20 @@ public SubTaskGroup createConfigureServerTasks( params.type = UpgradeTaskParams.UpgradeTaskType.GFlags; if (params.isMaster) { params.setProperty("processType", ServerType.MASTER.toString()); - params.gflags = getPrimaryClusterGFlags(ServerType.MASTER, universe); + params.gflags = + GFlagsUtil.getGFlagsForNode( + node, + ServerType.MASTER, + universe.getUniverseDetails().getClusterByUuid(cluster.uuid), + universe.getUniverseDetails().clusters); } else { params.setProperty("processType", ServerType.TSERVER.toString()); - params.gflags = getPrimaryClusterGFlags(ServerType.TSERVER, universe); + params.gflags = + GFlagsUtil.getGFlagsForNode( + node, + ServerType.TSERVER, + universe.getUniverseDetails().getClusterByUuid(cluster.uuid), + universe.getUniverseDetails().clusters); } } // Create the Ansible task to get the server info. @@ -2204,7 +2194,13 @@ public AnsibleConfigureServers.Params getAnsibleConfigureServerParams( UpgradeTaskSubType taskSubType) { AnsibleConfigureServers.Params params = getBaseAnsibleServerTaskParams(userIntent, node, processType, type, taskSubType); - Map gflags = getPrimaryClusterGFlags(processType, getUniverse()); + Universe universe = getUniverse(); + Map gflags = + GFlagsUtil.getGFlagsForNode( + node, + processType, + universe.getCluster(node.placementUuid), + universe.getUniverseDetails().clusters); // Add the universe uuid. params.universeUUID = taskParams().universeUUID; @@ -2218,7 +2214,6 @@ public AnsibleConfigureServers.Params getAnsibleConfigureServerParams( params.allowInsecure = taskParams().allowInsecure; params.setTxnTableWaitCountFlag = taskParams().setTxnTableWaitCountFlag; - Universe universe = Universe.getOrBadRequest(taskParams().universeUUID); UUID custUUID = Customer.get(universe.customerId).uuid; params.callhomeLevel = CustomerConfig.getCallhomeLevel(custUUID); params.rootCA = universe.getUniverseDetails().rootCA; diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java index a549995db97f..fe206e6d23d5 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/UniverseTaskBase.java @@ -109,6 +109,7 @@ import com.yugabyte.yw.common.UniverseInProgressException; import com.yugabyte.yw.common.Util; import com.yugabyte.yw.common.config.UniverseConfKeys; +import com.yugabyte.yw.common.gflags.SpecificGFlags; import com.yugabyte.yw.common.services.YBClientService; import com.yugabyte.yw.forms.BackupRequestParams; import com.yugabyte.yw.forms.BackupTableParams; @@ -153,6 +154,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -1664,15 +1666,27 @@ protected SubTaskGroup createUpdateMountedDisksTask( return subTaskGroup; } - /** Creates a task to persist customized gflags to be used by server processes. */ public SubTaskGroup updateGFlagsPersistTasks( Map masterGFlags, Map tserverGFlags) { + return updateGFlagsPersistTasks(null, masterGFlags, tserverGFlags, null); + } + + /** Creates a task to persist customized gflags to be used by server processes. */ + public SubTaskGroup updateGFlagsPersistTasks( + @Nullable Cluster cluster, + Map masterGFlags, + Map tserverGFlags, + @Nullable SpecificGFlags specificGFlags) { SubTaskGroup subTaskGroup = getTaskExecutor().createSubTaskGroup("UpdateAndPersistGFlags", executor); UpdateAndPersistGFlags.Params params = new UpdateAndPersistGFlags.Params(); params.universeUUID = taskParams().universeUUID; params.masterGFlags = masterGFlags; params.tserverGFlags = tserverGFlags; + params.specificGFlags = specificGFlags; + if (cluster != null) { + params.clusterUUIDs = Collections.singletonList(cluster.uuid); + } UpdateAndPersistGFlags task = createTask(UpdateAndPersistGFlags.class); task.initialize(params); subTaskGroup.addSubTask(task); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java index 80043a7ec4c4..ac4909d013bb 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/XClusterConfigTaskBase.java @@ -21,6 +21,7 @@ import com.yugabyte.yw.commissioner.tasks.subtasks.xcluster.XClusterConfigSync; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.Util; +import com.yugabyte.yw.common.gflags.GFlagsUtil; import com.yugabyte.yw.common.services.YBClientService; import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.forms.ITaskParams; @@ -1412,21 +1413,27 @@ public static Set getTableIds( public static boolean isTransactionalReplication( @Nullable Universe sourceUniverse, Universe targetUniverse) { - UniverseDefinitionTaskParams.UserIntent targetUniverseUserIntent = - targetUniverse.getUniverseDetails().getPrimaryCluster().userIntent; + Map targetMasterGFlags = + GFlagsUtil.getBaseGFlags( + ServerType.MASTER, + targetUniverse.getUniverseDetails().getPrimaryCluster(), + targetUniverse.getUniverseDetails().clusters); String gflagValueOnTarget = - targetUniverseUserIntent.masterGFlags.get(ENABLE_REPLICATE_TRANSACTION_STATUS_TABLE_GFLAG); + targetMasterGFlags.get(ENABLE_REPLICATE_TRANSACTION_STATUS_TABLE_GFLAG); Boolean gflagValueOnTargetBoolean = gflagValueOnTarget != null ? Boolean.valueOf(gflagValueOnTarget) : null; if (sourceUniverse != null) { + Map sourceMasterGFlags = + GFlagsUtil.getBaseGFlags( + ServerType.MASTER, + sourceUniverse.getUniverseDetails().getPrimaryCluster(), + sourceUniverse.getUniverseDetails().clusters); + // Replication between a universe with the gflag `enable_replicate_transaction_status_table` // and a universe without it is not allowed. - UniverseDefinitionTaskParams.UserIntent sourceUniverseUserIntent = - sourceUniverse.getUniverseDetails().getPrimaryCluster().userIntent; String gflagValueOnSource = - sourceUniverseUserIntent.masterGFlags.get( - ENABLE_REPLICATE_TRANSACTION_STATUS_TABLE_GFLAG); + sourceMasterGFlags.get(ENABLE_REPLICATE_TRANSACTION_STATUS_TABLE_GFLAG); Boolean gflagValueOnSourceBoolean = gflagValueOnSource != null ? Boolean.valueOf(gflagValueOnSource) : null; if (!Objects.equals(gflagValueOnSourceBoolean, gflagValueOnTargetBoolean)) { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java index a847dd90ade6..576cd3df65c5 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/KubernetesCommandExecutor.java @@ -30,6 +30,7 @@ import com.yugabyte.yw.common.certmgmt.CertificateHelper; import com.yugabyte.yw.common.certmgmt.EncryptionInTransitUtil; import com.yugabyte.yw.common.certmgmt.providers.CertificateProviderInterface; +import com.yugabyte.yw.common.gflags.GFlagsUtil; import com.yugabyte.yw.common.helm.HelmUtils; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseTaskParams; @@ -531,10 +532,12 @@ private String generateHelmOverride() { Map regionConfig = new HashMap(); Universe u = Universe.getOrBadRequest(taskParams().universeUUID); - UniverseDefinitionTaskParams.UserIntent userIntent = + + UniverseDefinitionTaskParams.Cluster cluster = taskParams().isReadOnlyCluster - ? u.getUniverseDetails().getReadOnlyClusters().get(0).userIntent - : u.getUniverseDetails().getPrimaryCluster().userIntent; + ? u.getUniverseDetails().getReadOnlyClusters().get(0) + : u.getUniverseDetails().getPrimaryCluster(); + UniverseDefinitionTaskParams.UserIntent userIntent = cluster.userIntent; InstanceType instanceType = InstanceType.get(UUID.fromString(userIntent.provider), userIntent.instanceType); if (instanceType == null) { @@ -832,13 +835,12 @@ private String generateHelmOverride() { partition.put("master", taskParams().masterPartition); overrides.put("partition", partition); - UUID placementUuid = - taskParams().isReadOnlyCluster - ? u.getUniverseDetails().getReadOnlyClusters().get(0).uuid - : u.getUniverseDetails().getPrimaryCluster().uuid; + UUID placementUuid = cluster.uuid; Map gflagOverrides = new HashMap<>(); // Go over master flags. - Map masterOverrides = new HashMap(userIntent.masterGFlags); + Map masterOverrides = + new HashMap<>( + GFlagsUtil.getBaseGFlags(ServerType.MASTER, cluster, u.getUniverseDetails().clusters)); if (placementCloud != null && masterOverrides.get("placement_cloud") == null) { masterOverrides.put("placement_cloud", placementCloud); } @@ -862,7 +864,8 @@ private String generateHelmOverride() { // Go over tserver flags. Map tserverOverrides = - new HashMap(primaryClusterIntent.tserverGFlags); + new HashMap( + GFlagsUtil.getBaseGFlags(ServerType.TSERVER, cluster, u.getUniverseDetails().clusters)); if (!primaryClusterIntent .enableYSQL) { // In the UI, we can choose not to show these entries for read replica. tserverOverrides.put("enable_ysql", "false"); @@ -924,7 +927,7 @@ private String generateHelmOverride() { // loadbalancers, so the annotations will be at the provider level. // TODO (Arnav): Update this to use overrides created at the provider, region or // zone level. - Map annotations = new HashMap(); + Map annotations; String overridesYAML = null; if (!azConfig.containsKey("OVERRIDES")) { if (!regionConfig.containsKey("OVERRIDES")) { @@ -939,7 +942,7 @@ private String generateHelmOverride() { } if (overridesYAML != null) { - annotations = (HashMap) yaml.load(overridesYAML); + annotations = yaml.load(overridesYAML); if (annotations != null) { HelmUtils.mergeYaml(overrides, annotations); } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateAndPersistGFlags.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateAndPersistGFlags.java index ce4b590e101a..e9dd84ad4633 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateAndPersistGFlags.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/UpdateAndPersistGFlags.java @@ -13,13 +13,16 @@ import com.yugabyte.yw.commissioner.BaseTaskDependencies; import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase; import com.yugabyte.yw.common.gflags.GFlagsUtil; +import com.yugabyte.yw.common.gflags.SpecificGFlags; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster; -import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; import com.yugabyte.yw.forms.UniverseTaskParams; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.Universe.UniverseUpdater; +import java.util.List; import java.util.Map; +import java.util.UUID; +import java.util.stream.Collectors; import javax.inject.Inject; import lombok.extern.slf4j.Slf4j; @@ -35,6 +38,8 @@ protected UpdateAndPersistGFlags(BaseTaskDependencies baseTaskDependencies) { public static class Params extends UniverseTaskParams { public Map masterGFlags; public Map tserverGFlags; + public List clusterUUIDs; + public SpecificGFlags specificGFlags; } protected Params taskParams() { @@ -64,14 +69,21 @@ public void run(Universe universe) { throw new RuntimeException(msg); } - // Update the gflags. - UserIntent userIntent = universeDetails.getPrimaryCluster().userIntent; - userIntent.masterGFlags = taskParams().masterGFlags; - userIntent.tserverGFlags = taskParams().tserverGFlags; - GFlagsUtil.syncGflagsToIntent(userIntent.tserverGFlags, userIntent); - GFlagsUtil.syncGflagsToIntent(userIntent.masterGFlags, userIntent); + List clusterList; + if (taskParams().clusterUUIDs != null && taskParams().clusterUUIDs.size() > 0) { + clusterList = + universeDetails + .clusters + .stream() + .filter(c -> taskParams().clusterUUIDs.contains(c.uuid)) + .collect(Collectors.toList()); + } else { + clusterList = universeDetails.clusters; + } - for (Cluster cluster : universeDetails.getReadOnlyClusters()) { + // Update the gflags. + for (Cluster cluster : clusterList) { + cluster.userIntent.specificGFlags = taskParams().specificGFlags; cluster.userIntent.masterGFlags = taskParams().masterGFlags; cluster.userIntent.tserverGFlags = taskParams().tserverGFlags; GFlagsUtil.syncGflagsToIntent(cluster.userIntent.tserverGFlags, cluster.userIntent); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgrade.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgrade.java index 1478a0ea4199..b033064a2167 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgrade.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsKubernetesUpgrade.java @@ -8,6 +8,7 @@ import com.yugabyte.yw.forms.GFlagsUpgradeParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.Cluster; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; +import com.yugabyte.yw.models.Universe; import javax.inject.Inject; public class GFlagsKubernetesUpgrade extends KubernetesUpgradeTaskBase { @@ -31,10 +32,12 @@ public SubTaskGroupType getTaskSubGroupType() { public void run() { runUpgrade( () -> { + // TODO: support specific gflags Cluster cluster = getUniverse().getUniverseDetails().getPrimaryCluster(); UserIntent userIntent = cluster.userIntent; + Universe universe = getUniverse(); // Verify the request params and fail if invalid - taskParams().verifyParams(getUniverse()); + taskParams().verifyParams(universe); // Update the list of parameter key/values in the universe with the new ones. updateGFlagsPersistTasks(taskParams().masterGFlags, taskParams().tserverGFlags) .setSubTaskGroupType(getTaskSubGroupType()); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgrade.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgrade.java index 12012bac1436..828c3e5a34b2 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgrade.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgrade.java @@ -3,6 +3,7 @@ package com.yugabyte.yw.commissioner.tasks.upgrade; import com.yugabyte.yw.commissioner.BaseTaskDependencies; +import com.yugabyte.yw.commissioner.TaskExecutor; import com.yugabyte.yw.commissioner.UpgradeTaskBase; import com.yugabyte.yw.commissioner.UserTaskDetails.SubTaskGroupType; import com.yugabyte.yw.common.config.UniverseConfKeys; @@ -12,9 +13,13 @@ import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.NodeDetails; import com.yugabyte.yw.models.helpers.NodeDetails.NodeState; -import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; import javax.inject.Inject; import lombok.extern.slf4j.Slf4j; @@ -45,68 +50,156 @@ public NodeState getNodeState() { public void run() { runUpgrade( () -> { - UniverseDefinitionTaskParams.UserIntent userIntent = getUserIntent(); - - boolean changedByMasterFlags = - GFlagsUtil.syncGflagsToIntent(taskParams().masterGFlags, userIntent); - boolean changedByTserverFlags = - GFlagsUtil.syncGflagsToIntent(taskParams().tserverGFlags, userIntent); - log.debug( - "Intent changed by master {} by tserver {}", - changedByMasterFlags, - changedByTserverFlags); - - // Fetch master and tserver nodes if there is change in gflags - List masterNodes = - (changedByMasterFlags || changedByTserverFlags) - || !taskParams().masterGFlags.equals(getUserIntent().masterGFlags) - ? fetchMasterNodes(taskParams().upgradeOption) - : new ArrayList<>(); - - List tServerNodes = - (changedByMasterFlags || changedByTserverFlags) - || !taskParams().tserverGFlags.equals(getUserIntent().tserverGFlags) - ? fetchTServerNodes(taskParams().upgradeOption) - : new ArrayList<>(); - Universe universe = getUniverse(); - - if (!config.getBoolean("yb.cloud.enabled") - && !confGetter.getConfForScope(universe, UniverseConfKeys.gflagsAllowUserOverride)) { - masterNodes.forEach( - node -> - checkForbiddenToOverrideGFlags( - node, - userIntent, - universe, - ServerType.MASTER, - taskParams().masterGFlags, - config)); - tServerNodes.forEach( - node -> - checkForbiddenToOverrideGFlags( - node, - userIntent, - universe, - ServerType.TSERVER, - taskParams().tserverGFlags, - config)); - } - // Verify the request params and fail if invalid taskParams().verifyParams(universe); - // Upgrade GFlags in all nodes - createGFlagUpgradeTasks(userIntent, masterNodes, tServerNodes); - // Update the list of parameter key/values in the universe with the new ones. - updateGFlagsPersistTasks(taskParams().masterGFlags, taskParams().tserverGFlags) - .setSubTaskGroupType(getTaskSubGroupType()); + + boolean checkForbiddenToOverride = + !config.getBoolean("yb.cloud.enabled") + && !confGetter.getConfForScope( + universe, UniverseConfKeys.gflagsAllowUserOverride); + List curClusters = + universe.getUniverseDetails().clusters; + Map newClusters = + taskParams().getNewVersionsOfClusters(universe); + for (UniverseDefinitionTaskParams.Cluster curCluster : curClusters) { + UniverseDefinitionTaskParams.UserIntent userIntent = curCluster.userIntent; + UniverseDefinitionTaskParams.Cluster newCluster = newClusters.get(curCluster.uuid); + Map masterGflags = + GFlagsUtil.getBaseGFlags(ServerType.MASTER, newCluster, newClusters.values()); + Map tserverGflags = + GFlagsUtil.getBaseGFlags(ServerType.TSERVER, newCluster, newClusters.values()); + log.debug( + "Cluster {} master: new flags {} old flags {}", + curCluster.clusterType, + masterGflags, + GFlagsUtil.getBaseGFlags(ServerType.MASTER, curCluster, curClusters)); + log.debug( + "Cluster {} tserver: new flags {} old flags {}", + curCluster.clusterType, + tserverGflags, + GFlagsUtil.getBaseGFlags(ServerType.TSERVER, curCluster, curClusters)); + + boolean changedByMasterFlags = + curCluster.clusterType == UniverseDefinitionTaskParams.ClusterType.PRIMARY + && GFlagsUtil.syncGflagsToIntent(masterGflags, userIntent); + boolean changedByTserverFlags = + curCluster.clusterType == UniverseDefinitionTaskParams.ClusterType.PRIMARY + && GFlagsUtil.syncGflagsToIntent(tserverGflags, userIntent); + log.debug( + "Intent changed by master {} by tserver {}", + changedByMasterFlags, + changedByTserverFlags); + + // Fetch master and tserver nodes if there is change in gflags + List masterNodes = fetchMasterNodes(taskParams().upgradeOption); + List tServerNodes = fetchTServerNodes(taskParams().upgradeOption); + boolean applyToAllNodes = changedByMasterFlags || changedByTserverFlags; + masterNodes = + masterNodes + .stream() + .filter(n -> n.placementUuid.equals(curCluster.uuid)) + .filter( + n -> + applyToAllNodes + || nodeHasGflagsChanges( + n, + ServerType.MASTER, + curCluster, + curClusters, + newCluster, + newClusters.values())) + .collect(Collectors.toList()); + tServerNodes = + tServerNodes + .stream() + .filter(n -> n.placementUuid.equals(curCluster.uuid)) + .filter( + n -> + applyToAllNodes + || nodeHasGflagsChanges( + n, + ServerType.TSERVER, + curCluster, + curClusters, + newCluster, + newClusters.values())) + .collect(Collectors.toList()); + + if (checkForbiddenToOverride) { + masterNodes.forEach( + node -> + checkForbiddenToOverrideGFlags( + node, + userIntent, + universe, + ServerType.MASTER, + GFlagsUtil.getGFlagsForNode( + node, ServerType.MASTER, newCluster, newClusters.values()), + config)); + tServerNodes.forEach( + node -> + checkForbiddenToOverrideGFlags( + node, + userIntent, + universe, + ServerType.TSERVER, + GFlagsUtil.getGFlagsForNode( + node, ServerType.TSERVER, newCluster, newClusters.values()), + config)); + } + // Upgrade GFlags in all nodes + createGFlagUpgradeTasks( + userIntent, + masterNodes, + tServerNodes, + curCluster, + curClusters, + newCluster, + newClusters.values()); + // Update the list of parameter key/values in the universe with the new ones. + if (hasUpdatedGFlags(newCluster.userIntent, curCluster.userIntent)) { + updateGFlagsPersistTasks( + curCluster, + newCluster.userIntent.masterGFlags, + newCluster.userIntent.tserverGFlags, + newCluster.userIntent.specificGFlags) + .setSubTaskGroupType(getTaskSubGroupType()); + } + } }); } + private boolean hasUpdatedGFlags( + UniverseDefinitionTaskParams.UserIntent newIntent, + UniverseDefinitionTaskParams.UserIntent curIntent) { + return !Objects.equals(newIntent.specificGFlags, curIntent.specificGFlags) + || !Objects.equals(newIntent.masterGFlags, curIntent.masterGFlags) + || !Objects.equals(newIntent.tserverGFlags, curIntent.tserverGFlags); + } + + private boolean nodeHasGflagsChanges( + NodeDetails node, + ServerType serverType, + UniverseDefinitionTaskParams.Cluster curCluster, + List curClusters, + UniverseDefinitionTaskParams.Cluster newClusterVersion, + Collection newClusters) { + Map newGflags = + GFlagsUtil.getGFlagsForNode(node, serverType, newClusterVersion, newClusters); + Map oldGflags = + GFlagsUtil.getGFlagsForNode(node, serverType, curCluster, curClusters); + return !newGflags.equals(oldGflags); + } + private void createGFlagUpgradeTasks( UniverseDefinitionTaskParams.UserIntent userIntent, List masterNodes, - List tServerNodes) { + List tServerNodes, + UniverseDefinitionTaskParams.Cluster curCluster, + List curClusters, + UniverseDefinitionTaskParams.Cluster newCluster, + Collection newClusters) { switch (taskParams().upgradeOption) { case ROLLING_UPGRADE: createRollingUpgradeTaskFlow( @@ -115,8 +208,10 @@ private void createGFlagUpgradeTasks( userIntent, nodes, processTypes, - taskParams().masterGFlags, - taskParams().tserverGFlags), + curCluster, + curClusters, + newCluster, + newClusters), masterNodes, tServerNodes, RUN_BEFORE_STOPPING, @@ -129,8 +224,10 @@ private void createGFlagUpgradeTasks( userIntent, nodes, processTypes, - taskParams().masterGFlags, - taskParams().tserverGFlags), + curCluster, + curClusters, + newCluster, + newClusters), masterNodes, tServerNodes, RUN_BEFORE_STOPPING, @@ -144,8 +241,10 @@ private void createGFlagUpgradeTasks( userIntent, nodeList, processTypes, - taskParams().masterGFlags, - taskParams().tserverGFlags); + curCluster, + curClusters, + newCluster, + newClusters); createSetFlagInMemoryTasks( nodeList, processType, @@ -162,4 +261,35 @@ private void createGFlagUpgradeTasks( break; } } + + private void createServerConfFileUpdateTasks( + UniverseDefinitionTaskParams.UserIntent userIntent, + List nodes, + Set processTypes, + UniverseDefinitionTaskParams.Cluster curCluster, + Collection curClusters, + UniverseDefinitionTaskParams.Cluster newCluster, + Collection newClusters) { + // If the node list is empty, we don't need to do anything. + if (nodes.isEmpty()) { + return; + } + String subGroupDescription = + String.format( + "AnsibleConfigureServers (%s) for: %s", + SubTaskGroupType.UpdatingGFlags, taskParams().nodePrefix); + TaskExecutor.SubTaskGroup subTaskGroup = + getTaskExecutor().createSubTaskGroup(subGroupDescription, executor); + for (NodeDetails node : nodes) { + ServerType processType = getSingle(processTypes); + Map newGFlags = + GFlagsUtil.getGFlagsForNode(node, processType, newCluster, newClusters); + Map oldGFlags = + GFlagsUtil.getGFlagsForNode(node, processType, curCluster, curClusters); + subTaskGroup.addSubTask( + getAnsibleConfigureServerTask(userIntent, node, processType, oldGFlags, newGFlags)); + } + subTaskGroup.setSubTaskGroupType(SubTaskGroupType.UpdatingGFlags); + getRunnableTask().addSubTaskGroup(subTaskGroup); + } } diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/ResizeNode.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/ResizeNode.java index 61ef86c0441b..98a155ce52e2 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/ResizeNode.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/ResizeNode.java @@ -62,8 +62,10 @@ public void run() { UserIntent userIntentForFlags = getUserIntent(); + // TODO: support specific gflags boolean updateMasterFlags; boolean updateTserverFlags; + // TODO: Support specific gflags here if (taskParams().flagsProvided()) { boolean changedByMasterFlags = GFlagsUtil.syncGflagsToIntent(taskParams().masterGFlags, userIntentForFlags); diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareUpgrade.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareUpgrade.java index 7e44592ea4fc..bb12b0e17723 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareUpgrade.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/upgrade/SoftwareUpgrade.java @@ -9,6 +9,7 @@ import com.yugabyte.yw.commissioner.tasks.XClusterConfigTaskBase; import com.yugabyte.yw.common.Util; import com.yugabyte.yw.common.config.UniverseConfKeys; +import com.yugabyte.yw.common.gflags.GFlagsUtil; import com.yugabyte.yw.forms.SoftwareUpgradeParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UpgradeTaskParams.UpgradeTaskSubType; @@ -145,8 +146,10 @@ private void createDownloadTasks(Collection nodes, String softwareV private void createXClusterSourceRootCertDirPathGFlagTasks() { Universe targetUniverse = getUniverse(); UniverseDefinitionTaskParams targetUniverseDetails = targetUniverse.getUniverseDetails(); + UniverseDefinitionTaskParams.Cluster targetPrimaryCluster = + targetUniverseDetails.getPrimaryCluster(); UniverseDefinitionTaskParams.UserIntent targetPrimaryUserIntent = - targetUniverseDetails.getPrimaryCluster().userIntent; + targetPrimaryCluster.userIntent; List xClusterConfigsAsTarget = XClusterConfig.getByTargetUniverseUUID(targetUniverse.universeUUID); @@ -158,14 +161,14 @@ private void createXClusterSourceRootCertDirPathGFlagTasks() { XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG); targetUniverseDetails.xClusterInfo.sourceRootCertDirPath = manualSourceRootCertDirPath.toString(); - targetPrimaryUserIntent.masterGFlags.remove( - XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG); - targetPrimaryUserIntent.tserverGFlags.remove( - XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG); + GFlagsUtil.removeGFlag( + targetPrimaryUserIntent, + XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG, + ServerType.TSERVER, + ServerType.MASTER); } else { targetUniverseDetails.xClusterInfo.sourceRootCertDirPath = - XClusterConfigTaskBase.getProducerCertsDir( - targetUniverseDetails.getPrimaryCluster().userIntent.provider); + XClusterConfigTaskBase.getProducerCertsDir(targetPrimaryUserIntent.provider); } log.debug( "sourceRootCertDirPath={} will be used", targetUniverseDetails.getSourceRootCertDirPath()); @@ -232,7 +235,10 @@ private void createXClusterSourceRootCertDirPathGFlagTasks() { // nodes, and it should regenerate the conf files. if (manualSourceRootCertDirPath != null) { updateGFlagsPersistTasks( - targetPrimaryUserIntent.masterGFlags, targetPrimaryUserIntent.tserverGFlags) + targetPrimaryCluster, + targetPrimaryUserIntent.masterGFlags, + targetPrimaryUserIntent.tserverGFlags, + targetPrimaryUserIntent.specificGFlags) .setSubTaskGroupType(SubTaskGroupType.UpdatingGFlags); } else { createGFlagsOverrideTasks(targetUniverse.getMasters(), ServerType.MASTER); diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java index 3f1691225a05..b3c703d32b4f 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java @@ -1208,6 +1208,14 @@ private static SkipCertValidationType getSkipCertValidationType( skipHostValidation = GFlagsUtil.shouldSkipServerEndpointVerification(userIntent.masterGFlags) || GFlagsUtil.shouldSkipServerEndpointVerification(userIntent.tserverGFlags); + if (userIntent.specificGFlags != null) { + skipHostValidation = + skipHostValidation + || GFlagsUtil.shouldSkipServerEndpointVerification( + userIntent.specificGFlags.getGFlags(null, ServerType.MASTER)) + || GFlagsUtil.shouldSkipServerEndpointVerification( + userIntent.specificGFlags.getGFlags(null, ServerType.TSERVER)); + } } return skipHostValidation ? SkipCertValidationType.HOSTNAME : SkipCertValidationType.NONE; } diff --git a/managed/src/main/java/com/yugabyte/yw/common/gflags/GFlagsUtil.java b/managed/src/main/java/com/yugabyte/yw/common/gflags/GFlagsUtil.java index 8993c7f5c064..125d9509a0a7 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/gflags/GFlagsUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/gflags/GFlagsUtil.java @@ -30,6 +30,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -42,6 +43,7 @@ import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.commons.collections.MapUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -536,8 +538,23 @@ public static boolean shouldSkipServerEndpointVerification(Map g */ public static void checkGflagsAndIntentConsistency( UniverseDefinitionTaskParams.UserIntent userIntent) { - for (Map gflags : - Arrays.asList(userIntent.masterGFlags, userIntent.tserverGFlags)) { + List> masterAndTserverGFlags = + Arrays.asList(userIntent.masterGFlags, userIntent.tserverGFlags); + if (userIntent.specificGFlags != null) { + masterAndTserverGFlags = + Arrays.asList( + userIntent + .specificGFlags + .getPerProcessFlags() + .value + .getOrDefault(UniverseTaskBase.ServerType.MASTER, new HashMap<>()), + userIntent + .specificGFlags + .getPerProcessFlags() + .value + .getOrDefault(UniverseTaskBase.ServerType.TSERVER, new HashMap<>())); + } + for (Map gflags : masterAndTserverGFlags) { GFLAG_TO_INTENT_ACCESSOR.forEach( (gflagKey, accessor) -> { if (gflags.containsKey(gflagKey)) { @@ -642,6 +659,43 @@ public static boolean syncGflagsToIntent( return result.get(); } + public static Map getBaseGFlags( + UniverseTaskBase.ServerType serverType, + UniverseDefinitionTaskParams.Cluster cluster, + Collection allClusters) { + return getGFlagsForNode(null, serverType, cluster, allClusters); + } + + public static Map getGFlagsForNode( + @Nullable NodeDetails node, + UniverseTaskBase.ServerType serverType, + UniverseDefinitionTaskParams.Cluster cluster, + Collection allClusters) { + UserIntent userIntent = cluster.userIntent; + UniverseDefinitionTaskParams.Cluster primary = + allClusters + .stream() + .filter(c -> c.clusterType == UniverseDefinitionTaskParams.ClusterType.PRIMARY) + .findFirst() + .orElse(null); + if (userIntent.specificGFlags != null) { + if (userIntent.specificGFlags.isInheritFromPrimary()) { + if (cluster.clusterType == UniverseDefinitionTaskParams.ClusterType.PRIMARY) { + throw new IllegalStateException("Primary cluster has inherit gflags"); + } + return getGFlagsForNode(node, serverType, primary, allClusters); + } + return userIntent.specificGFlags.getGFlags(node, serverType); + } else { + if (cluster.clusterType == UniverseDefinitionTaskParams.ClusterType.ASYNC) { + return getGFlagsForNode(node, serverType, primary, allClusters); + } + return serverType == UniverseTaskBase.ServerType.MASTER + ? userIntent.masterGFlags + : userIntent.tserverGFlags; + } + } + private static String getMountPoints(AnsibleConfigureServers.Params taskParam) { if (taskParam.deviceInfo.mountPoints != null) { return taskParam.deviceInfo.mountPoints; @@ -770,6 +824,26 @@ public static String checkForbiddenToOverride( return null; } + public static void removeGFlag( + UniverseDefinitionTaskParams.UserIntent userIntent, + String gflagKey, + UniverseTaskBase.ServerType... serverTypes) { + if (userIntent.specificGFlags != null) { + userIntent.specificGFlags.removeGFlag(gflagKey, serverTypes); + } else { + for (UniverseTaskBase.ServerType serverType : serverTypes) { + switch (serverType) { + case MASTER: + userIntent.masterGFlags.remove(gflagKey); + break; + case TSERVER: + userIntent.tserverGFlags.remove(gflagKey); + break; + } + } + } + } + private interface StringIntentAccessor { Function strGetter(); diff --git a/managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java b/managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java new file mode 100644 index 000000000000..c0dbaa3ad185 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java @@ -0,0 +1,127 @@ +// Copyright (c) YugaByte, Inc. + +package com.yugabyte.yw.common.gflags; + +import static com.yugabyte.yw.commissioner.tasks.UniverseTaskBase.ServerType.MASTER; +import static com.yugabyte.yw.commissioner.tasks.UniverseTaskBase.ServerType.TSERVER; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.ImmutableMap; +import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase; +import com.yugabyte.yw.models.helpers.NodeDetails; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import javax.annotation.Nullable; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +public class SpecificGFlags { + + @EqualsAndHashCode + public static class PerProcessFlags { + public Map> value = new HashMap<>(); + } + + private boolean inheritFromPrimary; + private PerProcessFlags perProcessFlags; + private Map perAZ = new HashMap<>(); + + @JsonIgnore + public Map getGFlags( + NodeDetails nodeDetails, UniverseTaskBase.ServerType process) { + UUID azUUID = nodeDetails == null ? null : nodeDetails.azUuid; + return getGFlags(azUUID, process); + } + + private Map getGFlags( + @Nullable UUID azUuid, UniverseTaskBase.ServerType process) { + Map result = new HashMap<>(); + if (perProcessFlags != null) { + result.putAll(perProcessFlags.value.getOrDefault(process, new HashMap<>())); + } + PerProcessFlags azFlags = perAZ.get(azUuid); + if (azFlags != null) { + result.putAll(azFlags.value.getOrDefault(process, new HashMap<>())); + } + return result; + } + + @JsonIgnore + public void validateConsistency() { + Collection azUuids = new ArrayList<>(Collections.singletonList(null)); + if (perAZ != null) { + azUuids.addAll(perAZ.keySet()); + } + for (UUID azUuid : azUuids) { + GFlagsUtil.checkConsistency( + getGFlags(azUuid, UniverseTaskBase.ServerType.MASTER), + getGFlags(azUuid, UniverseTaskBase.ServerType.TSERVER)); + } + } + + @JsonIgnore + public SpecificGFlags clone() { + SpecificGFlags newValue = new SpecificGFlags(); + newValue.setInheritFromPrimary(inheritFromPrimary); + newValue.perProcessFlags = clone(perProcessFlags); + perAZ.forEach( + (k, v) -> { + newValue.perAZ.put(k, clone(v)); + }); + return newValue; + } + + @JsonIgnore + public void removeGFlag(String gflagKey, UniverseTaskBase.ServerType... serverTypes) { + if (perProcessFlags != null) { + for (UniverseTaskBase.ServerType serverType : serverTypes) { + perProcessFlags.value.getOrDefault(serverType, new HashMap<>()).remove(gflagKey); + } + } + } + + private PerProcessFlags clone(PerProcessFlags perProcessFlags) { + if (perProcessFlags == null) { + return null; + } + PerProcessFlags result = new PerProcessFlags(); + result.value = new HashMap<>(perProcessFlags.value); + return result; + } + + public static boolean isEmpty(SpecificGFlags specificGFlags) { + if (specificGFlags == null) { + return true; + } + List allGflags = new ArrayList<>(specificGFlags.perAZ.values()); + allGflags.add(specificGFlags.getPerProcessFlags()); + for (PerProcessFlags flags : allGflags) { + if (flags != null && flags.value != null) { + for (Map value : flags.value.values()) { + if (!value.isEmpty()) { + return false; + } + } + } + } + return true; + } + + public static SpecificGFlags construct( + Map masterGFlags, Map tserverGFlags) { + SpecificGFlags result = new SpecificGFlags(); + SpecificGFlags.PerProcessFlags perProcessFlags = new SpecificGFlags.PerProcessFlags(); + perProcessFlags.value = + ImmutableMap.of( + MASTER, masterGFlags, + TSERVER, tserverGFlags); + result.setPerProcessFlags(perProcessFlags); + return result; + } +} diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UniverseCRUDHandler.java b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UniverseCRUDHandler.java index 92f5fde7bcd4..9b97daf53fca 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UniverseCRUDHandler.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UniverseCRUDHandler.java @@ -229,14 +229,14 @@ public void configure(Customer customer, UniverseConfigureTaskParams taskParams) // TODO(Rahul): When we support multiple read only clusters, change clusterType to cluster // uuid. Cluster cluster = getClusterFromTaskParams(taskParams); - UniverseDefinitionTaskParams.UserIntent primaryIntent = cluster.userIntent; + UniverseDefinitionTaskParams.UserIntent userIntent = cluster.userIntent; checkGeoPartitioningParameters(customer, taskParams, OpType.CONFIGURE); - primaryIntent.masterGFlags = trimFlags(primaryIntent.masterGFlags); - primaryIntent.tserverGFlags = trimFlags(primaryIntent.tserverGFlags); - if (StringUtils.isEmpty(primaryIntent.accessKeyCode)) { - primaryIntent.accessKeyCode = appConfig.getString("yb.security.default.access.key"); + userIntent.masterGFlags = trimFlags(userIntent.masterGFlags); + userIntent.tserverGFlags = trimFlags(userIntent.tserverGFlags); + if (StringUtils.isEmpty(userIntent.accessKeyCode)) { + userIntent.accessKeyCode = appConfig.getString("yb.security.default.access.key"); } try { Universe universe = PlacementInfoUtil.getUniverseForParams(taskParams); @@ -1074,6 +1074,8 @@ public UUID createReadReplicaCluster( taskParams.clusters.add(primaryCluster); validateConsistency(primaryCluster, readOnlyCluster); + boolean isCloud = runtimeConfigFactory.forCustomer(customer).getBoolean("yb.cloud.enabled"); + // Set the provider code. Provider provider = Provider.getOrBadRequest(UUID.fromString(readOnlyCluster.userIntent.provider)); diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UpgradeUniverseHandler.java b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UpgradeUniverseHandler.java index 5b47ab370d10..308622b52c3b 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UpgradeUniverseHandler.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/UpgradeUniverseHandler.java @@ -2,14 +2,6 @@ package com.yugabyte.yw.controllers.handlers; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -21,6 +13,7 @@ import com.yugabyte.yw.common.KubernetesManagerFactory; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.Util; +import com.yugabyte.yw.common.YbcManager; import com.yugabyte.yw.common.certmgmt.CertConfigType; import com.yugabyte.yw.common.certmgmt.CertificateHelper; import com.yugabyte.yw.common.config.RuntimeConfigFactory; @@ -46,10 +39,15 @@ import com.yugabyte.yw.models.CustomerTask; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.helpers.TaskType; - +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import lombok.extern.slf4j.Slf4j; import play.mvc.Http.Status; -import com.yugabyte.yw.common.YbcManager; @Slf4j public class UpgradeUniverseHandler { @@ -192,6 +190,7 @@ public JsonNode constructGFlagAuditPayload( if (requestParams.getPrimaryCluster() == null) { return null; } + // TODO: support specific gflags UserIntent newUserIntent = requestParams.getPrimaryCluster().userIntent; Map newMasterGFlags = newUserIntent.masterGFlags; Map newTserverGFlags = newUserIntent.tserverGFlags; diff --git a/managed/src/main/java/com/yugabyte/yw/forms/GFlagsUpgradeParams.java b/managed/src/main/java/com/yugabyte/yw/forms/GFlagsUpgradeParams.java index 3b9aa74236c5..b9415bcef8ca 100644 --- a/managed/src/main/java/com/yugabyte/yw/forms/GFlagsUpgradeParams.java +++ b/managed/src/main/java/com/yugabyte/yw/forms/GFlagsUpgradeParams.java @@ -4,15 +4,24 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.gflags.GFlagsUtil; import com.yugabyte.yw.models.Universe; +import com.yugabyte.yw.models.helpers.NodeDetails; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; import play.mvc.Http.Status; @JsonIgnoreProperties(ignoreUnknown = true) @JsonDeserialize(converter = GFlagsUpgradeParams.Converter.class) +@Slf4j public class GFlagsUpgradeParams extends UpgradeTaskParams { public Map masterGFlags = new HashMap<>(); @@ -26,7 +35,64 @@ public boolean isKubernetesUpgradeSupported() { @Override public void verifyParams(Universe universe) { super.verifyParams(universe); + if (isUsingSpecificGFlags(universe)) { + verifySpecificGFlags(universe); + } else { + verifyParamsOld(universe); + } + } + + private void verifySpecificGFlags(Universe universe) { + Map newClusters = + clusters.stream().collect(Collectors.toMap(c -> c.uuid, c -> c)); + boolean hasClustersToUpdate = false; + for (Cluster curCluster : universe.getUniverseDetails().clusters) { + Cluster newCluster = newClusters.get(curCluster.uuid); + if (newCluster == null + || Objects.equals( + newCluster.userIntent.specificGFlags, curCluster.userIntent.specificGFlags)) { + continue; + } + if (newCluster.clusterType == ClusterType.PRIMARY) { + if (newCluster.userIntent.specificGFlags == null) { + throw new PlatformServiceException( + Status.BAD_REQUEST, "Primary cluster should have non-empty specificGFlags"); + } + if (newCluster.userIntent.specificGFlags.isInheritFromPrimary()) { + throw new PlatformServiceException( + Status.BAD_REQUEST, "Cannot inherit gflags for primary cluster"); + } + } + if (newCluster.userIntent.specificGFlags != null) { + newCluster.userIntent.specificGFlags.validateConsistency(); + } + hasClustersToUpdate = true; + if (upgradeOption == UpgradeOption.NON_RESTART_UPGRADE) { + for (NodeDetails nodeDetails : universe.getNodesInCluster(curCluster.uuid)) { + for (UniverseTaskBase.ServerType serverType : + Arrays.asList( + UniverseTaskBase.ServerType.MASTER, UniverseTaskBase.ServerType.TSERVER)) { + Map newGflags = + GFlagsUtil.getGFlagsForNode( + nodeDetails, serverType, newCluster, newClusters.values()); + Map currentGflags = + GFlagsUtil.getGFlagsForNode( + nodeDetails, serverType, curCluster, universe.getUniverseDetails().clusters); + if (!GFlagsUtil.getDeletedGFlags(currentGflags, newGflags).isEmpty()) { + throw new PlatformServiceException( + Status.BAD_REQUEST, "Cannot delete gFlags through non-restart upgrade option."); + } + } + } + } + } + if (!hasClustersToUpdate) { + throw new PlatformServiceException( + Status.BAD_REQUEST, "No changes in gflags (modify specificGflags in cluster)"); + } + } + private void verifyParamsOld(Universe universe) { UserIntent userIntent = universe.getUniverseDetails().getPrimaryCluster().userIntent; if (masterGFlags.equals(userIntent.masterGFlags) && tserverGFlags.equals(userIntent.tserverGFlags)) { @@ -45,5 +111,46 @@ public void verifyParams(Universe universe) { GFlagsUtil.checkConsistency(masterGFlags, tserverGFlags); } + public Map getNewVersionsOfClusters( + Universe universe) { + boolean usingSpecificGFlags = isUsingSpecificGFlags(universe); + Map clustersFromParams = + clusters.stream().collect(Collectors.toMap(c -> c.uuid, c -> c)); + Map result = new HashMap<>(); + for (UniverseDefinitionTaskParams.Cluster curCluster : universe.getUniverseDetails().clusters) { + Cluster clusterFromParams = clustersFromParams.get(curCluster.uuid); + UniverseDefinitionTaskParams.Cluster newVersion = + new UniverseDefinitionTaskParams.Cluster( + curCluster.clusterType, curCluster.userIntent.clone()); + newVersion.uuid = curCluster.uuid; + if (!usingSpecificGFlags) { + newVersion.userIntent.masterGFlags = this.masterGFlags; + newVersion.userIntent.tserverGFlags = this.tserverGFlags; + } else if (clusterFromParams != null) { + newVersion.userIntent.specificGFlags = clusterFromParams.userIntent.specificGFlags; + } + result.put(newVersion.uuid, newVersion); + } + return result; + } + + /** + * Checks whether current clusters or newly provided clusters have specificGFlags + * + * @param universe + * @return + */ + private boolean isUsingSpecificGFlags(Universe universe) { + boolean result = + Stream.concat(universe.getUniverseDetails().clusters.stream(), this.clusters.stream()) + .filter(c -> c.userIntent.specificGFlags != null) + .findFirst() + .isPresent(); + if (result) { + log.debug("Using specific gflags"); + } + return result; + } + public static class Converter extends BaseConverter {} } diff --git a/managed/src/main/java/com/yugabyte/yw/forms/UniverseDefinitionTaskParams.java b/managed/src/main/java/com/yugabyte/yw/forms/UniverseDefinitionTaskParams.java index 8b3f2da81633..8eed8d7387a7 100644 --- a/managed/src/main/java/com/yugabyte/yw/forms/UniverseDefinitionTaskParams.java +++ b/managed/src/main/java/com/yugabyte/yw/forms/UniverseDefinitionTaskParams.java @@ -19,6 +19,7 @@ import com.yugabyte.yw.common.PlacementInfoUtil; import com.yugabyte.yw.common.PlatformServiceException; import com.yugabyte.yw.common.gflags.GFlagsUtil; +import com.yugabyte.yw.common.gflags.SpecificGFlags; import com.yugabyte.yw.models.AvailabilityZone; import com.yugabyte.yw.models.Provider; import com.yugabyte.yw.models.Region; @@ -306,6 +307,13 @@ public void validate(boolean validateGFlagsConsistency, boolean isAuthEnforced) if (validateGFlagsConsistency) { GFlagsUtil.checkGflagsAndIntentConsistency(userIntent); } + if (userIntent.specificGFlags != null) { + if (clusterType == ClusterType.PRIMARY + && userIntent.specificGFlags.isInheritFromPrimary()) { + throw new IllegalStateException("Cannot inherit gflags for primary cluster"); + } + userIntent.specificGFlags.validateConsistency(); + } } /** @@ -529,6 +537,9 @@ public static class UserIntent { // Device info for dedicated master nodes. @Nullable @ApiModelProperty public DeviceInfo masterDeviceInfo; + // New version of gflags. If present - replaces old masterGFlags/tserverGFlags thing + @ApiModelProperty public SpecificGFlags specificGFlags; + @Override public String toString() { return "UserIntent " @@ -582,6 +593,7 @@ public UserIntent clone() { newUserIntent.accessKeyCode = accessKeyCode; newUserIntent.assignPublicIP = assignPublicIP; newUserIntent.assignStaticPublicIP = assignStaticPublicIP; + newUserIntent.specificGFlags = specificGFlags == null ? null : specificGFlags.clone(); newUserIntent.masterGFlags = new HashMap<>(masterGFlags); newUserIntent.tserverGFlags = new HashMap<>(tserverGFlags); newUserIntent.useTimeSync = useTimeSync; @@ -1026,11 +1038,15 @@ public List getSourceXClusterConfigs() { */ @JsonIgnore public File getSourceRootCertDirPath() { - UniverseDefinitionTaskParams.UserIntent userIntent = getPrimaryCluster().userIntent; + Map masterGflags = + GFlagsUtil.getBaseGFlags(UniverseTaskBase.ServerType.MASTER, getPrimaryCluster(), clusters); + Map tserverGflags = + GFlagsUtil.getBaseGFlags( + UniverseTaskBase.ServerType.TSERVER, getPrimaryCluster(), clusters); String gflagValueOnMasters = - userIntent.masterGFlags.get(XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG); + masterGflags.get(XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG); String gflagValueOnTServers = - userIntent.tserverGFlags.get(XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG); + tserverGflags.get(XClusterConfigTaskBase.SOURCE_ROOT_CERTS_DIR_GFLAG); if (gflagValueOnMasters != null || gflagValueOnTServers != null) { if (!Objects.equals(gflagValueOnMasters, gflagValueOnTServers)) { throw new IllegalStateException( diff --git a/managed/src/main/resources/swagger.json b/managed/src/main/resources/swagger.json index 0ac1cbbb2d93..507d1eb04422 100644 --- a/managed/src/main/resources/swagger.json +++ b/managed/src/main/resources/swagger.json @@ -6233,6 +6233,21 @@ "required" : [ "buildNumber" ], "type" : "object" }, + "PerProcessFlags" : { + "properties" : { + "value" : { + "additionalProperties" : { + "additionalProperties" : { + "type" : "string" + }, + "type" : "object" + }, + "type" : "object" + } + }, + "required" : [ "value" ], + "type" : "object" + }, "PerfAdvisorSettingsFormData" : { "properties" : { "connection_skew_interval_mins" : { @@ -8635,6 +8650,24 @@ "required" : [ "clusters", "creatingUser", "kubernetesUpgradeSupported", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "upgradeOption", "upgradeSystemCatalog", "ybSoftwareVersion" ], "type" : "object" }, + "SpecificGFlags" : { + "properties" : { + "inheritFromPrimary" : { + "type" : "boolean" + }, + "perAZ" : { + "additionalProperties" : { + "$ref" : "#/definitions/PerProcessFlags" + }, + "type" : "object" + }, + "perProcessFlags" : { + "$ref" : "#/definitions/PerProcessFlags" + } + }, + "required" : [ "inheritFromPrimary", "perAZ", "perProcessFlags" ], + "type" : "object" + }, "StateChangeAuditInfo" : { "properties" : { "customerId" : { @@ -10879,6 +10912,9 @@ "format" : "int32", "type" : "integer" }, + "specificGFlags" : { + "$ref" : "#/definitions/SpecificGFlags" + }, "tserverGFlags" : { "additionalProperties" : { "type" : "string" diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgradeTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgradeTest.java index d21b55ed8372..09f3127aa1d1 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgradeTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/upgrade/GFlagsUpgradeTest.java @@ -9,31 +9,44 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase.ServerType; import com.yugabyte.yw.commissioner.tasks.params.NodeTaskParams; +import com.yugabyte.yw.common.ApiUtils; +import com.yugabyte.yw.common.PlacementInfoUtil; import com.yugabyte.yw.common.gflags.GFlagsUtil; +import com.yugabyte.yw.common.gflags.SpecificGFlags; import com.yugabyte.yw.forms.GFlagsUpgradeParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; import com.yugabyte.yw.forms.UpgradeTaskParams.UpgradeOption; +import com.yugabyte.yw.models.AvailabilityZone; +import com.yugabyte.yw.models.Region; import com.yugabyte.yw.models.TaskInfo; import com.yugabyte.yw.models.Universe; +import com.yugabyte.yw.models.helpers.DeviceInfo; +import com.yugabyte.yw.models.helpers.PlacementInfo; import com.yugabyte.yw.models.helpers.TaskType; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -43,8 +56,11 @@ import play.libs.Json; @RunWith(MockitoJUnitRunner.class) +@Slf4j public class GFlagsUpgradeTest extends UpgradeTaskTest { + private int expectedUniverseVersion = 2; + @InjectMocks GFlagsUpgrade gFlagsUpgrade; private static final List ROLLING_UPGRADE_TASK_SEQUENCE_MASTER = @@ -94,12 +110,39 @@ public class GFlagsUpgradeTest extends UpgradeTaskTest { @Before public void setUp() { super.setUp(); - gFlagsUpgrade.setUserTaskUUID(UUID.randomUUID()); } + private UUID addReadReplica() { + UserIntent curIntent = defaultUniverse.getUniverseDetails().getPrimaryCluster().userIntent; + + UniverseDefinitionTaskParams.UserIntent userIntent = + new UniverseDefinitionTaskParams.UserIntent(); + userIntent.numNodes = 3; + userIntent.ybSoftwareVersion = curIntent.ybSoftwareVersion; + userIntent.accessKeyCode = curIntent.accessKeyCode; + userIntent.regionList = ImmutableList.of(region.uuid); + userIntent.providerType = curIntent.providerType; + userIntent.provider = curIntent.provider; + userIntent.deviceInfo = new DeviceInfo(); + userIntent.deviceInfo.numVolumes = 2; + + PlacementInfo pi = new PlacementInfo(); + PlacementInfoUtil.addPlacementZone(az1.uuid, pi, 1, 1, false); + PlacementInfoUtil.addPlacementZone(az2.uuid, pi, 1, 1, false); + PlacementInfoUtil.addPlacementZone(az3.uuid, pi, 1, 1, true); + + defaultUniverse = + Universe.saveDetails( + defaultUniverse.universeUUID, + ApiUtils.mockUniverseUpdaterWithReadReplica(userIntent, pi)); + expectedUniverseVersion++; + + return defaultUniverse.getUniverseDetails().getReadOnlyClusters().get(0).uuid; + } + private TaskInfo submitTask(GFlagsUpgradeParams requestParams) { - return submitTask(requestParams, TaskType.GFlagsUpgrade, commissioner); + return submitTask(requestParams, TaskType.GFlagsUpgrade, commissioner, expectedUniverseVersion); } private TaskInfo submitTask(GFlagsUpgradeParams requestParams, int expectedVersion) { @@ -610,4 +653,334 @@ public void testContradictoryGFlags() { "G-Flag value for 'enable_ysql' is inconsistent " + "between master and tserver ('false' vs 'true')")); } + + @Test + public void testGFlagsUpgradeForRR() { + UUID clusterId = addReadReplica(); + + defaultUniverse = + Universe.saveDetails( + defaultUniverse.universeUUID, + (u) -> { + u.getUniverseDetails().getPrimaryCluster().userIntent.specificGFlags = + new SpecificGFlags(); + }); + expectedUniverseVersion++; + + GFlagsUpgradeParams taskParams = new GFlagsUpgradeParams(); + SpecificGFlags roGFlags = + SpecificGFlags.construct( + ImmutableMap.of("master-flag", "m1"), ImmutableMap.of("tserver-flag", "t1")); + taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; + taskParams.getReadOnlyClusters().get(0).userIntent.specificGFlags = roGFlags; + + TaskInfo taskInfo = submitTask(taskParams); + assertEquals(Success, taskInfo.getTaskState()); + List subTasks = taskInfo.getSubTasks(); + + Set nodeNames = new HashSet<>(); + defaultUniverse + .getUniverseDetails() + .getNodesInCluster(clusterId) + .forEach(node -> nodeNames.add(node.getNodeName())); + + Set changedNodes = new HashSet<>(); + subTasks + .stream() + .filter(t -> t.getTaskType() == TaskType.AnsibleConfigureServers) + .map(t -> t.getTaskDetails().get("nodeName").asText()) + .forEach(changedNodes::add); + assertEquals(nodeNames, changedNodes); + + defaultUniverse = Universe.getOrBadRequest(defaultUniverse.universeUUID); + assertEquals( + new SpecificGFlags(), + defaultUniverse.getUniverseDetails().getPrimaryCluster().userIntent.specificGFlags); + assertEquals( + roGFlags, + defaultUniverse + .getUniverseDetails() + .getReadOnlyClusters() + .get(0) + .userIntent + .specificGFlags); + } + + @Test + public void testGFlagsUpgradeForRRStartInheriting() throws Exception { + UUID clusterId = addReadReplica(); + + defaultUniverse = + Universe.saveDetails( + defaultUniverse.universeUUID, + universe -> { + universe.getUniverseDetails().getPrimaryCluster().userIntent.specificGFlags = + SpecificGFlags.construct( + Collections.emptyMap(), ImmutableMap.of("tserver-flag", "t1")); + + UniverseDefinitionTaskParams.Cluster rr = + universe.getUniverseDetails().getReadOnlyClusters().get(0); + rr.userIntent.specificGFlags = + SpecificGFlags.construct( + ImmutableMap.of("master-flag", "rm1"), + ImmutableMap.of("tserver-flag", "rt1", "tserver-flag2", "t2")); + }); + expectedUniverseVersion++; + + GFlagsUpgradeParams taskParams = new GFlagsUpgradeParams(); + taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; + SpecificGFlags roGFlags = new SpecificGFlags(); + roGFlags.setInheritFromPrimary(true); + taskParams.getReadOnlyClusters().get(0).userIntent.specificGFlags = roGFlags; + + TaskInfo taskInfo = submitTask(taskParams); + assertEquals(Success, taskInfo.getTaskState()); + List subTasks = taskInfo.getSubTasks(); + + try { + Set nodeNames = new HashSet<>(); + defaultUniverse + .getUniverseDetails() + .getNodesInCluster(clusterId) + .forEach(node -> nodeNames.add(node.getNodeName())); + + Set changedNodes = new HashSet<>(); + subTasks + .stream() + .filter(t -> t.getTaskType() == TaskType.AnsibleConfigureServers) + .map(t -> t.getTaskDetails().get("nodeName").asText()) + .forEach(changedNodes::add); + assertEquals(nodeNames, changedNodes); + + assertEquals( + new HashMap<>(Collections.singletonMap("tserver-flag", "t1")), + getGflagsForNode(subTasks, changedNodes.iterator().next(), TSERVER)); + + assertEquals( + Collections.singleton("tserver-flag2"), + getGflagsToRemoveForNode(subTasks, changedNodes.iterator().next(), TSERVER)); + + defaultUniverse = Universe.getOrBadRequest(defaultUniverse.universeUUID); + assertEquals( + roGFlags, + defaultUniverse + .getUniverseDetails() + .getReadOnlyClusters() + .get(0) + .userIntent + .specificGFlags); + } catch (Exception e) { + log.error("", e); + throw e; + } + } + + @Test + public void testGFlagsUpgradePrimaryWithRR() { + addReadReplica(); + + GFlagsUpgradeParams taskParams = new GFlagsUpgradeParams(); + SpecificGFlags specificGFlags = + SpecificGFlags.construct( + ImmutableMap.of("master-flag", "m1"), ImmutableMap.of("tserver-flag", "t1")); + taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; + taskParams.getPrimaryCluster().userIntent.specificGFlags = specificGFlags; + + TaskInfo taskInfo = submitTask(taskParams); + assertEquals(Success, taskInfo.getTaskState()); + List subTasks = taskInfo.getSubTasks(); + + Set changedNodes = new HashSet<>(); + subTasks + .stream() + .filter(t -> t.getTaskType() == TaskType.AnsibleConfigureServers) + .map(t -> t.getTaskDetails().get("nodeName").asText()) + .forEach(changedNodes::add); + assertEquals(6, changedNodes.size()); // all nodes are affected + + defaultUniverse = Universe.getOrBadRequest(defaultUniverse.universeUUID); + assertNull( + defaultUniverse + .getUniverseDetails() + .getReadOnlyClusters() + .get(0) + .userIntent + .specificGFlags); + assertEquals( + specificGFlags, + defaultUniverse.getUniverseDetails().getPrimaryCluster().userIntent.specificGFlags); + } + + @Test + public void testContradictoryGFlagsNewVersion() { + GFlagsUpgradeParams taskParams = new GFlagsUpgradeParams(); + SpecificGFlags specificGFlags = + SpecificGFlags.construct( + ImmutableMap.of(GFlagsUtil.ENABLE_YSQL, "true"), + ImmutableMap.of(GFlagsUtil.ENABLE_YSQL, "false")); + taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; + taskParams.getPrimaryCluster().userIntent.specificGFlags = specificGFlags; + + TaskInfo taskInfo = submitTask(taskParams); + assertEquals(Failure, taskInfo.getTaskState()); + assertThat( + taskInfo.getErrorMessage(), + containsString( + "G-Flag value for 'enable_ysql' is inconsistent " + + "between master and tserver ('true' vs 'false')")); + } + + @Test + public void testSpecificGFlagsNoChanges() { + defaultUniverse = + Universe.saveDetails( + defaultUniverse.universeUUID, + universe -> { + SpecificGFlags specificGFlags = + SpecificGFlags.construct( + ImmutableMap.of("master-flag", "m"), ImmutableMap.of("tserver-flag", "t1")); + universe.getUniverseDetails().getPrimaryCluster().userIntent.specificGFlags = + specificGFlags; + }); + expectedUniverseVersion++; + + GFlagsUpgradeParams taskParams = new GFlagsUpgradeParams(); + taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; + TaskInfo taskInfo = submitTask(taskParams); + assertEquals(Failure, taskInfo.getTaskState()); + assertThat( + taskInfo.getErrorMessage(), + containsString("No changes in gflags (modify specificGflags in cluster)")); + } + + @Test + public void testGflagsDeletedErrorNewVersion() { + GFlagsUpgradeParams taskParams = new GFlagsUpgradeParams(); + taskParams.upgradeOption = UpgradeOption.NON_RESTART_UPGRADE; + Universe.saveDetails( + defaultUniverse.universeUUID, + universe -> { + SpecificGFlags specificGFlags = + SpecificGFlags.construct( + ImmutableMap.of("master-flag", "m"), ImmutableMap.of("tserver-flag", "t1")); + universe.getUniverseDetails().getPrimaryCluster().userIntent.specificGFlags = + specificGFlags; + }); + expectedUniverseVersion++; + + SpecificGFlags specificGFlags = + SpecificGFlags.construct(ImmutableMap.of("master-flag", "m2"), ImmutableMap.of("a", "b")); + taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; + taskParams.getPrimaryCluster().userIntent.specificGFlags = specificGFlags; + + TaskInfo taskInfo = submitTask(taskParams); + assertEquals(Failure, taskInfo.getTaskState()); + assertThat( + taskInfo.getErrorMessage(), + containsString("Cannot delete gFlags through non-restart upgrade option.")); + } + + @Test + public void testPerAZGflags() { + Map nodeToAzUUID = new HashMap<>(); + Region r = Region.create(defaultProvider, "region-1", "PlacementRegion 1", "default-image"); + Universe.saveDetails( + defaultUniverse.universeUUID, + universe -> { + universe + .getUniverseDetails() + .nodeDetailsSet + .forEach( + node -> { + AvailabilityZone az = + AvailabilityZone.createOrThrow( + region, node.cloudInfo.az, node.cloudInfo.az, "subnet-1"); + node.azUuid = az.uuid; + nodeToAzUUID.put(node.nodeName, az.uuid); + }); + }); + expectedUniverseVersion++; + + GFlagsUpgradeParams taskParams = new GFlagsUpgradeParams(); + SpecificGFlags specificGFlags = + SpecificGFlags.construct( + ImmutableMap.of("master-flag", "m1"), ImmutableMap.of("tserver-flag", "t1")); + Map perAZ = new HashMap<>(); + specificGFlags.setPerAZ(perAZ); + List nodeNames = new ArrayList<>(nodeToAzUUID.keySet()); + // First -> redefining existing + String first = nodeNames.get(0); + SpecificGFlags.PerProcessFlags firstPerProcessFlags = new SpecificGFlags.PerProcessFlags(); + firstPerProcessFlags.value = + ImmutableMap.of( + MASTER, ImmutableMap.of("master-flag", "m2"), + TSERVER, ImmutableMap.of("tserver-flag", "t2")); + perAZ.put(nodeToAzUUID.get(first), firstPerProcessFlags); + + // Second -> adding new + String second = nodeNames.get(1); + SpecificGFlags.PerProcessFlags secondPerProcessFlags = new SpecificGFlags.PerProcessFlags(); + secondPerProcessFlags.value = + ImmutableMap.of( + MASTER, ImmutableMap.of("master-flag2", "m2"), + TSERVER, ImmutableMap.of("tserver-flag2", "t2")); + perAZ.put(nodeToAzUUID.get(second), secondPerProcessFlags); + // Third -> no changes + String third = nodeNames.get(2); + taskParams.clusters = defaultUniverse.getUniverseDetails().clusters; + taskParams.getPrimaryCluster().userIntent.specificGFlags = specificGFlags; + + TaskInfo taskInfo = submitTask(taskParams); + List subTasks = taskInfo.getSubTasks(); + assertEquals(Success, taskInfo.getTaskState()); + + assertEquals( + new HashMap<>(Collections.singletonMap("tserver-flag", "t2")), + getGflagsForNode(subTasks, first, TSERVER)); + assertEquals( + new HashMap<>(Collections.singletonMap("master-flag", "m2")), + getGflagsForNode(subTasks, first, MASTER)); + assertEquals( + new HashMap<>(ImmutableMap.of("tserver-flag", "t1", "tserver-flag2", "t2")), + getGflagsForNode(subTasks, second, TSERVER)); + assertEquals( + new HashMap<>(ImmutableMap.of("master-flag", "m1", "master-flag2", "m2")), + getGflagsForNode(subTasks, second, MASTER)); + assertEquals( + new HashMap<>(Collections.singletonMap("tserver-flag", "t1")), + getGflagsForNode(subTasks, third, TSERVER)); + assertEquals( + new HashMap<>(Collections.singletonMap("master-flag", "m1")), + getGflagsForNode(subTasks, third, MASTER)); + } + + private Map getGflagsForNode( + List tasks, String nodeName, ServerType process) { + TaskInfo taskInfo = findGflagsTask(tasks, nodeName, process); + JsonNode gflags = taskInfo.getTaskDetails().get("gflags"); + return Json.fromJson(gflags, Map.class); + } + + private Set getGflagsToRemoveForNode( + List tasks, String nodeName, ServerType process) { + TaskInfo taskInfo = findGflagsTask(tasks, nodeName, process); + ArrayNode gflagsToRemove = (ArrayNode) taskInfo.getTaskDetails().get("gflagsToRemove"); + return Json.fromJson(gflagsToRemove, Set.class); + } + + private TaskInfo findGflagsTask(List tasks, String nodeName, ServerType process) { + return tasks + .stream() + .filter(t -> t.getTaskType() == TaskType.AnsibleConfigureServers) + .filter(t -> t.getTaskDetails().get("nodeName").asText().equals(nodeName)) + .filter( + t -> + t.getTaskDetails() + .get("properties") + .get("processType") + .asText() + .equals(process.toString())) + .findFirst() + .get(); + } } diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/UpgradeUniverseControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/UpgradeUniverseControllerTest.java index 2981529b3cab..e0de673d8057 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/UpgradeUniverseControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/UpgradeUniverseControllerTest.java @@ -96,7 +96,6 @@ import play.inject.guice.GuiceApplicationBuilder; import play.libs.Json; import play.mvc.Result; -import play.test.WithApplication; @RunWith(JUnitParamsRunner.class) public class UpgradeUniverseControllerTest extends PlatformGuiceApplicationBaseTest { From 0ba2a7bc2f7b1f983c4adff272cb6780227325d4 Mon Sep 17 00:00:00 2001 From: Liza Rekadze <77016159+lizayugabyte@users.noreply.github.com> Date: Tue, 28 Feb 2023 11:54:40 -0500 Subject: [PATCH 02/81] [doc] PR for Describe deleting the /opt/yugabyte directory on the YBA node PLAT-7353 (#16106) * Added content * added another directory to delete * added clean up instructions for stable --------- Co-authored-by: aishwarya24 --- .../set-up-cloud-provider/on-premises.md | 25 +++++++++++++------ .../set-up-cloud-provider/on-premises.md | 16 +++++++++--- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md index 8082acbc9ab8..1aeb45bfc05f 100644 --- a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md +++ b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md @@ -507,7 +507,7 @@ As an alternative to setting crontab permissions, you can install systemd-specif 1. Enable the `yugabyte` user to run the following commands as sudo or root: ```sh - yugabyte ALL=(ALL:ALL) NOPASSWD: + yugabyte ALL=(ALL:ALL) NOPASSWD: /bin/systemctl start yb-master, \ /bin/systemctl stop yb-master, \ /bin/systemctl restart yb-master, \ @@ -743,9 +743,9 @@ As an alternative to setting crontab permissions, you can install systemd-specif [Install] WantedBy=timers.target ``` - + `yb-bind_check.service` - + ```properties [Unit] Description=Yugabyte IP bind check @@ -754,11 +754,11 @@ As an alternative to setting crontab permissions, you can install systemd-specif Before=yb-controller.service yb-tserver.service yb-master.service yb-collect_metrics.timer StartLimitInterval=100 StartLimitBurst=10 - + [Path] PathExists=/home/yugabyte/controller/bin/yb-controller-server PathExists=/home/yugabyte/controller/conf/server.conf - + [Service] # Start ExecStart=/home/yugabyte/controller/bin/yb-controller-server \ @@ -771,7 +771,7 @@ As an alternative to setting crontab permissions, you can install systemd-specif # Logs StandardOutput=syslog StandardError=syslog - + [Install] WantedBy=default.target ``` @@ -975,7 +975,7 @@ docker stop replicated-statsd ``` ```sh -docker rm -f replicated replicated-ui replicated-operator \ replicated-premkit replicated-statsd retraced-api retraced-processor \ retraced-cron retraced-nsqd retraced-postgres +docker rm -f replicated replicated-ui replicated-operator replicated-premkit replicated-statsd retraced-api retraced-processor retraced-cron retraced-nsqd retraced-postgres ``` ```sh @@ -991,7 +991,7 @@ yum remove -y replicated replicated-ui replicated-operator ``` ```sh -rm -rf /var/lib/replicated* /etc/replicated* /etc/init/replicated* \ /etc/default/replicated* /etc/systemd/system/replicated* \ /etc/sysconfig/replicated* \ /etc/systemd/system/multi-user.target.wants/replicated* \ /run/replicated* +rm -rf /var/lib/replicated* /etc/replicated* /etc/init/replicated* /etc/default/replicated* /etc/systemd/system/replicated* /etc/sysconfig/replicated* /etc/systemd/system/multi-user.target.wants/replicated* /run/replicated* ``` ```sh @@ -1000,3 +1000,12 @@ yum remove docker-ce rpm -qa | grep -i docker yum remove docker-ce-cli ``` + +Finally, execute the following commands to delete the `/opt/yugabyte` directory on the node to prevent failure if later you decide to install YugabyteDB Anywhere on a node that was previously removed using the preceding instructions: + +```sh +rm -rf /var/lib/containerd +rm -rf /home/replicated +rm -rf /opt/containerd +rm -rf /opt/yugabyte +``` diff --git a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md index 5148a1c0ddb1..62a407b59187 100644 --- a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md +++ b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/set-up-cloud-provider/on-premises.md @@ -506,7 +506,8 @@ As an alternative to setting crontab permissions, you can install systemd-specif 1. Enable the `yugabyte` user to run the following commands as sudo or root: ```sh - yugabyte ALL=(ALL:ALL) NOPASSWD: /bin/systemctl start yb-master, \ + yugabyte ALL=(ALL:ALL) NOPASSWD: + /bin/systemctl start yb-master, \ /bin/systemctl stop yb-master, \ /bin/systemctl restart yb-master, \ /bin/systemctl enable yb-master, \ @@ -936,7 +937,7 @@ docker stop replicated-statsd ``` ```sh -docker rm -f replicated replicated-ui replicated-operator \ replicated-premkit replicated-statsd retraced-api retraced-processor \ retraced-cron retraced-nsqd retraced-postgres +docker rm -f replicated replicated-ui replicated-operator replicated-premkit replicated-statsd retraced-api retraced-processor retraced-cron retraced-nsqd retraced-postgres ``` ```sh @@ -952,7 +953,7 @@ yum remove -y replicated replicated-ui replicated-operator ``` ```sh -rm -rf /var/lib/replicated* /etc/replicated* /etc/init/replicated* \ /etc/default/replicated* /etc/systemd/system/replicated* \ /etc/sysconfig/replicated* \ /etc/systemd/system/multi-user.target.wants/replicated* \ /run/replicated* +rm -rf /var/lib/replicated* /etc/replicated* /etc/init/replicated* /etc/default/replicated* /etc/systemd/system/replicated* /etc/sysconfig/replicated* /etc/systemd/system/multi-user.target.wants/replicated* /run/replicated* ``` ```sh @@ -961,3 +962,12 @@ yum remove docker-ce rpm -qa | grep -i docker yum remove docker-ce-cli ``` + +Finally, execute the following commands to delete the `/opt/yugabyte` directory on the node to prevent failure if later you decide to install YugabyteDB Anywhere on a node that was previously removed using the preceding instructions: + +```sh +rm -rf /var/lib/containerd +rm -rf /home/replicated +rm -rf /opt/containerd +rm -rf /opt/yugabyte +``` From d4d1717c1bfabe2d8feeef6ba36b65ef29e4f154 Mon Sep 17 00:00:00 2001 From: Daniel Shubin Date: Tue, 28 Feb 2023 17:21:56 +0000 Subject: [PATCH 03/81] [PLAT-7492] Made the yba-ctl install dir yba-ctl Summary: this was changed a few weeks ago from /opt/yba-ctl to /opt/ybactl, and it has caused more pain then expected. Reverting that specific change Test Plan: ran install Reviewers: sanketh, muthu Reviewed By: sanketh, muthu Subscribers: jenkins-bot, sanketh, muthu, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23226 --- managed/yba-installer/cmd/init.go | 2 +- managed/yba-installer/common/directory_manager_ybactl.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/managed/yba-installer/cmd/init.go b/managed/yba-installer/cmd/init.go index 2fbdcc74e72d..4a15f8f4dae1 100644 --- a/managed/yba-installer/cmd/init.go +++ b/managed/yba-installer/cmd/init.go @@ -116,7 +116,7 @@ func handleRootCheck(cmdName string) { } } default: - if _, err := os.Stat(common.RootInstallDir); !errors.Is(err, fs.ErrNotExist) { + if _, err := os.Stat(common.YbactlRootInstallDir); !errors.Is(err, fs.ErrNotExist) { // If /opt/yba-ctl/yba-ctl.yml exists, it was put there be root? if !common.HasSudoAccess() { fmt.Println("Please run yba-ctl as root") diff --git a/managed/yba-installer/common/directory_manager_ybactl.go b/managed/yba-installer/common/directory_manager_ybactl.go index 761bbc193b52..b21b1f66b897 100644 --- a/managed/yba-installer/common/directory_manager_ybactl.go +++ b/managed/yba-installer/common/directory_manager_ybactl.go @@ -5,8 +5,8 @@ import ( "path/filepath" ) -// RootInstallDir is the install directory of yba-ctl for root installs -const RootInstallDir string = "/opt/yba-ctl" +// YbactlRootInstallDir is the install directory of yba-ctl for root installs +const YbactlRootInstallDir string = "/opt/yba-ctl" var ycp ybaCtlPaths = newYbaCtlPaths() @@ -52,7 +52,7 @@ func newYbaCtlPaths() ybaCtlPaths { // InstallDir gets the yba-ctl install directory. func (ycp ybaCtlPaths) InstallDir() string { - return filepath.Join(ycp.installBase, "ybactl") + return filepath.Join(ycp.installBase, "yba-ctl") } // LogFile is the yba-ctl log file path. From 6c69fb126e49bcf923ea9468e707b6447c587f65 Mon Sep 17 00:00:00 2001 From: Hari Krishna Sunder Date: Mon, 27 Feb 2023 22:37:43 -0800 Subject: [PATCH 04/81] [#16252] xCluster: Break up AlterUniverseReplication function Summary: Breaking up AlterUniverseReplication into UpdateProducerAddress, RemoveTablesFromReplication, AddTablesToReplication and RenameUniverseReplication Test Plan: All xCluster tests Reviewers: rahuldesirazu, slingam, jhe Reviewed By: jhe Subscribers: ybase, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D23201 --- ent/src/yb/master/catalog_manager.h | 21 +- ent/src/yb/master/xrepl_catalog_manager.cc | 525 +++++++++++---------- 2 files changed, 289 insertions(+), 257 deletions(-) diff --git a/ent/src/yb/master/catalog_manager.h b/ent/src/yb/master/catalog_manager.h index e4f75b618f16..c8498af98cd3 100644 --- a/ent/src/yb/master/catalog_manager.h +++ b/ent/src/yb/master/catalog_manager.h @@ -188,11 +188,24 @@ class CatalogManager : public yb::master::CatalogManager, SnapshotCoordinatorCon AlterUniverseReplicationResponsePB* resp, rpc::RpcContext* rpc); + Status UpdateProducerAddress( + scoped_refptr universe, + const AlterUniverseReplicationRequestPB* req); + + Status RemoveTablesFromReplication( + scoped_refptr universe, + const AlterUniverseReplicationRequestPB* req); + + Status AddTablesToReplication( + scoped_refptr universe, + const AlterUniverseReplicationRequestPB* req, + AlterUniverseReplicationResponsePB* resp, + rpc::RpcContext* rpc); + // Rename an existing Universe Replication. - Status RenameUniverseReplication(scoped_refptr universe, - const AlterUniverseReplicationRequestPB* req, - AlterUniverseReplicationResponsePB* resp, - rpc::RpcContext* rpc); + Status RenameUniverseReplication( + scoped_refptr universe, + const AlterUniverseReplicationRequestPB* req); Status ChangeXClusterRole(const ChangeXClusterRoleRequestPB* req, ChangeXClusterRoleResponsePB* resp, diff --git a/ent/src/yb/master/xrepl_catalog_manager.cc b/ent/src/yb/master/xrepl_catalog_manager.cc index 9a95527e83fe..0aceccbc4d1a 100644 --- a/ent/src/yb/master/xrepl_catalog_manager.cc +++ b/ent/src/yb/master/xrepl_catalog_manager.cc @@ -3372,309 +3372,330 @@ Status CatalogManager::AlterUniverseReplication( req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); } - // Config logic... + Status s; if (req->producer_master_addresses_size() > 0) { - // 'set_master_addresses' - // TODO: Verify the input. Setup an RPC Task, ListTables, ensure same. + s = UpdateProducerAddress(original_ri, req); + } else if (req->producer_table_ids_to_remove_size() > 0) { + s = RemoveTablesFromReplication(original_ri, req); + } else if (req->producer_table_ids_to_add_size() > 0) { + RETURN_NOT_OK(AddTablesToReplication(original_ri, req, resp, rpc)); + } else if (req->has_new_producer_universe_id()) { + s = RenameUniverseReplication(original_ri, req); + } - { - // 1a. Persistent Config: Update the Universe Config for Master. - auto l = original_ri->LockForWrite(); - l.mutable_data()->pb.mutable_producer_master_addresses()->CopyFrom( - req->producer_master_addresses()); + if (!s.ok()) { + return SetupError(resp->mutable_error(), s); + } - // 1b. Persistent Config: Update the Consumer Registry (updates TServers) - auto cluster_config = ClusterConfig(); - auto cl = cluster_config->LockForWrite(); - auto producer_map = cl.mutable_data()->pb.mutable_consumer_registry()->mutable_producer_map(); - auto it = producer_map->find(req->producer_id()); - if (it == producer_map->end()) { - LOG(WARNING) << "Valid Producer Universe not in Consumer Registry: " << req->producer_id(); - return STATUS( - NotFound, "Could not find CDC producer universe", req->ShortDebugString(), - MasterError(MasterErrorPB::OBJECT_NOT_FOUND)); - } - (*it).second.mutable_master_addrs()->CopyFrom(req->producer_master_addresses()); - cl.mutable_data()->pb.set_version(cl.mutable_data()->pb.version() + 1); + CreateXClusterSafeTimeTableAndStartService(); - { - // Need both these updates to be atomic. - auto w = sys_catalog_->NewWriter(leader_ready_term()); - RETURN_NOT_OK( - w->Mutate(QLWriteRequestPB::QL_STMT_UPDATE, original_ri.get(), cluster_config.get())); - RETURN_NOT_OK(CheckStatus( - sys_catalog_->SyncWrite(w.get()), - "Updating universe replication info and cluster config in sys-catalog")); - } - l.Commit(); - cl.Commit(); + return Status::OK(); +} + +Status CatalogManager::UpdateProducerAddress( + scoped_refptr universe, const AlterUniverseReplicationRequestPB* req) { + CHECK_GT(req->producer_master_addresses_size(), 0); + + // TODO: Verify the input. Setup an RPC Task, ListTables, ensure same. + + { + // 1a. Persistent Config: Update the Universe Config for Master. + auto l = universe->LockForWrite(); + l.mutable_data()->pb.mutable_producer_master_addresses()->CopyFrom( + req->producer_master_addresses()); + + // 1b. Persistent Config: Update the Consumer Registry (updates TServers) + auto cluster_config = ClusterConfig(); + auto cl = cluster_config->LockForWrite(); + auto producer_map = cl.mutable_data()->pb.mutable_consumer_registry()->mutable_producer_map(); + auto it = producer_map->find(req->producer_id()); + if (it == producer_map->end()) { + LOG(WARNING) << "Valid Producer Universe not in Consumer Registry: " << req->producer_id(); + return STATUS( + NotFound, "Could not find CDC producer universe", req->ShortDebugString(), + MasterError(MasterErrorPB::OBJECT_NOT_FOUND)); } + it->second.mutable_master_addrs()->CopyFrom(req->producer_master_addresses()); + cl.mutable_data()->pb.set_version(cl.mutable_data()->pb.version() + 1); - // 2. Memory Update: Change cdc_rpc_tasks (Master cache) { - auto result = original_ri->GetOrCreateCDCRpcTasks(req->producer_master_addresses()); - if (!result.ok()) { - return SetupError(resp->mutable_error(), MasterErrorPB::INTERNAL_ERROR, result.status()); - } + // Need both these updates to be atomic. + auto w = sys_catalog_->NewWriter(leader_ready_term()); + RETURN_NOT_OK( + w->Mutate(QLWriteRequestPB::QL_STMT_UPDATE, universe.get(), cluster_config.get())); + RETURN_NOT_OK(CheckStatus( + sys_catalog_->SyncWrite(w.get()), + "Updating universe replication info and cluster config in sys-catalog")); } - } else if (req->producer_table_ids_to_remove_size() > 0) { - // 'remove_table' - auto it = req->producer_table_ids_to_remove(); - std::set table_ids_to_remove(it.begin(), it.end()); - std::set consumer_table_ids_to_remove; - // Filter out any tables that aren't in the existing replication config. - { - auto l = original_ri->LockForRead(); - auto tbl_iter = l->pb.tables(); - std::set existing_tables(tbl_iter.begin(), tbl_iter.end()), filtered_list; - set_intersection( - table_ids_to_remove.begin(), table_ids_to_remove.end(), existing_tables.begin(), - existing_tables.end(), std::inserter(filtered_list, filtered_list.begin())); - filtered_list.swap(table_ids_to_remove); + l.Commit(); + cl.Commit(); + } + + // 2. Memory Update: Change cdc_rpc_tasks (Master cache) + { + auto result = universe->GetOrCreateCDCRpcTasks(req->producer_master_addresses()); + if (!result.ok()) { + return result.status(); } + } - vector streams_to_remove; + return Status::OK(); +} - { - auto l = original_ri->LockForWrite(); - auto cluster_config = ClusterConfig(); +Status CatalogManager::RemoveTablesFromReplication( + scoped_refptr universe, const AlterUniverseReplicationRequestPB* req) { + CHECK_GT(req->producer_table_ids_to_remove_size() , 0); - // 1. Update the Consumer Registry (removes from TServers). - - auto cl = cluster_config->LockForWrite(); - auto pm = cl.mutable_data()->pb.mutable_consumer_registry()->mutable_producer_map(); - auto producer_entry = pm->find(req->producer_id()); - if (producer_entry != pm->end()) { - // Remove the Tables Specified (not part of the key). - auto stream_map = producer_entry->second.mutable_stream_map(); - for (auto& p : *stream_map) { - if (table_ids_to_remove.count(p.second.producer_table_id()) > 0) { - streams_to_remove.push_back(p.first); - // Also fetch the consumer table ids here so we can clean the in-memory maps after. - consumer_table_ids_to_remove.insert(p.second.consumer_table_id()); - } - } - if (streams_to_remove.size() == stream_map->size()) { - // If this ends with an empty Map, disallow and force user to delete. - LOG(WARNING) << "CDC 'remove_table' tried to remove all tables." << req->producer_id(); - return STATUS( - InvalidArgument, - "Cannot remove all tables with alter. Use delete_universe_replication instead.", - req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); - } else if (streams_to_remove.empty()) { - // If this doesn't delete anything, notify the user. - return STATUS( - InvalidArgument, "Removal matched no entries.", req->ShortDebugString(), - MasterError(MasterErrorPB::INVALID_REQUEST)); - } - for (auto& key : streams_to_remove) { - stream_map->erase(stream_map->find(key)); - } - } - cl.mutable_data()->pb.set_version(cl.mutable_data()->pb.version() + 1); + auto it = req->producer_table_ids_to_remove(); + std::set table_ids_to_remove(it.begin(), it.end()); + std::set consumer_table_ids_to_remove; + // Filter out any tables that aren't in the existing replication config. + { + auto l = universe->LockForRead(); + auto tbl_iter = l->pb.tables(); + std::set existing_tables(tbl_iter.begin(), tbl_iter.end()), filtered_list; + set_intersection( + table_ids_to_remove.begin(), table_ids_to_remove.end(), existing_tables.begin(), + existing_tables.end(), std::inserter(filtered_list, filtered_list.begin())); + filtered_list.swap(table_ids_to_remove); + } - // 2. Remove from Master Configs on Producer and Consumer. + vector streams_to_remove; - Status producer_status = Status::OK(); - if (!l->pb.table_streams().empty()) { - // Delete Relevant Table->StreamID mappings on Consumer. - auto table_streams = l.mutable_data()->pb.mutable_table_streams(); - auto validated_tables = l.mutable_data()->pb.mutable_validated_tables(); - for (auto& key : table_ids_to_remove) { - table_streams->erase(table_streams->find(key)); - validated_tables->erase(validated_tables->find(key)); - } - for (int i = 0; i < l.mutable_data()->pb.tables_size(); i++) { - if (table_ids_to_remove.count(l.mutable_data()->pb.tables(i)) > 0) { - l.mutable_data()->pb.mutable_tables()->DeleteSubrange(i, 1); - --i; - } - } - // Delete CDC stream config on the Producer. - auto result = original_ri->GetOrCreateCDCRpcTasks(l->pb.producer_master_addresses()); - if (!result.ok()) { - LOG(ERROR) << "Unable to create cdc rpc task. CDC streams won't be deleted: " << result; - producer_status = STATUS( - InternalError, "Cannot create cdc rpc task.", req->ShortDebugString(), - MasterError(MasterErrorPB::INTERNAL_ERROR)); - } else { - producer_status = (*result)->client()->DeleteCDCStream( - streams_to_remove, true /* force_delete */, req->remove_table_ignore_errors()); - if (!producer_status.ok()) { - std::stringstream os; - std::copy( - streams_to_remove.begin(), streams_to_remove.end(), - std::ostream_iterator(os, ", ")); - LOG(ERROR) << "Unable to delete CDC streams: " << os.str() - << " on producer due to error: " << producer_status - << ". Try setting the ignore-errors option."; - } - } - } + { + auto l = universe->LockForWrite(); + auto cluster_config = ClusterConfig(); - // Currently, due to the sys_catalog write below, atomicity cannot be guaranteed for - // both producer and consumer deletion, and the atomicity of producer is compromised. - if (!producer_status.ok()) { - return SetupError(resp->mutable_error(), producer_status); - } + // 1. Update the Consumer Registry (removes from TServers). - { - // Need both these updates to be atomic. - auto w = sys_catalog_->NewWriter(leader_ready_term()); - auto s = - w->Mutate(QLWriteRequestPB::QL_STMT_UPDATE, original_ri.get(), cluster_config.get()); - if (s.ok()) { - s = sys_catalog_->SyncWrite(w.get()); - } - if (!s.ok()) { - LOG(DFATAL) << "Updating universe replication info and cluster config in sys-catalog " - "failed. However, the deletion of streams on the producer has been issued." - " Please retry the command with the ignore-errors option to make sure that" - " streams are deleted properly on the consumer."; - return SetupError(resp->mutable_error(), s); + auto cl = cluster_config->LockForWrite(); + auto pm = cl.mutable_data()->pb.mutable_consumer_registry()->mutable_producer_map(); + auto producer_entry = pm->find(req->producer_id()); + if (producer_entry != pm->end()) { + // Remove the Tables Specified (not part of the key). + auto stream_map = producer_entry->second.mutable_stream_map(); + for (auto& p : *stream_map) { + if (table_ids_to_remove.count(p.second.producer_table_id()) > 0) { + streams_to_remove.push_back(p.first); + // Also fetch the consumer table ids here so we can clean the in-memory maps after. + consumer_table_ids_to_remove.insert(p.second.consumer_table_id()); } } + if (streams_to_remove.size() == stream_map->size()) { + // If this ends with an empty Map, disallow and force user to delete. + LOG(WARNING) << "CDC 'remove_table' tried to remove all tables." << req->producer_id(); + return STATUS( + InvalidArgument, + "Cannot remove all tables with alter. Use delete_universe_replication instead.", + req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); + } else if (streams_to_remove.empty()) { + // If this doesn't delete anything, notify the user. + return STATUS( + InvalidArgument, "Removal matched no entries.", req->ShortDebugString(), + MasterError(MasterErrorPB::INVALID_REQUEST)); + } + for (auto& key : streams_to_remove) { + stream_map->erase(stream_map->find(key)); + } + } + cl.mutable_data()->pb.set_version(cl.mutable_data()->pb.version() + 1); - l.Commit(); - cl.Commit(); + // 2. Remove from Master Configs on Producer and Consumer. - // Also remove it from the in-memory map of consumer tables. - LockGuard lock(mutex_); - for (const auto& table : consumer_table_ids_to_remove) { - if (xcluster_consumer_tables_to_stream_map_[table].erase(req->producer_id()) < 1) { - LOG(WARNING) << "Failed to remove consumer table from mapping. " - << "table_id: " << table << ": universe_id: " << req->producer_id(); + Status producer_status = Status::OK(); + if (!l->pb.table_streams().empty()) { + // Delete Relevant Table->StreamID mappings on Consumer. + auto table_streams = l.mutable_data()->pb.mutable_table_streams(); + auto validated_tables = l.mutable_data()->pb.mutable_validated_tables(); + for (auto& key : table_ids_to_remove) { + table_streams->erase(table_streams->find(key)); + validated_tables->erase(validated_tables->find(key)); + } + for (int i = 0; i < l.mutable_data()->pb.tables_size(); i++) { + if (table_ids_to_remove.count(l.mutable_data()->pb.tables(i)) > 0) { + l.mutable_data()->pb.mutable_tables()->DeleteSubrange(i, 1); + --i; } - if (xcluster_consumer_tables_to_stream_map_[table].empty()) { - xcluster_consumer_tables_to_stream_map_.erase(table); + } + // Delete CDC stream config on the Producer. + auto result = universe->GetOrCreateCDCRpcTasks(l->pb.producer_master_addresses()); + if (!result.ok()) { + LOG(ERROR) << "Unable to create cdc rpc task. CDC streams won't be deleted: " << result; + producer_status = STATUS( + InternalError, "Cannot create cdc rpc task.", req->ShortDebugString(), + MasterError(MasterErrorPB::INTERNAL_ERROR)); + } else { + producer_status = (*result)->client()->DeleteCDCStream( + streams_to_remove, true /* force_delete */, req->remove_table_ignore_errors()); + if (!producer_status.ok()) { + std::stringstream os; + std::copy( + streams_to_remove.begin(), streams_to_remove.end(), + std::ostream_iterator(os, ", ")); + LOG(ERROR) << "Unable to delete CDC streams: " << os.str() + << " on producer due to error: " << producer_status + << ". Try setting the ignore-errors option."; } } } - } else if (req->producer_table_ids_to_add_size() > 0) { - // 'add_table' - string alter_producer_id = req->producer_id() + ".ALTER"; - // If user passed in bootstrap ids, check that there is a bootstrap id for every table. - if (req->producer_bootstrap_ids_to_add().size() > 0 && - req->producer_table_ids_to_add().size() != req->producer_bootstrap_ids_to_add().size()) { - return STATUS( - InvalidArgument, "Number of bootstrap ids must be equal to number of tables", - req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); - } + // Currently, due to the sys_catalog write below, atomicity cannot be guaranteed for + // both producer and consumer deletion, and the atomicity of producer is compromised. + RETURN_NOT_OK(producer_status); - // Verify no 'alter' command running. - scoped_refptr alter_ri; - { - SharedLock lock(mutex_); - alter_ri = FindPtrOrNull(universe_replication_map_, alter_producer_id); - } { - if (alter_ri != nullptr) { - LOG(INFO) << "Found " << alter_producer_id << "... Removing"; - if (alter_ri->LockForRead()->is_deleted_or_failed()) { - // Delete previous Alter if it's completed but failed. - master::DeleteUniverseReplicationRequestPB delete_req; - delete_req.set_producer_id(alter_ri->id()); - master::DeleteUniverseReplicationResponsePB delete_resp; - Status s = DeleteUniverseReplication(&delete_req, &delete_resp, rpc); - if (!s.ok()) { - if (delete_resp.has_error()) { - resp->mutable_error()->Swap(delete_resp.mutable_error()); - return s; - } - return SetupError(resp->mutable_error(), s); - } - } else { - return STATUS( - InvalidArgument, "Alter for CDC producer currently running", req->ShortDebugString(), - MasterError(MasterErrorPB::INVALID_REQUEST)); - } + // Need both these updates to be atomic. + auto w = sys_catalog_->NewWriter(leader_ready_term()); + auto s = w->Mutate(QLWriteRequestPB::QL_STMT_UPDATE, universe.get(), cluster_config.get()); + if (s.ok()) { + s = sys_catalog_->SyncWrite(w.get()); + } + if (!s.ok()) { + LOG(DFATAL) << "Updating universe replication info and cluster config in sys-catalog " + "failed. However, the deletion of streams on the producer has been issued." + " Please retry the command with the ignore-errors option to make sure that" + " streams are deleted properly on the consumer."; + return s; } } - // Map each table id to its corresponding bootstrap id. - std::unordered_map table_id_to_bootstrap_id; - if (req->producer_bootstrap_ids_to_add().size() > 0) { - for (int i = 0; i < req->producer_table_ids_to_add().size(); i++) { - table_id_to_bootstrap_id[req->producer_table_ids_to_add(i)] = - req->producer_bootstrap_ids_to_add(i); - } + l.Commit(); + cl.Commit(); - // Ensure that table ids are unique. We need to do this here even though - // the same check is performed by SetupUniverseReplication because - // duplicate table ids can cause a bootstrap id entry in table_id_to_bootstrap_id - // to be overwritten. - if (table_id_to_bootstrap_id.size() != - implicit_cast(req->producer_table_ids_to_add().size())) { - return STATUS( - InvalidArgument, - "When providing bootstrap ids, " - "the list of tables must be unique", - req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); + // Also remove it from the in-memory map of consumer tables. + LockGuard lock(mutex_); + for (const auto& table : consumer_table_ids_to_remove) { + if (xcluster_consumer_tables_to_stream_map_[table].erase(req->producer_id()) < 1) { + LOG(WARNING) << "Failed to remove consumer table from mapping. " + << "table_id: " << table << ": universe_id: " << req->producer_id(); + } + if (xcluster_consumer_tables_to_stream_map_[table].empty()) { + xcluster_consumer_tables_to_stream_map_.erase(table); } } + } - // Only add new tables. Ignore tables that are currently being replicated. - auto tid_iter = req->producer_table_ids_to_add(); - std::unordered_set new_tables(tid_iter.begin(), tid_iter.end()); - { - auto l = original_ri->LockForRead(); - for (auto t : l->pb.tables()) { - auto pos = new_tables.find(t); - if (pos != new_tables.end()) { - new_tables.erase(pos); + return Status::OK(); +} + +Status CatalogManager::AddTablesToReplication( + scoped_refptr universe, const AlterUniverseReplicationRequestPB* req, + AlterUniverseReplicationResponsePB* resp, rpc::RpcContext* rpc) { + CHECK_GT(req->producer_table_ids_to_add_size() , 0); + + string alter_producer_id = req->producer_id() + ".ALTER"; + + // If user passed in bootstrap ids, check that there is a bootstrap id for every table. + if (req->producer_bootstrap_ids_to_add().size() > 0 && + req->producer_table_ids_to_add().size() != req->producer_bootstrap_ids_to_add().size()) { + return STATUS( + InvalidArgument, "Number of bootstrap ids must be equal to number of tables", + req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); + } + + // Verify no 'alter' command running. + scoped_refptr alter_ri; + { + SharedLock lock(mutex_); + alter_ri = FindPtrOrNull(universe_replication_map_, alter_producer_id); + } + { + if (alter_ri != nullptr) { + LOG(INFO) << "Found " << alter_producer_id << "... Removing"; + if (alter_ri->LockForRead()->is_deleted_or_failed()) { + // Delete previous Alter if it's completed but failed. + master::DeleteUniverseReplicationRequestPB delete_req; + delete_req.set_producer_id(alter_ri->id()); + master::DeleteUniverseReplicationResponsePB delete_resp; + Status s = DeleteUniverseReplication(&delete_req, &delete_resp, rpc); + if (!s.ok()) { + if (delete_resp.has_error()) { + resp->mutable_error()->Swap(delete_resp.mutable_error()); + return s; + } + return SetupError(resp->mutable_error(), s); } + } else { + return STATUS( + InvalidArgument, "Alter for CDC producer currently running", req->ShortDebugString(), + MasterError(MasterErrorPB::INVALID_REQUEST)); } } - if (new_tables.empty()) { + } + + // Map each table id to its corresponding bootstrap id. + std::unordered_map table_id_to_bootstrap_id; + if (req->producer_bootstrap_ids_to_add().size() > 0) { + for (int i = 0; i < req->producer_table_ids_to_add().size(); i++) { + table_id_to_bootstrap_id[req->producer_table_ids_to_add(i)] = + req->producer_bootstrap_ids_to_add(i); + } + + // Ensure that table ids are unique. We need to do this here even though + // the same check is performed by SetupUniverseReplication because + // duplicate table ids can cause a bootstrap id entry in table_id_to_bootstrap_id + // to be overwritten. + if (table_id_to_bootstrap_id.size() != + implicit_cast(req->producer_table_ids_to_add().size())) { return STATUS( - InvalidArgument, "CDC producer already contains all requested tables", + InvalidArgument, + "When providing bootstrap ids, " + "the list of tables must be unique", req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); } + } - // 1. create an ALTER table request that mirrors the original 'setup_replication'. - master::SetupUniverseReplicationRequestPB setup_req; - master::SetupUniverseReplicationResponsePB setup_resp; - setup_req.set_producer_id(alter_producer_id); - setup_req.mutable_producer_master_addresses()->CopyFrom( - original_ri->LockForRead()->pb.producer_master_addresses()); - for (auto t : new_tables) { - setup_req.add_producer_table_ids(t); - - // Add bootstrap id to request if it exists. - auto bootstrap_id_lookup_result = table_id_to_bootstrap_id.find(t); - if (bootstrap_id_lookup_result != table_id_to_bootstrap_id.end()) { - setup_req.add_producer_bootstrap_ids(bootstrap_id_lookup_result->second); + // Only add new tables. Ignore tables that are currently being replicated. + auto tid_iter = req->producer_table_ids_to_add(); + std::unordered_set new_tables(tid_iter.begin(), tid_iter.end()); + { + auto l = universe->LockForRead(); + for (auto t : l->pb.tables()) { + auto pos = new_tables.find(t); + if (pos != new_tables.end()) { + new_tables.erase(pos); } } + } + if (new_tables.empty()) { + return STATUS( + InvalidArgument, "CDC producer already contains all requested tables", + req->ShortDebugString(), MasterError(MasterErrorPB::INVALID_REQUEST)); + } - // 2. run the 'setup_replication' pipeline on the ALTER Table - Status s = SetupUniverseReplication(&setup_req, &setup_resp, rpc); - if (!s.ok()) { - if (setup_resp.has_error()) { - resp->mutable_error()->Swap(setup_resp.mutable_error()); - return s; - } - return SetupError(resp->mutable_error(), s); - } - // NOTE: ALTER merges back into original after completion. - } else if (req->has_new_producer_universe_id()) { - Status s = RenameUniverseReplication(original_ri, req, resp, rpc); - if (!s.ok()) { - return SetupError(resp->mutable_error(), s); + // 1. create an ALTER table request that mirrors the original 'setup_replication'. + master::SetupUniverseReplicationRequestPB setup_req; + master::SetupUniverseReplicationResponsePB setup_resp; + setup_req.set_producer_id(alter_producer_id); + setup_req.mutable_producer_master_addresses()->CopyFrom( + universe->LockForRead()->pb.producer_master_addresses()); + for (auto t : new_tables) { + setup_req.add_producer_table_ids(t); + + // Add bootstrap id to request if it exists. + auto bootstrap_id_lookup_result = table_id_to_bootstrap_id.find(t); + if (bootstrap_id_lookup_result != table_id_to_bootstrap_id.end()) { + setup_req.add_producer_bootstrap_ids(bootstrap_id_lookup_result->second); } } - CreateXClusterSafeTimeTableAndStartService(); + // 2. run the 'setup_replication' pipeline on the ALTER Table + Status s = SetupUniverseReplication(&setup_req, &setup_resp, rpc); + if (!s.ok()) { + if (setup_resp.has_error()) { + resp->mutable_error()->Swap(setup_resp.mutable_error()); + return s; + } + return SetupError(resp->mutable_error(), s); + } return Status::OK(); } Status CatalogManager::RenameUniverseReplication( - scoped_refptr universe, - const AlterUniverseReplicationRequestPB* req, - AlterUniverseReplicationResponsePB* resp, - rpc::RpcContext* rpc) { + scoped_refptr universe, const AlterUniverseReplicationRequestPB* req) { + CHECK(req->has_new_producer_universe_id()); + const string old_universe_replication_id = universe->id(); const string new_producer_universe_id = req->new_producer_universe_id(); if (old_universe_replication_id == new_producer_universe_id) { @@ -3728,8 +3749,6 @@ Status CatalogManager::RenameUniverseReplication( universe_replication_map_.erase(old_universe_replication_id); } - CreateXClusterSafeTimeTableAndStartService(); - return Status::OK(); } From 4a0ed99e0075ee9fa15b8f55ba06c7f284ffad1f Mon Sep 17 00:00:00 2001 From: yusong-yan Date: Wed, 22 Feb 2023 18:20:16 +0000 Subject: [PATCH 05/81] [#15830] docdb : Split NonRespondingMaster into two tests Summary: After Oleg fixed yb_backup in [[ https://phabricator.dev.yugabyte.com/D22648 | D22648 ]], NonRespondingMaster test should pass either with or without waiting for Master register TServers. Thus, we can split this test into two. One waits for registering TServers, the other one doesn't wait. Test Plan: ./yb_build.sh --cxx-test pgwrapper_pg_mini-test --gtest_filter PgMiniTest.NonRespondingMaster --clang15 -n 100 ./yb_build.sh --cxx-test pgwrapper_pg_mini-test --gtest_filter PgMiniTest.NonRespondingMasterWithTSWaiting --clang15 -n 100 Reviewers: rthallam, oleg Reviewed By: oleg Subscribers: bogdan, ybase Differential Revision: https://phabricator.dev.yugabyte.com/D23098 --- src/yb/yql/pgwrapper/pg_mini-test.cc | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/yb/yql/pgwrapper/pg_mini-test.cc b/src/yb/yql/pgwrapper/pg_mini-test.cc index 3caa1914fef3..27213cc26fb2 100644 --- a/src/yb/yql/pgwrapper/pg_mini-test.cc +++ b/src/yb/yql/pgwrapper/pg_mini-test.cc @@ -102,6 +102,8 @@ namespace yb { namespace pgwrapper { namespace { +YB_STRONGLY_TYPED_BOOL(WaitForTS); + template class TxnHelper { public: @@ -229,7 +231,7 @@ class PgMiniSingleTServerTest : public PgMiniTest { class PgMiniMasterFailoverTest : public PgMiniTest { protected: void ElectNewLeaderAfterShutdown(); - + void TestNonRespondingMaster(WaitForTS wait_for_ts); public: size_t NumMasters() override { return 3; @@ -2977,10 +2979,7 @@ TEST_F(PgMiniTest, YB_DISABLE_TEST_IN_TSAN(CompactionAfterDBDrop)) { ASSERT_LE(new_file_size, base_file_size + 100_KB); } -// Use special mode when non leader master times out all rpcs. -// Then step down master leader and perform backup. -TEST_F_EX(PgMiniTest, YB_DISABLE_TEST_IN_SANITIZERS(NonRespondingMaster), - PgMiniMasterFailoverTest) { +void PgMiniMasterFailoverTest::TestNonRespondingMaster(WaitForTS wait_for_ts) { FLAGS_TEST_timeout_non_leader_master_rpcs = true; tools::TmpDirProvider tmp_dir; @@ -3001,10 +3000,12 @@ TEST_F_EX(PgMiniTest, YB_DISABLE_TEST_IN_SANITIZERS(NonRespondingMaster), return false; }, 10s, "Wait leader change")); - master::TSManager& ts_manager = ASSERT_RESULT(cluster_->GetLeaderMiniMaster())->ts_manager(); - ASSERT_OK(WaitFor([this, &ts_manager]() -> Result { - return ts_manager.GetAllDescriptors().size() == NumTabletServers(); - }, 10s, "Wait all TServers to be registered")); + if (wait_for_ts) { + master::TSManager& ts_manager = ASSERT_RESULT(cluster_->GetLeaderMiniMaster())->ts_manager(); + ASSERT_OK(WaitFor([this, &ts_manager]() -> Result { + return ts_manager.GetAllDescriptors().size() == NumTabletServers(); + }, 10s, "Wait all TServers to be registered")); + } ASSERT_OK(tools::RunBackupCommand( pg_host_port(), cluster_->GetMasterAddresses(), cluster_->GetTserverHTTPAddresses(), @@ -3012,6 +3013,17 @@ TEST_F_EX(PgMiniTest, YB_DISABLE_TEST_IN_SANITIZERS(NonRespondingMaster), "create"})); } +// Use special mode when non leader master times out all rpcs. +// Then step down master leader and perform backup. +TEST_F_EX(PgMiniTest, YB_DISABLE_TEST_IN_SANITIZERS(NonRespondingMaster), + PgMiniMasterFailoverTest) { + TestNonRespondingMaster(WaitForTS::kFalse); +} + +TEST_F_EX(PgMiniTest, YB_DISABLE_TEST_IN_SANITIZERS(NonRespondingMasterWithTSWaiting), + PgMiniMasterFailoverTest) { + TestNonRespondingMaster(WaitForTS::kTrue); +} // The test checks that YSQL doesn't wait for sent RPC response in case of process termination. TEST_F(PgMiniTest, YB_DISABLE_TEST_IN_TSAN(NoWaitForRPCOnTermination)) { From 73175738006845e5ccdcdb784d34af7ebbe22d36 Mon Sep 17 00:00:00 2001 From: Shubham Date: Tue, 28 Feb 2023 19:42:16 +0000 Subject: [PATCH 06/81] [CLOUD-11878] Allow editing of used provider for YBM Summary: [CLOUD-11878] Allow editing of used provider for YBM Test Plan: Editing of used provider should not throw exception Reviewers: sb-yb, tbedi Reviewed By: tbedi Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23204 --- .../CloudProviderApiController.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/CloudProviderApiController.java b/managed/src/main/java/com/yugabyte/yw/controllers/CloudProviderApiController.java index 57013aae7654..7978547b921a 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/CloudProviderApiController.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/CloudProviderApiController.java @@ -20,6 +20,7 @@ import com.yugabyte.yw.commissioner.tasks.CloudBootstrap; import com.yugabyte.yw.models.helpers.TaskType; import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.common.config.RuntimeConfigFactory; import com.yugabyte.yw.controllers.handlers.CloudProviderHandler; import com.yugabyte.yw.forms.EditAccessKeyRotationScheduleParams; import com.yugabyte.yw.forms.PlatformResults; @@ -54,6 +55,7 @@ public class CloudProviderApiController extends AuthenticatedController { @Inject private CloudProviderHandler cloudProviderHandler; + @Inject private RuntimeConfigFactory runtimeConfigFactory; @ApiOperation( value = "List cloud providers", @@ -123,14 +125,19 @@ public Result edit(UUID customerUUID, UUID providerUUID, boolean validate) { Customer customer = Customer.getOrBadRequest(customerUUID); Provider provider = Provider.getOrBadRequest(customerUUID, providerUUID); - long universeCount = provider.getUniverseCount(); - if (universeCount > 0) { - throw new PlatformServiceException( - FORBIDDEN, - String.format( - "There %s %d universe%s using this provider, cannot modify", - universeCount > 1 ? "are" : "is", universeCount, universeCount > 1 ? "s" : "")); + if (!runtimeConfigFactory.globalRuntimeConf().getBoolean("yb.cloud.enabled")) { + // Relaxing the edit provider call for used provider for YBM specific + // use case, as they already rely on edit provider flow. + long universeCount = provider.getUniverseCount(); + if (universeCount > 0) { + throw new PlatformServiceException( + FORBIDDEN, + String.format( + "There %s %d universe%s using this provider, cannot modify", + universeCount > 1 ? "are" : "is", universeCount, universeCount > 1 ? "s" : "")); + } } + JsonNode requestBody = mayBeMassageRequest(request().body().asJson(), true); Provider editProviderReq = formFactory.getFormDataOrBadRequest(requestBody, Provider.class); UUID taskUUID = From 87d49ed339fa85026696d426c1c65ba28f9ad3c2 Mon Sep 17 00:00:00 2001 From: Hari Krishna Sunder Date: Mon, 27 Feb 2023 23:02:35 -0800 Subject: [PATCH 07/81] [#16251] DocDB: Remove enterprise tserver Summary: Move yb::tserver::enterprise::TabletServer functions into yb::tserver::TabletServer and CQLServerEnt into CQLServer Fixes #16251 Test Plan: All jenkins tests Reviewers: bogdan, slingam, amitanand, sergei Reviewed By: sergei Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D23199 --- ent/src/yb/tserver/CMakeLists-include.txt | 32 --- ent/src/yb/tserver/factory.h | 102 -------- ent/src/yb/tserver/tablet_server.h | 77 ------ ent/src/yb/tserver/tablet_server_ent.cc | 241 ------------------ src/yb/integration-tests/cdc_test_util.cc | 2 +- src/yb/integration-tests/twodc-test.cc | 2 +- src/yb/integration-tests/twodc_test_base.cc | 2 +- .../xcluster_safe_time-itest.cc | 4 +- src/yb/tserver/CMakeLists.txt | 18 +- src/yb/tserver/factory.h | 8 +- src/yb/tserver/heartbeater.cc | 10 +- src/yb/tserver/mini_tablet_server.cc | 2 +- ...emote_bootstrap_rocksdb_client-test_ent.cc | 0 ...mote_bootstrap_rocksdb_session-test_ent.cc | 0 src/yb/tserver/tablet_server.cc | 183 ++++++++++++- src/yb/tserver/tablet_server.h | 35 ++- src/yb/tserver/tablet_server_main_impl.cc | 3 +- ...tserver_metrics_heartbeat_data_provider.cc | 2 +- src/yb/yql/cql/cqlserver/cql_server.cc | 29 +++ src/yb/yql/cql/cqlserver/cql_server.h | 13 +- 20 files changed, 266 insertions(+), 499 deletions(-) delete mode 100644 ent/src/yb/tserver/CMakeLists-include.txt delete mode 100644 ent/src/yb/tserver/factory.h delete mode 100644 ent/src/yb/tserver/tablet_server.h delete mode 100644 ent/src/yb/tserver/tablet_server_ent.cc rename {ent/src => src}/yb/tserver/remote_bootstrap_rocksdb_client-test_ent.cc (100%) rename {ent/src => src}/yb/tserver/remote_bootstrap_rocksdb_session-test_ent.cc (100%) diff --git a/ent/src/yb/tserver/CMakeLists-include.txt b/ent/src/yb/tserver/CMakeLists-include.txt deleted file mode 100644 index f6dba56d7ff9..000000000000 --- a/ent/src/yb/tserver/CMakeLists-include.txt +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) YugaByte, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations -# under the License. - -set(YB_SOURCE_DIR ${CMAKE_SOURCE_DIR}/src) -set(YB_CURRENT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(YB_ENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/ent/src) -set(YB_BINARY_DIR ${CMAKE_BINARY_DIR}/src) -string(REPLACE ${CMAKE_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/ent - YB_ENT_CURRENT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) - -set(TSERVER_SRCS_EXTENSIONS - ${YB_ENT_CURRENT_SOURCE_DIR}/tablet_server_ent.cc - PARENT_SCOPE) - -# Additional tests support. -set(YB_ENT_CURRENT_SOURCE_DIR - ${YB_ENT_CURRENT_SOURCE_DIR} - PARENT_SCOPE) - -set(TSERVER_EXTENSIONS_TESTS - remote_bootstrap_rocksdb_session-test_ent - remote_bootstrap_rocksdb_client-test_ent - PARENT_SCOPE) diff --git a/ent/src/yb/tserver/factory.h b/ent/src/yb/tserver/factory.h deleted file mode 100644 index bacd590dd98e..000000000000 --- a/ent/src/yb/tserver/factory.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) YugaByte, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations -// under the License. -// - -#pragma once - -#include - -#include "yb/rpc/rpc_fwd.h" -#include "yb/rpc/secure_stream.h" - -#include "yb/yql/cql/cqlserver/cql_server_options.h" - -#include "yb/server/secure.h" - -#include "yb/tserver/ts_tablet_manager.h" - -#include "yb/util/result.h" - -DECLARE_string(cert_node_filename); - -namespace yb { - -namespace cqlserver { - -class CQLServer; -class CQLServerOptions; - -} - -namespace tserver { - -class TabletServer; -class TabletServerOptions; - -namespace enterprise { - -class CQLServer; -class TabletServer; - -class CQLServerEnt : public cqlserver::CQLServer { - public: - template - explicit CQLServerEnt(Args&&... args) : CQLServer(std::forward(args)...) { - } - - Status ReloadKeysAndCertificates() override { - if (!secure_context_) { - return Status::OK(); - } - - return server::ReloadSecureContextKeysAndCertificates( - secure_context_.get(), - fs_manager_->GetDefaultRootDir(), - server::SecureContextType::kExternal, - options_.HostsString()); - } - - private: - Status SetupMessengerBuilder(rpc::MessengerBuilder* builder) override { - RETURN_NOT_OK(CQLServer::SetupMessengerBuilder(builder)); - if (!FLAGS_cert_node_filename.empty()) { - secure_context_ = VERIFY_RESULT(server::SetupSecureContext( - fs_manager_->GetDefaultRootDir(), - FLAGS_cert_node_filename, - server::SecureContextType::kExternal, - builder)); - } else { - secure_context_ = VERIFY_RESULT(server::SetupSecureContext( - options_.HostsString(), *fs_manager_, server::SecureContextType::kExternal, builder)); - } - return Status::OK(); - } - - std::unique_ptr secure_context_; -}; - -class Factory { - public: - std::unique_ptr CreateTabletServer(const TabletServerOptions& options) { - return std::make_unique(options); - } - - std::unique_ptr CreateCQLServer( - const cqlserver::CQLServerOptions& options, IoService* io, - tserver::TabletServer* tserver) { - return std::make_unique(options, io, tserver); - } -}; - -} // namespace enterprise -} // namespace tserver -} // namespace yb diff --git a/ent/src/yb/tserver/tablet_server.h b/ent/src/yb/tserver/tablet_server.h deleted file mode 100644 index 22716d43d400..000000000000 --- a/ent/src/yb/tserver/tablet_server.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (c) YugaByte, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "../../../../src/yb/tserver/tablet_server.h" - -#include "yb/cdc/cdc_fwd.h" -#include "yb/encryption/encryption_fwd.h" -#include "yb/rpc/rpc_fwd.h" - -namespace yb { - -namespace tserver { -class CDCConsumer; -namespace enterprise { - - -class TabletServer : public yb::tserver::TabletServer { - typedef yb::tserver::TabletServer super; - public: - explicit TabletServer(const TabletServerOptions& opts); - TabletServer(const TabletServer&) = delete; - void operator=(const TabletServer&) = delete; - ~TabletServer(); - - void Shutdown() override; - - encryption::UniverseKeyManager* GetUniverseKeyManager(); - Status SetUniverseKeyRegistry( - const encryption::UniverseKeyRegistryPB& universe_key_registry) override; - Status SetConfigVersionAndConsumerRegistry(int32_t cluster_config_version, - const cdc::ConsumerRegistryPB* consumer_registry); - - int32_t cluster_config_version() const override; - - CDCConsumer* GetCDCConsumer(); - - Status ReloadKeysAndCertificates() override; - std::string GetCertificateDetails() override; - - void RegisterCertificateReloader(CertificateReloader reloader) override; - - // Mark the CDC service as enabled via heartbeat. - Status SetCDCServiceEnabled(); - - protected: - Status RegisterServices() override; - Status SetupMessengerBuilder(rpc::MessengerBuilder* builder) override; - - private: - - Status CreateCDCConsumer() REQUIRES(cdc_consumer_mutex_); - - std::unique_ptr secure_context_; - std::vector certificate_reloaders_; - - // CDC consumer. - mutable std::mutex cdc_consumer_mutex_; - std::unique_ptr cdc_consumer_ GUARDED_BY(cdc_consumer_mutex_); - - // CDC service. - std::shared_ptr cdc_service_; -}; - -} // namespace enterprise -} // namespace tserver -} // namespace yb diff --git a/ent/src/yb/tserver/tablet_server_ent.cc b/ent/src/yb/tserver/tablet_server_ent.cc deleted file mode 100644 index 1670ec29b58b..000000000000 --- a/ent/src/yb/tserver/tablet_server_ent.cc +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) YugaByte, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations -// under the License. - -#include "yb/tserver/tablet_server.h" - -#include "yb/cdc/cdc_service.h" -#include "yb/cdc/cdc_service_context.h" - -#include "yb/encryption/encrypted_file_factory.h" -#include "yb/encryption/header_manager_impl.h" -#include "yb/encryption/universe_key_manager.h" - -#include "yb/rpc/secure_stream.h" - -#include "yb/server/hybrid_clock.h" -#include "yb/server/secure.h" - -#include "yb/rpc/rpc.h" - -#include "yb/tablet/tablet_peer.h" - -#include "../../../../src/yb/tserver/backup_service.h" -#include "yb/tserver/cdc_consumer.h" -#include "yb/tserver/ts_tablet_manager.h" - -#include "yb/util/flags.h" -#include "yb/util/ntp_clock.h" - -#include "yb/rocksutil/rocksdb_encrypted_file_factory.h" - -using std::string; - -DEFINE_UNKNOWN_int32(ts_backup_svc_num_threads, 4, - "Number of RPC worker threads for the TS backup service"); -TAG_FLAG(ts_backup_svc_num_threads, advanced); - -DEFINE_UNKNOWN_int32(ts_backup_svc_queue_length, 50, - "RPC queue length for the TS backup service"); -TAG_FLAG(ts_backup_svc_queue_length, advanced); - -DEFINE_UNKNOWN_int32(xcluster_svc_queue_length, 5000, - "RPC queue length for the xCluster service"); -TAG_FLAG(xcluster_svc_queue_length, advanced); - -DECLARE_string(cert_node_filename); - -namespace yb { -namespace tserver { -namespace enterprise { - -namespace { - -class CDCServiceContextImpl : public cdc::CDCServiceContext { - public: - explicit CDCServiceContextImpl(TabletServer* tablet_server) - : tablet_server_(*tablet_server) { - } - - tablet::TabletPeerPtr LookupTablet(const TabletId& tablet_id) const override { - return tablet_server_.tablet_manager()->LookupTablet(tablet_id); - } - - Result GetTablet(const TabletId& tablet_id) const override { - return tablet_server_.tablet_manager()->GetTablet(tablet_id); - } - - Result GetServingTablet(const TabletId& tablet_id) const override { - return tablet_server_.tablet_manager()->GetServingTablet(tablet_id); - } - - const std::string& permanent_uuid() const override { - return tablet_server_.permanent_uuid(); - } - - std::unique_ptr MakeClientInitializer( - const std::string& client_name, MonoDelta default_timeout) const override { - return std::make_unique( - client_name, default_timeout, tablet_server_.permanent_uuid(), &tablet_server_.options(), - tablet_server_.metric_entity(), tablet_server_.mem_tracker(), tablet_server_.messenger()); - } - - private: - TabletServer& tablet_server_; -}; - -} // namespace - -TabletServer::TabletServer(const TabletServerOptions& opts) - : super(opts) {} - -TabletServer::~TabletServer() { - Shutdown(); -} - -void TabletServer::Shutdown() { - auto cdc_consumer = GetCDCConsumer(); - if (cdc_consumer) { - cdc_consumer->Shutdown(); - } - super::Shutdown(); -} - -Status TabletServer::RegisterServices() { -#if !defined(__APPLE__) - server::HybridClock::RegisterProvider(NtpClock::Name(), [](const std::string&) { - return std::make_shared(); - }); -#endif - - cdc_service_ = std::make_shared( - std::make_unique(this), metric_entity(), - metric_registry()); - - RETURN_NOT_OK(RpcAndWebServerBase::RegisterService( - FLAGS_ts_backup_svc_queue_length, - std::make_unique(tablet_manager_.get(), metric_entity()))); - - RETURN_NOT_OK(RpcAndWebServerBase::RegisterService( - FLAGS_xcluster_svc_queue_length, - cdc_service_)); - - return super::RegisterServices(); -} - -Status TabletServer::SetupMessengerBuilder(rpc::MessengerBuilder* builder) { - RETURN_NOT_OK(super::SetupMessengerBuilder(builder)); - - secure_context_ = VERIFY_RESULT( - server::SetupInternalSecureContext(options_.HostsString(), *fs_manager_, builder)); - - return Status::OK(); -} - -CDCConsumer* TabletServer::GetCDCConsumer() { - std::lock_guard l(cdc_consumer_mutex_); - return cdc_consumer_.get(); -} - -encryption::UniverseKeyManager* TabletServer::GetUniverseKeyManager() { - return opts_.universe_key_manager; -} - -Status TabletServer::SetUniverseKeyRegistry( - const encryption::UniverseKeyRegistryPB& universe_key_registry) { - opts_.universe_key_manager->SetUniverseKeyRegistry(universe_key_registry); - return Status::OK(); -} - -Status TabletServer::CreateCDCConsumer() { - auto is_leader_clbk = [this](const string& tablet_id){ - auto tablet_peer = tablet_manager_->LookupTablet(tablet_id); - if (!tablet_peer) { - return false; - } - return tablet_peer->LeaderStatus() == consensus::LeaderStatus::LEADER_AND_READY; - }; - - cdc_consumer_ = VERIFY_RESULT( - CDCConsumer::Create(std::move(is_leader_clbk), proxy_cache_.get(), this)); - return Status::OK(); -} - -Status TabletServer::SetConfigVersionAndConsumerRegistry(int32_t cluster_config_version, - const cdc::ConsumerRegistryPB* consumer_registry) { - std::lock_guard l(cdc_consumer_mutex_); - - // Only create a cdc consumer if consumer_registry is not null. - if (!cdc_consumer_ && consumer_registry) { - RETURN_NOT_OK(CreateCDCConsumer()); - } - if (cdc_consumer_) { - cdc_consumer_->RefreshWithNewRegistryFromMaster(consumer_registry, cluster_config_version); - } - return Status::OK(); -} - -int32_t TabletServer::cluster_config_version() const { - std::lock_guard l(cdc_consumer_mutex_); - // If no CDC consumer, we will return -1, which will force the master to send the consumer - // registry if one exists. If we receive one, we will create a new CDC consumer in - // SetConsumerRegistry. - if (!cdc_consumer_) { - return -1; - } - return cdc_consumer_->cluster_config_version(); -} - -Status TabletServer::ReloadKeysAndCertificates() { - if (!secure_context_) { - return Status::OK(); - } - - RETURN_NOT_OK(server::ReloadSecureContextKeysAndCertificates( - secure_context_.get(), - fs_manager_->GetDefaultRootDir(), - server::SecureContextType::kInternal, - options_.HostsString())); - - std::lock_guard l(cdc_consumer_mutex_); - if (cdc_consumer_) { - RETURN_NOT_OK(cdc_consumer_->ReloadCertificates()); - } - - for (const auto& reloader : certificate_reloaders_) { - RETURN_NOT_OK(reloader()); - } - - return Status::OK(); -} - -std::string TabletServer::GetCertificateDetails() { - if(!secure_context_) return ""; - - return secure_context_.get()->GetCertificateDetails(); -} - -void TabletServer::RegisterCertificateReloader(CertificateReloader reloader) { - certificate_reloaders_.push_back(std::move(reloader)); -} - -Status TabletServer::SetCDCServiceEnabled() { - if (!cdc_service_) { - LOG(WARNING) << "CDC Service Not Registered"; - } else { - cdc_service_->SetCDCServiceEnabled(); - } - return Status::OK(); -} - -} // namespace enterprise -} // namespace tserver -} // namespace yb diff --git a/src/yb/integration-tests/cdc_test_util.cc b/src/yb/integration-tests/cdc_test_util.cc index e4c8285ed3a4..48150335bf01 100644 --- a/src/yb/integration-tests/cdc_test_util.cc +++ b/src/yb/integration-tests/cdc_test_util.cc @@ -103,7 +103,7 @@ size_t NumProducerTabletsPolled(MiniCluster* cluster) { size_t size = 0; for (const auto& mini_tserver : cluster->mini_tablet_servers()) { size_t new_size = 0; - auto* tserver = dynamic_cast(mini_tserver->server()); + auto* tserver = mini_tserver->server(); tserver::CDCConsumer* cdc_consumer; if (tserver && (cdc_consumer = tserver->GetCDCConsumer()) && mini_tserver->is_started()) { auto tablets_running = cdc_consumer->TEST_producer_tablets_running(); diff --git a/src/yb/integration-tests/twodc-test.cc b/src/yb/integration-tests/twodc-test.cc index 06f1f007035b..da2d198263cc 100644 --- a/src/yb/integration-tests/twodc-test.cc +++ b/src/yb/integration-tests/twodc-test.cc @@ -2574,7 +2574,7 @@ TEST_P(TwoDCTest, TestAlterDDLWithRestarts) { ASSERT_NE(old_ts, new_ts); // Verify that the new Consumer poller had read the ALTER DDL and stopped polling. - auto* tserver = dynamic_cast(new_ts->server()); + auto* tserver = new_ts->server(); CDCConsumer* cdc_consumer; ASSERT_TRUE(tserver && (cdc_consumer = tserver->GetCDCConsumer())); ASSERT_OK(LoggedWaitFor([&]() -> Result { diff --git a/src/yb/integration-tests/twodc_test_base.cc b/src/yb/integration-tests/twodc_test_base.cc index 72ee9f3e19ff..abe1f05dbbb1 100644 --- a/src/yb/integration-tests/twodc_test_base.cc +++ b/src/yb/integration-tests/twodc_test_base.cc @@ -441,7 +441,7 @@ Status TwoDCTestBase::GetCDCStreamForTable( uint32_t TwoDCTestBase::GetSuccessfulWriteOps(MiniCluster* cluster) { uint32_t size = 0; for (const auto& mini_tserver : cluster->mini_tablet_servers()) { - auto* tserver = dynamic_cast(mini_tserver->server()); + auto* tserver = mini_tserver->server(); CDCConsumer* cdc_consumer; if (tserver && (cdc_consumer = tserver->GetCDCConsumer())) { size += cdc_consumer->GetNumSuccessfulWriteRpcs(); diff --git a/src/yb/integration-tests/xcluster_safe_time-itest.cc b/src/yb/integration-tests/xcluster_safe_time-itest.cc index 8116aecab7b2..59d46c04caad 100644 --- a/src/yb/integration-tests/xcluster_safe_time-itest.cc +++ b/src/yb/integration-tests/xcluster_safe_time-itest.cc @@ -423,7 +423,7 @@ class XClusterConsistencyTest : public XClusterYsqlTest { void StoreReadTimes() { uint32_t count = 0; for (const auto& mini_tserver : producer_cluster()->mini_tablet_servers()) { - auto* tserver = dynamic_cast(mini_tserver->server()); + auto* tserver = mini_tserver->server(); auto cdc_service = dynamic_cast( tserver->rpc_server()->TEST_service_pool("yb.cdc.CDCService")->TEST_get_service().get()); @@ -448,7 +448,7 @@ class XClusterConsistencyTest : public XClusterYsqlTest { uint32_t CountTabletsWithNewReadTimes() { uint32_t count = 0; for (const auto& mini_tserver : producer_cluster()->mini_tablet_servers()) { - auto* tserver = dynamic_cast(mini_tserver->server()); + auto* tserver = mini_tserver->server(); auto cdc_service = dynamic_cast( tserver->rpc_server()->TEST_service_pool("yb.cdc.CDCService")->TEST_get_service().get()); diff --git a/src/yb/tserver/CMakeLists.txt b/src/yb/tserver/CMakeLists.txt index 68124c5a8f79..3e5695b7a815 100644 --- a/src/yb/tserver/CMakeLists.txt +++ b/src/yb/tserver/CMakeLists.txt @@ -33,8 +33,6 @@ set(YB_PCH_PREFIX tserver) set(YB_PCH_DEP_LIBS opid_proto yb_test_util) -YB_INCLUDE_EXTENSIONS() - ######################################### # tserver_proto ######################################### @@ -204,8 +202,7 @@ set(TSERVER_SRCS cdc_consumer.cc cdc_poller.cc twodc_output_client.cc - twodc_write_implementations.cc - ${TSERVER_SRCS_EXTENSIONS}) + twodc_write_implementations.cc) set(TSERVER_DEPS protobuf @@ -313,15 +310,6 @@ ADD_YB_TEST(backup_service-test) ADD_YB_TEST(encrypted_sstable-test) YB_TEST_TARGET_LINK_LIBRARIES(encrypted_sstable-test encryption_test_util tserver_test_util tserver) -# ------------------------------------------------------------------------------------------------- -# Tests in the "ent" directory -# ------------------------------------------------------------------------------------------------- - -if(YB_ENT_CURRENT_SOURCE_DIR) - # Set the test source file folder. - set(CMAKE_CURRENT_LIST_DIR ${YB_ENT_CURRENT_SOURCE_DIR}) - foreach(test ${TSERVER_EXTENSIONS_TESTS}) - ADD_YB_TEST(${test}) - endforeach() -endif() +ADD_YB_TEST(remote_bootstrap_rocksdb_session-test_ent) +ADD_YB_TEST(remote_bootstrap_rocksdb_client-test_ent) \ No newline at end of file diff --git a/src/yb/tserver/factory.h b/src/yb/tserver/factory.h index 71333337f9e2..03dce406f462 100644 --- a/src/yb/tserver/factory.h +++ b/src/yb/tserver/factory.h @@ -25,12 +25,8 @@ class Factory { } std::unique_ptr CreateCQLServer( - const cqlserver::CQLServerOptions& options, rpc::IoService* io, - tserver::TabletServer* tserver) { - return std::make_unique( - options, io, tserver, - std::bind(&TSTabletManager::PreserveLocalLeadersOnly, tserver->tablet_manager(), - std::placeholders::_1)); + const cqlserver::CQLServerOptions& options, IoService* io, tserver::TabletServer* tserver) { + return std::make_unique(options, io, tserver); } }; diff --git a/src/yb/tserver/heartbeater.cc b/src/yb/tserver/heartbeater.cc index c9bc585b05bc..6dbfabdbd8bc 100644 --- a/src/yb/tserver/heartbeater.cc +++ b/src/yb/tserver/heartbeater.cc @@ -506,17 +506,17 @@ Status Heartbeater::Thread::TryHeartbeat() { } else { cluster_config_version = resp.cluster_config_version(); } - RETURN_NOT_OK(static_cast(server_)-> - SetConfigVersionAndConsumerRegistry(cluster_config_version, &resp.consumer_registry())); + RETURN_NOT_OK(server_->SetConfigVersionAndConsumerRegistry( + cluster_config_version, &resp.consumer_registry())); } else if (resp.has_cluster_config_version()) { - RETURN_NOT_OK(static_cast(server_)-> - SetConfigVersionAndConsumerRegistry(resp.cluster_config_version(), nullptr)); + RETURN_NOT_OK( + server_->SetConfigVersionAndConsumerRegistry(resp.cluster_config_version(), nullptr)); } // Check whether the cluster is a producer of a CDC stream. if (resp.has_xcluster_enabled_on_producer() && resp.xcluster_enabled_on_producer()) { - RETURN_NOT_OK(static_cast(server_)->SetCDCServiceEnabled()); + RETURN_NOT_OK(server_->SetCDCServiceEnabled()); } // At this point we know resp is a successful heartbeat response from the master so set it as diff --git a/src/yb/tserver/mini_tablet_server.cc b/src/yb/tserver/mini_tablet_server.cc index 797318eaa605..3e92ef765d93 100644 --- a/src/yb/tserver/mini_tablet_server.cc +++ b/src/yb/tserver/mini_tablet_server.cc @@ -145,7 +145,7 @@ Result> MiniTabletServer::CreateMiniTabletServ Status MiniTabletServer::Start() { CHECK(!started_); - std::unique_ptr server(new enterprise::TabletServer(opts_)); + std::unique_ptr server(new TabletServer(opts_)); RETURN_NOT_OK(server->Init()); RETURN_NOT_OK(server->Start()); diff --git a/ent/src/yb/tserver/remote_bootstrap_rocksdb_client-test_ent.cc b/src/yb/tserver/remote_bootstrap_rocksdb_client-test_ent.cc similarity index 100% rename from ent/src/yb/tserver/remote_bootstrap_rocksdb_client-test_ent.cc rename to src/yb/tserver/remote_bootstrap_rocksdb_client-test_ent.cc diff --git a/ent/src/yb/tserver/remote_bootstrap_rocksdb_session-test_ent.cc b/src/yb/tserver/remote_bootstrap_rocksdb_session-test_ent.cc similarity index 100% rename from ent/src/yb/tserver/remote_bootstrap_rocksdb_session-test_ent.cc rename to src/yb/tserver/remote_bootstrap_rocksdb_session-test_ent.cc diff --git a/src/yb/tserver/tablet_server.cc b/src/yb/tserver/tablet_server.cc index 474d3bdbb128..e9711b0706f5 100644 --- a/src/yb/tserver/tablet_server.cc +++ b/src/yb/tserver/tablet_server.cc @@ -62,9 +62,12 @@ #include "yb/rpc/messenger.h" #include "yb/rpc/service_if.h" #include "yb/rpc/yb_rpc.h" +#include "yb/rpc/secure_stream.h" #include "yb/server/rpc_server.h" +#include "yb/server/secure.h" #include "yb/server/webserver.h" +#include "yb/server/hybrid_clock.h" #include "yb/tablet/maintenance_manager.h" #include "yb/tablet/tablet_peer.h" @@ -78,6 +81,11 @@ #include "yb/tserver/ts_tablet_manager.h" #include "yb/tserver/tserver-path-handlers.h" #include "yb/tserver/tserver_service.proxy.h" +#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/backup_service.h" + +#include "yb/cdc/cdc_service.h" +#include "yb/cdc/cdc_service_context.h" #include "yb/util/flags.h" #include "yb/util/logging.h" @@ -87,6 +95,7 @@ #include "yb/util/size_literals.h" #include "yb/util/status.h" #include "yb/util/status_log.h" +#include "yb/util/ntp_clock.h" using std::make_shared; using std::shared_ptr; @@ -168,9 +177,54 @@ DEFINE_UNKNOWN_int32(tserver_yb_client_default_timeout_ms, kTServerYbClientDefau DEFINE_test_flag(bool, select_all_status_tablets, false, ""); +DEFINE_UNKNOWN_int32(ts_backup_svc_num_threads, 4, + "Number of RPC worker threads for the TS backup service"); +TAG_FLAG(ts_backup_svc_num_threads, advanced); + +DEFINE_UNKNOWN_int32(ts_backup_svc_queue_length, 50, + "RPC queue length for the TS backup service"); +TAG_FLAG(ts_backup_svc_queue_length, advanced); + +DEFINE_UNKNOWN_int32(xcluster_svc_queue_length, 5000, + "RPC queue length for the xCluster service"); +TAG_FLAG(xcluster_svc_queue_length, advanced); + +DECLARE_string(cert_node_filename); namespace yb { namespace tserver { +namespace { + +class CDCServiceContextImpl : public cdc::CDCServiceContext { + public: + explicit CDCServiceContextImpl(TabletServer* tablet_server) : tablet_server_(*tablet_server) {} + + tablet::TabletPeerPtr LookupTablet(const TabletId& tablet_id) const override { + return tablet_server_.tablet_manager()->LookupTablet(tablet_id); + } + + Result GetTablet(const TabletId& tablet_id) const override { + return tablet_server_.tablet_manager()->GetTablet(tablet_id); + } + + Result GetServingTablet(const TabletId& tablet_id) const override { + return tablet_server_.tablet_manager()->GetServingTablet(tablet_id); + } + + const std::string& permanent_uuid() const override { return tablet_server_.permanent_uuid(); } + + std::unique_ptr MakeClientInitializer( + const std::string& client_name, MonoDelta default_timeout) const override { + return std::make_unique( + client_name, default_timeout, tablet_server_.permanent_uuid(), &tablet_server_.options(), + tablet_server_.metric_entity(), tablet_server_.mem_tracker(), tablet_server_.messenger()); + } + + private: + TabletServer& tablet_server_; +}; +} // namespace + TabletServer::TabletServer(const TabletServerOptions& opts) : DbServerBase("TabletServer", opts, "yb.tabletserver", server::CreateMemTrackerForServer()), fail_heartbeats_for_tests_(false), @@ -409,6 +463,21 @@ void TabletServer::AutoInitServiceFlags() { } Status TabletServer::RegisterServices() { +#if !defined(__APPLE__) + server::HybridClock::RegisterProvider( + NtpClock::Name(), [](const std::string&) { return std::make_shared(); }); +#endif + + cdc_service_ = std::make_shared( + std::make_unique(this), metric_entity(), metric_registry()); + + RETURN_NOT_OK(RpcAndWebServerBase::RegisterService( + FLAGS_ts_backup_svc_queue_length, + std::make_unique(tablet_manager_.get(), metric_entity()))); + + RETURN_NOT_OK( + RpcAndWebServerBase::RegisterService(FLAGS_xcluster_svc_queue_length, cdc_service_)); + auto tablet_server_service = std::make_shared(this); tablet_server_service_ = tablet_server_service; LOG(INFO) << "yb::tserver::TabletServiceImpl created at " << tablet_server_service.get(); @@ -475,6 +544,11 @@ void TabletServer::Shutdown() { bool expected = true; if (initted_.compare_exchange_strong(expected, false, std::memory_order_acq_rel)) { + auto cdc_consumer = GetCDCConsumer(); + if (cdc_consumer) { + cdc_consumer->Shutdown(); + } + maintenance_manager_->Shutdown(); WARN_NOT_OK(heartbeater_->Stop(), "Failed to stop TS Heartbeat thread"); @@ -523,11 +597,6 @@ bool TabletServer::LeaderAndReady(const TabletId& tablet_id, bool allow_stale) c return peer->LeaderStatus(allow_stale) == consensus::LeaderStatus::LEADER_AND_READY; } -Status TabletServer::SetUniverseKeyRegistry( - const encryption::UniverseKeyRegistryPB& universe_key_registry) { - return Status::OK(); -} - void TabletServer::set_cluster_uuid(const std::string& cluster_uuid) { std::lock_guard l(lock_); cluster_uuid_ = cluster_uuid; @@ -879,5 +948,109 @@ void TabletServer::InvalidatePgTableCache() { } } +Status TabletServer::SetupMessengerBuilder(rpc::MessengerBuilder* builder) { + RETURN_NOT_OK(DbServerBase::SetupMessengerBuilder(builder)); + + secure_context_ = VERIFY_RESULT( + server::SetupInternalSecureContext(options_.HostsString(), *fs_manager_, builder)); + + return Status::OK(); +} + +CDCConsumer* TabletServer::GetCDCConsumer() { + std::lock_guard l(cdc_consumer_mutex_); + return cdc_consumer_.get(); +} + +encryption::UniverseKeyManager* TabletServer::GetUniverseKeyManager() { + return opts_.universe_key_manager; +} + +Status TabletServer::SetUniverseKeyRegistry( + const encryption::UniverseKeyRegistryPB& universe_key_registry) { + opts_.universe_key_manager->SetUniverseKeyRegistry(universe_key_registry); + return Status::OK(); +} + +Status TabletServer::CreateCDCConsumer() { + auto is_leader_clbk = [this](const string& tablet_id) { + auto tablet_peer = tablet_manager_->LookupTablet(tablet_id); + if (!tablet_peer) { + return false; + } + return tablet_peer->LeaderStatus() == consensus::LeaderStatus::LEADER_AND_READY; + }; + + cdc_consumer_ = + VERIFY_RESULT(CDCConsumer::Create(std::move(is_leader_clbk), proxy_cache_.get(), this)); + return Status::OK(); +} + +Status TabletServer::SetConfigVersionAndConsumerRegistry( + int32_t cluster_config_version, const cdc::ConsumerRegistryPB* consumer_registry) { + std::lock_guard l(cdc_consumer_mutex_); + + // Only create a cdc consumer if consumer_registry is not null. + if (!cdc_consumer_ && consumer_registry) { + RETURN_NOT_OK(CreateCDCConsumer()); + } + if (cdc_consumer_) { + cdc_consumer_->RefreshWithNewRegistryFromMaster(consumer_registry, cluster_config_version); + } + return Status::OK(); +} + +int32_t TabletServer::cluster_config_version() const { + std::lock_guard l(cdc_consumer_mutex_); + // If no CDC consumer, we will return -1, which will force the master to send the consumer + // registry if one exists. If we receive one, we will create a new CDC consumer in + // SetConsumerRegistry. + if (!cdc_consumer_) { + return -1; + } + return cdc_consumer_->cluster_config_version(); +} + +Status TabletServer::ReloadKeysAndCertificates() { + if (!secure_context_) { + return Status::OK(); + } + + RETURN_NOT_OK(server::ReloadSecureContextKeysAndCertificates( + secure_context_.get(), + fs_manager_->GetDefaultRootDir(), + server::SecureContextType::kInternal, + options_.HostsString())); + + std::lock_guard l(cdc_consumer_mutex_); + if (cdc_consumer_) { + RETURN_NOT_OK(cdc_consumer_->ReloadCertificates()); + } + + for (const auto& reloader : certificate_reloaders_) { + RETURN_NOT_OK(reloader()); + } + + return Status::OK(); +} + +std::string TabletServer::GetCertificateDetails() { + if (!secure_context_) return ""; + + return secure_context_.get()->GetCertificateDetails(); +} + +void TabletServer::RegisterCertificateReloader(CertificateReloader reloader) { + certificate_reloaders_.push_back(std::move(reloader)); +} + +Status TabletServer::SetCDCServiceEnabled() { + if (!cdc_service_) { + LOG(WARNING) << "CDC Service Not Registered"; + } else { + cdc_service_->SetCDCServiceEnabled(); + } + return Status::OK(); +} } // namespace tserver } // namespace yb diff --git a/src/yb/tserver/tablet_server.h b/src/yb/tserver/tablet_server.h index 357c45e5112b..ee4cdc6ec818 100644 --- a/src/yb/tserver/tablet_server.h +++ b/src/yb/tserver/tablet_server.h @@ -38,8 +38,10 @@ #include #include "yb/consensus/metadata.pb.h" +#include "yb/cdc/cdc_fwd.h" #include "yb/cdc/cdc_consumer.fwd.h" #include "yb/client/client_fwd.h" +#include "yb/rpc/rpc_fwd.h" #include "yb/encryption/encryption_fwd.h" @@ -236,9 +238,7 @@ class TabletServer : public DbServerBase, public TabletServerIf { uint64_t GetSharedMemoryPostgresAuthKey(); // Currently only used by cdc. - virtual int32_t cluster_config_version() const { - return std::numeric_limits::max(); - } + virtual int32_t cluster_config_version() const; client::TransactionPool& TransactionPool() override; @@ -252,7 +252,7 @@ class TabletServer : public DbServerBase, public TabletServerIf { client::LocalTabletFilter CreateLocalTabletFilter() override; - void RegisterCertificateReloader(CertificateReloader reloader) override {} + void RegisterCertificateReloader(CertificateReloader reloader) override; const XClusterSafeTimeMap& GetXClusterSafeTimeMap() const; @@ -264,6 +264,19 @@ class TabletServer : public DbServerBase, public TabletServerIf { Status ListMasterServers(const ListMasterServersRequestPB* req, ListMasterServersResponsePB* resp) const; + encryption::UniverseKeyManager* GetUniverseKeyManager(); + + Status SetConfigVersionAndConsumerRegistry( + int32_t cluster_config_version, const cdc::ConsumerRegistryPB* consumer_registry); + + CDCConsumer* GetCDCConsumer(); + + // Mark the CDC service as enabled via heartbeat. + Status SetCDCServiceEnabled(); + + Status ReloadKeysAndCertificates() override; + std::string GetCertificateDetails() override; + protected: virtual Status RegisterServices(); @@ -276,6 +289,8 @@ class TabletServer : public DbServerBase, public TabletServerIf { MonoDelta default_client_timeout() override; void SetupAsyncClientInit(client::AsyncClientInitialiser* async_client_init) override; + Status SetupMessengerBuilder(rpc::MessengerBuilder* builder) override; + std::atomic initted_{false}; // If true, all heartbeats will be seen as failed. @@ -360,6 +375,18 @@ class TabletServer : public DbServerBase, public TabletServerIf { PgConfigReloader pg_config_reloader_; + Status CreateCDCConsumer() REQUIRES(cdc_consumer_mutex_); + + std::unique_ptr secure_context_; + std::vector certificate_reloaders_; + + // CDC consumer. + mutable std::mutex cdc_consumer_mutex_; + std::unique_ptr cdc_consumer_ GUARDED_BY(cdc_consumer_mutex_); + + // CDC service. + std::shared_ptr cdc_service_; + DISALLOW_COPY_AND_ASSIGN(TabletServer); }; diff --git a/src/yb/tserver/tablet_server_main_impl.cc b/src/yb/tserver/tablet_server_main_impl.cc index f603eebc2fb3..b0560e54c079 100644 --- a/src/yb/tserver/tablet_server_main_impl.cc +++ b/src/yb/tserver/tablet_server_main_impl.cc @@ -57,6 +57,7 @@ #include "yb/tserver/tserver_call_home.h" #include "yb/rpc/io_thread_pool.h" #include "yb/rpc/scheduler.h" +#include "yb/rpc/secure_stream.h" #include "yb/server/skewed_clock.h" #include "yb/server/secure.h" #include "yb/tserver/factory.h" @@ -211,7 +212,7 @@ int TabletServerMain(int argc, char** argv) { tablet_server_options->env = env.get(); tablet_server_options->rocksdb_env = rocksdb_env.get(); tablet_server_options->universe_key_manager = universe_key_manager.get(); - enterprise::Factory factory; + Factory factory; auto server = factory.CreateTabletServer(*tablet_server_options); diff --git a/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc b/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc index b2838c6435f2..669d939a09b1 100644 --- a/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc +++ b/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc @@ -96,7 +96,7 @@ void TServerMetricsHeartbeatDataProvider::DoAddData( } // Report replication errors from the CDC consumer. - auto consumer = down_cast(server()).GetCDCConsumer(); + auto consumer = server().GetCDCConsumer(); if (consumer != nullptr && should_add_replication_status) { const auto tablet_replication_error_map = consumer->GetReplicationErrors(); for (const auto& tablet_kv : tablet_replication_error_map) { diff --git a/src/yb/yql/cql/cqlserver/cql_server.cc b/src/yb/yql/cql/cqlserver/cql_server.cc index 507e13f15e75..255730f75d2b 100644 --- a/src/yb/yql/cql/cqlserver/cql_server.cc +++ b/src/yb/yql/cql/cqlserver/cql_server.cc @@ -27,6 +27,9 @@ #include "yb/tserver/tablet_server_interface.h" +#include "yb/server/secure.h" +#include "yb/rpc/secure_stream.h" + #include "yb/util/flags.h" #include "yb/util/net/dns_resolver.h" #include "yb/util/result.h" @@ -195,5 +198,31 @@ void CQLServer::CQLNodeListRefresh(const boost::system::error_code &ec) { RescheduleTimer(); } +Status CQLServer::ReloadKeysAndCertificates() { + if (!secure_context_) { + return Status::OK(); + } + + return server::ReloadSecureContextKeysAndCertificates( + secure_context_.get(), + fs_manager_->GetDefaultRootDir(), + server::SecureContextType::kExternal, + options_.HostsString()); +} + +Status CQLServer::SetupMessengerBuilder(rpc::MessengerBuilder* builder) { + RETURN_NOT_OK(RpcAndWebServerBase::SetupMessengerBuilder(builder)); + if (!FLAGS_cert_node_filename.empty()) { + secure_context_ = VERIFY_RESULT(server::SetupSecureContext( + fs_manager_->GetDefaultRootDir(), + FLAGS_cert_node_filename, + server::SecureContextType::kExternal, + builder)); + } else { + secure_context_ = VERIFY_RESULT(server::SetupSecureContext( + options_.HostsString(), *fs_manager_, server::SecureContextType::kExternal, builder)); + } + return Status::OK(); +} } // namespace cqlserver } // namespace yb diff --git a/src/yb/yql/cql/cqlserver/cql_server.h b/src/yb/yql/cql/cqlserver/cql_server.h index f72dcb089e76..cb6d4494acaf 100644 --- a/src/yb/yql/cql/cqlserver/cql_server.h +++ b/src/yb/yql/cql/cqlserver/cql_server.h @@ -63,21 +63,26 @@ class CQLServer : public server::RpcAndWebServerBase { boost::asio::io_service* io, tserver::TabletServerIf* tserver); - Status Start(); + Status Start() override; - void Shutdown(); + void Shutdown() override; tserver::TabletServerIf* tserver() const { return tserver_; } + Status ReloadKeysAndCertificates() override; + private: CQLServerOptions opts_; void CQLNodeListRefresh(const boost::system::error_code &e); void RescheduleTimer(); + std::unique_ptr BuildTopologyChangeEvent( + const std::string& event_type, const Endpoint& addr); + Status SetupMessengerBuilder(rpc::MessengerBuilder* builder) override; + boost::asio::deadline_timer timer_; tserver::TabletServerIf* const tserver_; - std::unique_ptr BuildTopologyChangeEvent(const std::string& event_type, - const Endpoint& addr); + std::unique_ptr secure_context_; DISALLOW_COPY_AND_ASSIGN(CQLServer); }; From decb126a204d6b516839f5a6f1a6d54ac39dd814 Mon Sep 17 00:00:00 2001 From: Amitanand Aiyer Date: Fri, 24 Feb 2023 12:00:52 -0800 Subject: [PATCH 08/81] [#11910] Docdb : Ensure that deadline passed in to PauseWritePermits is passed down accordingly. Summary: Ensure that deadline passed in to PauseWritePermits is passed down accordingly. Test Plan: Jenkins Reviewers: rsami, myang Reviewed By: myang Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D23153 --- src/yb/tablet/tablet.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yb/tablet/tablet.cc b/src/yb/tablet/tablet.cc index 293807e6dad5..e09a77b13311 100644 --- a/src/yb/tablet/tablet.cc +++ b/src/yb/tablet/tablet.cc @@ -3314,7 +3314,7 @@ ScopedRWOperationPause Tablet::PauseWritePermits(CoarseTimePoint deadline) { ScopedRWOperation Tablet::GetPermitToWrite(CoarseTimePoint deadline) { TRACE("Acquiring write permit"); auto se = ScopeExit([] { TRACE("Acquiring write permit done"); }); - return ScopedRWOperation(&write_ops_being_submitted_counter_); + return ScopedRWOperation(&write_ops_being_submitted_counter_, deadline); } Result Tablet::StillHasOrphanedPostSplitData() { From 3be7bcb1db4a1f4a4883e180a556af90aecba5ec Mon Sep 17 00:00:00 2001 From: Sahith Kurapati Date: Thu, 16 Feb 2023 19:09:19 +0000 Subject: [PATCH 09/81] [PLAT-5891] Modify the backup_keys.json to store metadata for cloud KMS key Summary: This diff the the master key metadata to the backup metadata file to store the cloud KMS key information. This master key metadata is different for each KMS provider. This information is useful for the user to know what KMS config was used at the time of backup. This is the metadata stored for each KMS provider: * AWS: * CMK_ID * REGION * Azure: * AZU_VAULT_URL_FIELDNAME * AZU_KEY_NAME_FIELDNAME * GCP: * LOCATION_ID_FIELDNAME * KEY_RING_ID_FIELDNAME * CRYPTO_KEY_ID_FIELDNAME * PROJECT_ID * Hashicorp: * HC_VAULT_ADDRESS * HC_VAULT_ENGINE * HC_VAULT_MOUNT_PATH * HC_VAULT_KEY_NAME Test Plan: Tested the following scenarios for both YBC based backup and non-YBC based backup: 1. Create an EAR universe with AWS KMS. Inserted data and took an s3 backup. Verified in the backup file that the AWS KMS specific metadata was persisted. 2. Create an EAR universe with Azure KMS. Inserted data and took an s3 backup. Verified in the backup file that the Azure KMS specific metadata was persisted. 3. Create an EAR universe with Google KMS. Inserted data and took an s3 backup. Verified in the backup file that the Google KMS specific metadata was persisted. 4. Create an EAR universe with Hashicorp KMS. Inserted data and took an s3 backup. Verified in the backup file that the Hashicorp KMS specific metadata was persisted. 5. Tried restoring the backups to same universe / different universe. Worked as usual and as expected. Reviewers: vpatibandla, svarshney Reviewed By: svarshney Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D22980 --- .../common/kms/EncryptionAtRestManager.java | 32 ++++- .../yw/common/kms/services/AwsEARService.java | 20 +++ .../yw/common/kms/services/AzuEARService.java | 29 ++++- .../kms/services/EncryptionAtRestService.java | 2 + .../yw/common/kms/services/GcpEARService.java | 29 +++++ .../kms/services/HashicorpEARService.java | 21 ++++ .../kms/services/SmartKeyEARService.java | 10 ++ .../yw/common/kms/util/AwsEARServiceUtil.java | 26 ++-- .../yw/common/kms/util/AzuEARServiceUtil.java | 100 ++++++++++----- .../yw/common/kms/util/GcpEARServiceUtil.java | 119 ++++++++++++------ .../kms/util/HashicorpEARServiceUtil.java | 9 ++ .../EncryptionAtRestController.java | 56 +++++---- .../com/yugabyte/yw/models/KmsHistory.java | 12 ++ .../com/yugabyte/yw/common/TestHelper.java | 54 ++++++++ .../kms/services/AwsEARServiceTest.java | 65 +++++++--- .../services/EncryptionAtRestServiceTest.java | 6 + .../kms/util/AzuEARServiceUtilTest.java | 14 ++- .../EncryptionAtRestControllerTest.java | 17 +-- 18 files changed, 484 insertions(+), 137 deletions(-) diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/EncryptionAtRestManager.java b/managed/src/main/java/com/yugabyte/yw/common/kms/EncryptionAtRestManager.java index 8f460ad29439..e72729b220c5 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/EncryptionAtRestManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/EncryptionAtRestManager.java @@ -24,11 +24,13 @@ import com.yugabyte.yw.common.kms.util.KeyProvider; import com.yugabyte.yw.forms.EncryptionAtRestConfig; import com.yugabyte.yw.models.KmsConfig; +import com.yugabyte.yw.models.KmsHistory; import java.io.File; import java.lang.reflect.Constructor; import java.nio.file.Files; import java.util.List; import java.util.Objects; +import java.util.Set; import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -203,13 +205,38 @@ public List getUniverseKeyRefsForBackup(UUID universeUUID) { .collect(Collectors.toList()); } + public void addUniverseKeyMasterKeyMetadata( + ObjectNode backup, + List universeKeyRefs, + ArrayNode universeKeys, + UUID universeUUID) { + // Add all the universe key history. + universeKeyRefs.forEach(universeKeys::add); + // Add the master key metadata. + Set distinctKmsConfigUUIDs = KmsHistory.getDistinctKmsConfigUUIDs(universeUUID); + if (distinctKmsConfigUUIDs.size() == 1) { + KmsConfig kmsConfig = KmsConfig.get(distinctKmsConfigUUIDs.iterator().next()); + backup.set( + "master_key_metadata", + kmsConfig.keyProvider.getServiceInstance().getKeyMetadata(kmsConfig.configUUID)); + } else { + LOG.debug( + "Found {} master keys on universe '{}''. Not adding them to backup metadata: {}.", + distinctKmsConfigUUIDs.size(), + universeUUID, + distinctKmsConfigUUIDs.toString()); + } + } + // Backup universe key metadata to file public void backupUniverseKeyHistory(UUID universeUUID, String storageLocation) throws Exception { ObjectNode backup = Json.newObject(); ArrayNode universeKeys = backup.putArray("universe_keys"); List universeKeyRefs = getUniverseKeyRefsForBackup(universeUUID); if (universeKeyRefs.size() > 0) { - universeKeyRefs.forEach(universeKeys::add); + // Add the universe key details and master key details. + addUniverseKeyMasterKeyMetadata(backup, universeKeyRefs, universeKeys, universeUUID); + // Write the backup metadata object to file. ObjectMapper mapper = new ObjectMapper(); String backupContent = mapper.writeValueAsString(backup); File backupKeysFile = EncryptionAtRestUtil.getUniverseBackupKeysFile(storageLocation); @@ -241,7 +268,8 @@ public ObjectNode backupUniverseKeyHistory(UUID universeUUID) throws Exception { ArrayNode universeKeys = backup.putArray("universe_keys"); List universeKeyRefs = getUniverseKeyRefsForBackup(universeUUID); if (universeKeyRefs.size() > 0) { - universeKeyRefs.forEach(universeKeys::add); + // Add the universe key details and master key details. + addUniverseKeyMasterKeyMetadata(backup, universeKeyRefs, universeKeys, universeUUID); return backup; } return null; diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/services/AwsEARService.java b/managed/src/main/java/com/yugabyte/yw/common/kms/services/AwsEARService.java index d18430239c49..7f00b4f950b6 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/services/AwsEARService.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/services/AwsEARService.java @@ -10,17 +10,20 @@ package com.yugabyte.yw.common.kms.services; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.inject.Inject; import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.config.UniverseConfKeys; import com.yugabyte.yw.common.kms.algorithms.AwsAlgorithm; import com.yugabyte.yw.common.kms.util.AwsEARServiceUtil; +import com.yugabyte.yw.common.kms.util.EncryptionAtRestUtil; import com.yugabyte.yw.common.kms.util.KeyProvider; import com.yugabyte.yw.common.kms.util.AwsEARServiceUtil.AwsKmsAuthConfigField; import com.yugabyte.yw.forms.EncryptionAtRestConfig; import com.yugabyte.yw.models.Universe; +import java.util.List; import java.util.UUID; /** @@ -189,4 +192,21 @@ protected void cleanupWithService(UUID universeUUID, UUID configUUID) { } } } + + @Override + public ObjectNode getKeyMetadata(UUID configUUID) { + // Get all the auth config fields marked as metadata. + List awsKmsMetadataFields = AwsKmsAuthConfigField.getMetadataFields(); + ObjectNode authConfig = EncryptionAtRestUtil.getAuthConfig(configUUID); + ObjectNode keyMetadata = new ObjectMapper().createObjectNode(); + + for (String fieldName : awsKmsMetadataFields) { + if (authConfig.has(fieldName)) { + keyMetadata.set(fieldName, authConfig.get(fieldName)); + } + } + // Add key_provider field. + keyMetadata.put("key_provider", KeyProvider.AWS.name()); + return keyMetadata; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/services/AzuEARService.java b/managed/src/main/java/com/yugabyte/yw/common/kms/services/AzuEARService.java index 16df71ee0169..f1d3064c8545 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/services/AzuEARService.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/services/AzuEARService.java @@ -11,12 +11,17 @@ package com.yugabyte.yw.common.kms.services; +import java.util.List; import java.util.UUID; + +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.kms.algorithms.AzuAlgorithm; import com.yugabyte.yw.common.kms.util.AzuEARServiceUtil; +import com.yugabyte.yw.common.kms.util.EncryptionAtRestUtil; import com.yugabyte.yw.common.kms.util.KeyProvider; +import com.yugabyte.yw.common.kms.util.AzuEARServiceUtil.AzuKmsAuthConfigField; import com.yugabyte.yw.forms.EncryptionAtRestConfig; /** @@ -37,7 +42,7 @@ public boolean validateKeyAlgorithmAndSize(ObjectNode authConfig) { // Checks if authConfig has valid key algorithm and key size as specified in AzuAlgorithm.java String keyAlgorithm = azuEARServiceUtil.getConfigFieldValue( - authConfig, AzuEARServiceUtil.AZU_KEY_ALGORITHM_FIELDNAME); + authConfig, AzuKmsAuthConfigField.AZU_KEY_ALGORITHM.fieldName); int keySize = azuEARServiceUtil.getConfigKeySize(authConfig); AzuAlgorithm azuAlgorithm = validateEncryptionAlgorithm(keyAlgorithm); if (azuAlgorithm == null) { @@ -72,7 +77,7 @@ protected ObjectNode createAuthConfigWithService(UUID configUUID, ObjectNode con String.format( "Key vault or the credentials are invalid. key vault url = '%s'", azuEARServiceUtil.getConfigFieldValue( - config, AzuEARServiceUtil.AZU_VAULT_URL_FIELDNAME)); + config, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName)); LOG.error(errMsg); throw new RuntimeException(errMsg); } @@ -81,7 +86,8 @@ protected ObjectNode createAuthConfigWithService(UUID configUUID, ObjectNode con // If key exists, it is validated before usage // Else, a new master key is created String keyName = - azuEARServiceUtil.getConfigFieldValue(config, AzuEARServiceUtil.AZU_KEY_NAME_FIELDNAME); + azuEARServiceUtil.getConfigFieldValue( + config, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName); boolean checkKeyExists = azuEARServiceUtil.checkKeyExists(config, keyName); if (checkKeyExists) { if (azuEARServiceUtil.validateKeySettings(config, keyName)) { @@ -194,4 +200,21 @@ protected byte[] validateRetrieveKeyWithService( protected void cleanupWithService(UUID universeUUID, UUID configUUID) { // Do nothing to KMS when deleting universe with EAR enabled } + + @Override + public ObjectNode getKeyMetadata(UUID configUUID) { + // Get all the auth config fields marked as metadata. + List azuKmsMetadataFields = AzuKmsAuthConfigField.getMetadataFields(); + ObjectNode authConfig = EncryptionAtRestUtil.getAuthConfig(configUUID); + ObjectNode keyMetadata = new ObjectMapper().createObjectNode(); + + for (String fieldName : azuKmsMetadataFields) { + if (authConfig.has(fieldName)) { + keyMetadata.set(fieldName, authConfig.get(fieldName)); + } + } + // Add key_provider field. + keyMetadata.put("key_provider", KeyProvider.AZU.name()); + return keyMetadata; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestService.java b/managed/src/main/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestService.java index c1af0960569c..bff5fa550dc1 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestService.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestService.java @@ -324,4 +324,6 @@ public void restoreBackupEntry(UUID universeUUID, UUID configUUID, byte[] keyRef public void refreshService(UUID configUUID) { // Do Nothing - optionally override sub classes when required. } + + public abstract ObjectNode getKeyMetadata(UUID configUUID); } diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/services/GcpEARService.java b/managed/src/main/java/com/yugabyte/yw/common/kms/services/GcpEARService.java index c0045bd27fcb..148304c63afa 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/services/GcpEARService.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/services/GcpEARService.java @@ -11,13 +11,17 @@ package com.yugabyte.yw.common.kms.services; +import java.util.List; import java.util.UUID; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.kms.algorithms.GcpAlgorithm; +import com.yugabyte.yw.common.kms.util.EncryptionAtRestUtil; import com.yugabyte.yw.common.kms.util.GcpEARServiceUtil; import com.yugabyte.yw.common.kms.util.KeyProvider; +import com.yugabyte.yw.common.kms.util.GcpEARServiceUtil.GcpKmsAuthConfigField; import com.yugabyte.yw.forms.EncryptionAtRestConfig; import com.yugabyte.yw.models.KmsConfig; @@ -189,4 +193,29 @@ protected byte[] validateRetrieveKeyWithService( protected void cleanupWithService(UUID universeUUID, UUID configUUID) { // Do nothing to KMS when deleting universe with EAR enabled } + + @Override + public ObjectNode getKeyMetadata(UUID configUUID) { + // Get all the auth config fields marked as metadata. + List gcpKmsMetadataFields = GcpKmsAuthConfigField.getMetadataFields(); + ObjectNode authConfig = EncryptionAtRestUtil.getAuthConfig(configUUID); + ObjectNode keyMetadata = new ObjectMapper().createObjectNode(); + + for (String fieldName : gcpKmsMetadataFields) { + if (authConfig.has(fieldName)) { + keyMetadata.set(fieldName, authConfig.get(fieldName)); + } + } + // Add the GCP project ID to the key metadata as well. + // This is useful info to the user. + if (authConfig.has(GcpKmsAuthConfigField.GCP_CONFIG.fieldName) + && authConfig.get(GcpKmsAuthConfigField.GCP_CONFIG.fieldName).has("project_id")) { + keyMetadata.set( + "project_id", + authConfig.get(GcpKmsAuthConfigField.GCP_CONFIG.fieldName).get("project_id")); + } + // Add key_provider field. + keyMetadata.put("key_provider", KeyProvider.GCP.name()); + return keyMetadata; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/services/HashicorpEARService.java b/managed/src/main/java/com/yugabyte/yw/common/kms/services/HashicorpEARService.java index 2ccf9447b58e..fc4a609c8351 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/services/HashicorpEARService.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/services/HashicorpEARService.java @@ -11,16 +11,19 @@ package com.yugabyte.yw.common.kms.services; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.kms.algorithms.HashicorpVaultAlgorithm; import com.yugabyte.yw.common.kms.util.hashicorpvault.HashicorpVaultConfigParams; +import com.yugabyte.yw.common.kms.util.EncryptionAtRestUtil; import com.yugabyte.yw.common.kms.util.HashicorpEARServiceUtil; import com.yugabyte.yw.common.kms.util.KeyProvider; import com.yugabyte.yw.common.kms.util.hashicorpvault.VaultSecretEngineBase; import com.yugabyte.yw.forms.EncryptionAtRestConfig; import com.yugabyte.yw.models.KmsConfig; import java.util.UUID; +import java.util.Arrays; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -215,4 +218,22 @@ public void refreshService(UUID configUUID) { throw new RuntimeException(errMsg, e); } } + + @Override + public ObjectNode getKeyMetadata(UUID configUUID) { + ObjectNode authConfig = EncryptionAtRestUtil.getAuthConfig(configUUID); + ObjectNode keyMetadata = new ObjectMapper().createObjectNode(); + // All the hashicorp metadata fields. + List metadataFields = HashicorpEARServiceUtil.getMetadataFields(); + + // Add all the metadata fields. + for (String fieldName : metadataFields) { + if (authConfig.has(fieldName)) { + keyMetadata.set(fieldName, authConfig.get(fieldName)); + } + } + // Add key_provider field. + keyMetadata.put("key_provider", KeyProvider.HASHICORP.name()); + return keyMetadata; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/services/SmartKeyEARService.java b/managed/src/main/java/com/yugabyte/yw/common/kms/services/SmartKeyEARService.java index 7aadde758e38..38603afc6ffa 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/services/SmartKeyEARService.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/services/SmartKeyEARService.java @@ -11,6 +11,7 @@ package com.yugabyte.yw.common.kms.services; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; @@ -167,4 +168,13 @@ public byte[] validateRetrieveKeyWithService( keyVal = Base64.getDecoder().decode(response.get("value").asText()); return keyVal; } + + @Override + public ObjectNode getKeyMetadata(UUID configUUID) { + ObjectNode keyMetadata = new ObjectMapper().createObjectNode(); + + // Add key_provider field. + keyMetadata.put("key_provider", KeyProvider.SMARTKEY.name()); + return keyMetadata; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/util/AwsEARServiceUtil.java b/managed/src/main/java/com/yugabyte/yw/common/kms/util/AwsEARServiceUtil.java index dae31859408d..d094ceefc1cd 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/util/AwsEARServiceUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/util/AwsEARServiceUtil.java @@ -61,21 +61,23 @@ public class AwsEARServiceUtil { private static final String CMK_POLICY = "default_cmk_policy.json"; - // All fields in authConfig object sent from UI + // All fields in AWS KMS authConfig object sent from UI public enum AwsKmsAuthConfigField { - ACCESS_KEY_ID("AWS_ACCESS_KEY_ID", true), - SECRET_ACCESS_KEY("AWS_SECRET_ACCESS_KEY", true), - ENDPOINT("AWS_KMS_ENDPOINT", true), - CMK_POLICY("cmk_policy", true), - REGION("AWS_REGION", false), - CMK_ID("cmk_id", false); + ACCESS_KEY_ID("AWS_ACCESS_KEY_ID", true, false), + SECRET_ACCESS_KEY("AWS_SECRET_ACCESS_KEY", true, false), + ENDPOINT("AWS_KMS_ENDPOINT", true, false), + CMK_POLICY("cmk_policy", true, false), + REGION("AWS_REGION", false, true), + CMK_ID("cmk_id", false, true); public final String fieldName; public final boolean isEditable; + public final boolean isMetadata; - AwsKmsAuthConfigField(String fieldName, boolean isEditable) { + AwsKmsAuthConfigField(String fieldName, boolean isEditable, boolean isMetadata) { this.fieldName = fieldName; this.isEditable = isEditable; + this.isMetadata = isMetadata; } public static List getEditableFields() { @@ -93,6 +95,14 @@ public static List getNonEditableFields() { .map(configField -> configField.fieldName) .collect(Collectors.toList()); } + + public static List getMetadataFields() { + return Arrays.asList(values()) + .stream() + .filter(configField -> configField.isMetadata) + .map(configField -> configField.fieldName) + .collect(Collectors.toList()); + } } private static final Logger LOG = LoggerFactory.getLogger(AwsEARServiceUtil.class); diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtil.java b/managed/src/main/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtil.java index bc77335b133c..d86aea83ab6e 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtil.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Random; import java.util.UUID; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import com.azure.core.credential.TokenCredential; import com.azure.core.exception.ResourceNotFoundException; @@ -40,13 +41,50 @@ @Slf4j public class AzuEARServiceUtil { - public static final String CLIENT_ID_FIELDNAME = "CLIENT_ID"; - public static final String CLIENT_SECRET_FIELDNAME = "CLIENT_SECRET"; - public static final String TENANT_ID_FIELDNAME = "TENANT_ID"; - public static final String AZU_VAULT_URL_FIELDNAME = "AZU_VAULT_URL"; - public static final String AZU_KEY_NAME_FIELDNAME = "AZU_KEY_NAME"; - public static final String AZU_KEY_ALGORITHM_FIELDNAME = "AZU_KEY_ALGORITHM"; - public static final String AZU_KEY_SIZE_FIELDNAME = "AZU_KEY_SIZE"; + // All fields in Azure KMS authConfig object sent from UI + public enum AzuKmsAuthConfigField { + CLIENT_ID("CLIENT_ID", true, false), + CLIENT_SECRET("CLIENT_SECRET", true, false), + TENANT_ID("TENANT_ID", true, false), + AZU_VAULT_URL("AZU_VAULT_URL", false, true), + AZU_KEY_NAME("AZU_KEY_NAME", false, true), + AZU_KEY_ALGORITHM("AZU_KEY_ALGORITHM", false, false), + AZU_KEY_SIZE("AZU_KEY_SIZE", false, false); + + public final String fieldName; + public final boolean isEditable; + public final boolean isMetadata; + + AzuKmsAuthConfigField(String fieldName, boolean isEditable, boolean isMetadata) { + this.fieldName = fieldName; + this.isEditable = isEditable; + this.isMetadata = isMetadata; + } + + public static List getEditableFields() { + return Arrays.asList(values()) + .stream() + .filter(configField -> configField.isEditable) + .map(configField -> configField.fieldName) + .collect(Collectors.toList()); + } + + public static List getNonEditableFields() { + return Arrays.asList(values()) + .stream() + .filter(configField -> !configField.isEditable) + .map(configField -> configField.fieldName) + .collect(Collectors.toList()); + } + + public static List getMetadataFields() { + return Arrays.asList(values()) + .stream() + .filter(configField -> configField.isMetadata) + .map(configField -> configField.fieldName) + .collect(Collectors.toList()); + } + } public ObjectNode getAuthConfig(UUID configUUID) { return EncryptionAtRestUtil.getAuthConfig(configUUID); @@ -60,9 +98,10 @@ public ObjectNode getAuthConfig(UUID configUUID) { * @return the token credentials object */ public TokenCredential getCredentials(ObjectNode authConfig) { - String clientId = getConfigFieldValue(authConfig, CLIENT_ID_FIELDNAME); - String clientSecret = getConfigFieldValue(authConfig, CLIENT_SECRET_FIELDNAME); - String tenantId = getConfigFieldValue(authConfig, TENANT_ID_FIELDNAME); + String clientId = getConfigFieldValue(authConfig, AzuKmsAuthConfigField.CLIENT_ID.fieldName); + String clientSecret = + getConfigFieldValue(authConfig, AzuKmsAuthConfigField.CLIENT_SECRET.fieldName); + String tenantId = getConfigFieldValue(authConfig, AzuKmsAuthConfigField.TENANT_ID.fieldName); if (clientId == null || clientSecret == null || tenantId == null) { String errMsg = "Cannot get credentials. clientId, clientSecret, or tenantId is null."; @@ -89,7 +128,7 @@ public TokenCredential getCredentials(ObjectNode authConfig) { */ public KeyClient getKeyClient(ObjectNode authConfig) { return new KeyClientBuilder() - .vaultUrl(getConfigFieldValue(authConfig, AZU_VAULT_URL_FIELDNAME)) + .vaultUrl(getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName)) .credential(getCredentials(authConfig)) .buildClient(); } @@ -103,7 +142,7 @@ public KeyClient getKeyClient(ObjectNode authConfig) { */ public String getKeyId(ObjectNode authConfig, String keyVersion) { KeyClient keyClient = getKeyClient(authConfig); - String keyName = getConfigFieldValue(authConfig, AZU_KEY_NAME_FIELDNAME); + String keyName = getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName); String keyId = keyClient.getKey(keyName, keyVersion).getId(); return keyId; } @@ -152,8 +191,8 @@ public String getConfigFieldValue(ObjectNode authConfig, String fieldName) { */ public Integer getConfigKeySize(ObjectNode authConfig) { Integer keySize = null; - if (authConfig.has(AZU_KEY_SIZE_FIELDNAME)) { - keySize = authConfig.path(AZU_KEY_SIZE_FIELDNAME).asInt(); + if (authConfig.has(AzuKmsAuthConfigField.AZU_KEY_SIZE.fieldName)) { + keySize = authConfig.path(AzuKmsAuthConfigField.AZU_KEY_SIZE.fieldName).asInt(); } else { log.warn("Could not get AZU config key size. 'AZU_KEY_SIZE' not found."); return null; @@ -170,7 +209,7 @@ public Integer getConfigKeySize(ObjectNode authConfig) { */ public void createKey(ObjectNode authConfig) { KeyClient keyClient = getKeyClient(authConfig); - String keyName = getConfigFieldValue(authConfig, AZU_KEY_NAME_FIELDNAME); + String keyName = getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName); CreateRsaKeyOptions createRsaKeyOptions = new CreateRsaKeyOptions(keyName); createRsaKeyOptions = createRsaKeyOptions @@ -191,7 +230,7 @@ public void createKey(ObjectNode authConfig) { */ public KeyVaultKey getKey(ObjectNode authConfig) { KeyClient keyClient = getKeyClient(authConfig); - String keyName = getConfigFieldValue(authConfig, AZU_KEY_NAME_FIELDNAME); + String keyName = getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName); KeyVaultKey keyVaultKey = keyClient.getKey(keyName); return keyVaultKey; } @@ -250,7 +289,8 @@ public byte[] unwrapKey(ObjectNode authConfig, byte[] keyRef) { return keyBytes; } catch (Exception E) { // Else try to decrypt against all key versions for that key name - String keyName = getConfigFieldValue(authConfig, AZU_KEY_NAME_FIELDNAME); + String keyName = + getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName); log.debug( "Could not decrypt/unwrap using primary key version of key name '{}'. " + "Trying other key versions.", @@ -315,7 +355,7 @@ public boolean checkKeyVaultisValid(ObjectNode authConfig) { } catch (Exception e) { log.error( "Key vault or credentials are invalid for config with key vault url = " - + getConfigFieldValue(authConfig, AZU_VAULT_URL_FIELDNAME)); + + getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName)); return false; } return false; @@ -340,7 +380,8 @@ public boolean checkKeyExists(ObjectNode authConfig, String keyName) { String msg = String.format( "Key does not exist in the key vault with key name = '%s' and key vault url = '%s'", - keyName, getConfigFieldValue(authConfig, AZU_VAULT_URL_FIELDNAME)); + keyName, + getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName)); log.error(msg); return false; } @@ -361,17 +402,18 @@ public void checkKeyVaultAndKeyExists(ObjectNode authConfig) throws RuntimeExcep String errMsg = String.format( "Key vault or the credentials are invalid for key vault url = '%s'", - getConfigFieldValue(authConfig, AZU_VAULT_URL_FIELDNAME)); + getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName)); log.error(errMsg); throw new RuntimeException(errMsg); } // Ensure the key with given name exists in the key vault - String keyName = getConfigFieldValue(authConfig, AZU_KEY_NAME_FIELDNAME); + String keyName = getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName); if (!checkKeyExists(authConfig, keyName)) { String errMsg = String.format( "Key does not exist in the key vault with key name = '%s' and key vault url = '%s'", - keyName, getConfigFieldValue(authConfig, AZU_VAULT_URL_FIELDNAME)); + keyName, + getConfigFieldValue(authConfig, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName)); log.error(errMsg); throw new RuntimeException(errMsg); } @@ -436,13 +478,13 @@ public boolean validateKeySettings(ObjectNode authConfig, String keyName) { public boolean checkFieldsExist(ObjectNode formData) { List fieldsList = Arrays.asList( - CLIENT_ID_FIELDNAME, - CLIENT_SECRET_FIELDNAME, - TENANT_ID_FIELDNAME, - AZU_VAULT_URL_FIELDNAME, - AZU_KEY_NAME_FIELDNAME, - AZU_KEY_ALGORITHM_FIELDNAME, - AZU_KEY_SIZE_FIELDNAME); + AzuKmsAuthConfigField.CLIENT_ID.fieldName, + AzuKmsAuthConfigField.CLIENT_SECRET.fieldName, + AzuKmsAuthConfigField.TENANT_ID.fieldName, + AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName, + AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName, + AzuKmsAuthConfigField.AZU_KEY_ALGORITHM.fieldName, + AzuKmsAuthConfigField.AZU_KEY_SIZE.fieldName); for (String fieldKey : fieldsList) { if (!formData.has(fieldKey) || StringUtils.isBlank(formData.path(fieldKey).toString())) { return false; diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/util/GcpEARServiceUtil.java b/managed/src/main/java/com/yugabyte/yw/common/kms/util/GcpEARServiceUtil.java index 4ffb16f5b5e9..07dc1a664a30 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/util/GcpEARServiceUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/util/GcpEARServiceUtil.java @@ -50,6 +50,8 @@ import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; import lombok.extern.slf4j.Slf4j; @@ -57,13 +59,6 @@ @Slf4j public class GcpEARServiceUtil { - public static final String GCP_CONFIG_FIELDNAME = "GCP_CONFIG"; - public static final String LOCATION_ID_FIELDNAME = "LOCATION_ID"; - public static final String PROTECTION_LEVEL_FIELDNAME = "PROTECTION_LEVEL"; - public static final String GCP_KMS_ENDPOINT_FIELDNAME = "GCP_KMS_ENDPOINT"; - public static final String KEY_RING_ID_FIELDNAME = "KEY_RING_ID"; - public static final String CRYPTO_KEY_ID_FIELDNAME = "CRYPTO_KEY_ID"; - // bare minimum permissions required for using GCP KMS in YBA public static final List minRequiredPermissionsList = Arrays.asList( @@ -73,6 +68,50 @@ public class GcpEARServiceUtil { "cloudkms.cryptoKeyVersions.useToDecrypt", "cloudkms.locations.generateRandomBytes"); + // All fields in Google KMS authConfig object sent from UI + public enum GcpKmsAuthConfigField { + GCP_CONFIG("GCP_CONFIG", true, false), + LOCATION_ID("LOCATION_ID", false, true), + PROTECTION_LEVEL("PROTECTION_LEVEL", false, false), + GCP_KMS_ENDPOINT("GCP_KMS_ENDPOINT", false, false), + KEY_RING_ID("KEY_RING_ID", false, true), + CRYPTO_KEY_ID("CRYPTO_KEY_ID", false, true); + + public final String fieldName; + public final boolean isEditable; + public final boolean isMetadata; + + GcpKmsAuthConfigField(String fieldName, boolean isEditable, boolean isMetadata) { + this.fieldName = fieldName; + this.isEditable = isEditable; + this.isMetadata = isMetadata; + } + + public static List getEditableFields() { + return Arrays.asList(values()) + .stream() + .filter(configField -> configField.isEditable) + .map(configField -> configField.fieldName) + .collect(Collectors.toList()); + } + + public static List getNonEditableFields() { + return Arrays.asList(values()) + .stream() + .filter(configField -> !configField.isEditable) + .map(configField -> configField.fieldName) + .collect(Collectors.toList()); + } + + public static List getMetadataFields() { + return Arrays.asList(values()) + .stream() + .filter(configField -> configField.isMetadata) + .map(configField -> configField.fieldName) + .collect(Collectors.toList()); + } + } + public ObjectNode getAuthConfig(UUID configUUID) { return EncryptionAtRestUtil.getAuthConfig(configUUID); } @@ -86,9 +125,10 @@ public ObjectNode getAuthConfig(UUID configUUID) { */ public CredentialsProvider getCredentialsProvider(ObjectNode authConfig) { CredentialsProvider credentialsProvider = null; - if (authConfig.has(GCP_CONFIG_FIELDNAME) - && !StringUtils.isBlank(authConfig.path(GCP_CONFIG_FIELDNAME).toString())) { - String credentials = authConfig.path(GCP_CONFIG_FIELDNAME).toString(); + if (authConfig.has(GcpKmsAuthConfigField.GCP_CONFIG.fieldName) + && !StringUtils.isBlank( + authConfig.path(GcpKmsAuthConfigField.GCP_CONFIG.fieldName).toString())) { + String credentials = authConfig.path(GcpKmsAuthConfigField.GCP_CONFIG.fieldName).toString(); try { ServiceAccountCredentials serviceAccountCredentials = ServiceAccountCredentials.fromStream(new ByteArrayInputStream(credentials.getBytes())); @@ -117,10 +157,11 @@ public KeyManagementServiceClient getKMSClient(ObjectNode authConfig) { if (credentialsProvider != null) { keyManagementServiceSettingsBuilder.setCredentialsProvider(credentialsProvider); } - if (authConfig.has(GCP_KMS_ENDPOINT_FIELDNAME) - && !StringUtils.isBlank(authConfig.path(GCP_KMS_ENDPOINT_FIELDNAME).asText())) { + if (authConfig.has(GcpKmsAuthConfigField.GCP_KMS_ENDPOINT.fieldName) + && !StringUtils.isBlank( + authConfig.path(GcpKmsAuthConfigField.GCP_KMS_ENDPOINT.fieldName).asText())) { keyManagementServiceSettingsBuilder.setEndpoint( - authConfig.path(GCP_KMS_ENDPOINT_FIELDNAME).asText()); + authConfig.path(GcpKmsAuthConfigField.GCP_KMS_ENDPOINT.fieldName).asText()); } KeyManagementServiceSettings keyManagementServiceSettings; KeyManagementServiceClient gcpKmsClient = null; @@ -143,9 +184,10 @@ public KeyManagementServiceClient getKMSClient(ObjectNode authConfig) { */ public String getConfigProjectId(ObjectNode authConfig) { String projectId = ""; - if (authConfig.has(GCP_CONFIG_FIELDNAME) - && authConfig.get(GCP_CONFIG_FIELDNAME).has("project_id")) { - projectId = authConfig.get(GCP_CONFIG_FIELDNAME).get("project_id").asText(); + if (authConfig.has(GcpKmsAuthConfigField.GCP_CONFIG.fieldName) + && authConfig.get(GcpKmsAuthConfigField.GCP_CONFIG.fieldName).has("project_id")) { + projectId = + authConfig.get(GcpKmsAuthConfigField.GCP_CONFIG.fieldName).get("project_id").asText(); } else { log.info("Could not get GCP config project ID. 'GCP_CONFIG.project_id' not found."); return null; @@ -161,8 +203,8 @@ public String getConfigProjectId(ObjectNode authConfig) { */ public String getConfigEndpoint(ObjectNode authConfig) { String endpoint = ""; - if (authConfig.has(GCP_KMS_ENDPOINT_FIELDNAME)) { - endpoint = authConfig.path(GCP_KMS_ENDPOINT_FIELDNAME).asText(); + if (authConfig.has(GcpKmsAuthConfigField.GCP_KMS_ENDPOINT.fieldName)) { + endpoint = authConfig.path(GcpKmsAuthConfigField.GCP_KMS_ENDPOINT.fieldName).asText(); } else { log.info("Could not get GCP config endpoint from auth config. 'GCP_KMS_ENDPOINT' not found."); return null; @@ -178,8 +220,8 @@ public String getConfigEndpoint(ObjectNode authConfig) { */ public String getConfigKeyRingId(ObjectNode authConfig) { String keyRingId = ""; - if (authConfig.has(KEY_RING_ID_FIELDNAME)) { - keyRingId = authConfig.path(KEY_RING_ID_FIELDNAME).asText(); + if (authConfig.has(GcpKmsAuthConfigField.KEY_RING_ID.fieldName)) { + keyRingId = authConfig.path(GcpKmsAuthConfigField.KEY_RING_ID.fieldName).asText(); } else { log.info("Could not get GCP config key ring from auth config. 'KEY_RING_ID' not found."); return null; @@ -195,8 +237,8 @@ public String getConfigKeyRingId(ObjectNode authConfig) { */ public String getConfigCryptoKeyId(ObjectNode authConfig) { String cryptoKeyId = ""; - if (authConfig.has(CRYPTO_KEY_ID_FIELDNAME)) { - cryptoKeyId = authConfig.path(CRYPTO_KEY_ID_FIELDNAME).asText(); + if (authConfig.has(GcpKmsAuthConfigField.CRYPTO_KEY_ID.fieldName)) { + cryptoKeyId = authConfig.path(GcpKmsAuthConfigField.CRYPTO_KEY_ID.fieldName).asText(); } else { log.info( "Could not get GCP config crypto key ID from auth config. 'CRYPTO_KEY_ID' not found."); @@ -214,8 +256,8 @@ public String getConfigCryptoKeyId(ObjectNode authConfig) { */ public String getConfigLocationId(ObjectNode authConfig) { String locationId = ""; - if (authConfig.has(LOCATION_ID_FIELDNAME)) { - locationId = authConfig.path(LOCATION_ID_FIELDNAME).asText(); + if (authConfig.has(GcpKmsAuthConfigField.LOCATION_ID.fieldName)) { + locationId = authConfig.path(GcpKmsAuthConfigField.LOCATION_ID.fieldName).asText(); } else { log.info("Could not get GCP config location ID from auth config. 'LOCATION_ID' not found."); return null; @@ -225,12 +267,12 @@ public String getConfigLocationId(ObjectNode authConfig) { public ProtectionLevel getConfigProtectionLevel(ObjectNode authConfig) { String protectionLevel = ""; - if (authConfig.has(PROTECTION_LEVEL_FIELDNAME)) { - protectionLevel = authConfig.path(PROTECTION_LEVEL_FIELDNAME).asText(); + if (authConfig.has(GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName)) { + protectionLevel = authConfig.path(GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName).asText(); } else { log.info( "Could not get GCP config protection level from auth config. " - + "'PROTECTION_LEVEL_FIELDNAME' not found."); + + "'GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName' not found."); return null; } return ProtectionLevel.valueOf(protectionLevel); @@ -442,7 +484,9 @@ public boolean checkCryptoKeyExists(ObjectNode authConfig) { + "Changed protection level from {} to {}.", configProtectionLevel, cryptoKeyVersion.getProtectionLevel()); - authConfig.put(PROTECTION_LEVEL_FIELDNAME, cryptoKeyVersion.getProtectionLevel().toString()); + authConfig.put( + GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName, + cryptoKeyVersion.getProtectionLevel().toString()); } return true; } @@ -540,8 +584,11 @@ public CryptoKey createCryptoKey(ObjectNode authConfig, String keyRingRN, String // Set the protection level to either HSM or default SOFTWARE CryptoKeyVersionTemplate.Builder cryptoKeyVersionTemplateBuilder = CryptoKeyVersionTemplate.newBuilder(); - if (authConfig.has(PROTECTION_LEVEL_FIELDNAME) - && authConfig.path(PROTECTION_LEVEL_FIELDNAME).asText().equals("HSM")) { + if (authConfig.has(GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName) + && authConfig + .path(GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName) + .asText() + .equals("HSM")) { cryptoKeyVersionTemplateBuilder.setProtectionLevel(ProtectionLevel.HSM); } // Use default google symmetric encryption algorithm @@ -684,7 +731,7 @@ public boolean testGcpPermissions(ObjectNode authConfig, List permission public CloudResourceManager createCloudResourceManagerService(ObjectNode authConfig) throws IOException, GeneralSecurityException { - String credentials = authConfig.path(GCP_CONFIG_FIELDNAME).toString(); + String credentials = authConfig.path(GcpKmsAuthConfigField.GCP_CONFIG.fieldName).toString(); GoogleCredentials credential = GoogleCredentials.fromStream(new ByteArrayInputStream(credentials.getBytes())) @@ -709,10 +756,10 @@ public CloudResourceManager createCloudResourceManagerService(ObjectNode authCon public boolean checkFieldsExist(ObjectNode formData) { List fieldsList = Arrays.asList( - GCP_CONFIG_FIELDNAME, - LOCATION_ID_FIELDNAME, - KEY_RING_ID_FIELDNAME, - CRYPTO_KEY_ID_FIELDNAME); + GcpKmsAuthConfigField.GCP_CONFIG.fieldName, + GcpKmsAuthConfigField.LOCATION_ID.fieldName, + GcpKmsAuthConfigField.KEY_RING_ID.fieldName, + GcpKmsAuthConfigField.CRYPTO_KEY_ID.fieldName); for (String fieldKey : fieldsList) { if (!formData.has(fieldKey) || StringUtils.isBlank(formData.path(fieldKey).toString())) { return false; @@ -742,7 +789,7 @@ public void validateKMSProviderConfigFormData(ObjectNode formData) throws Except client.close(); // Check if custom key ring id and crypto key id is given if (checkFieldsExist(formData)) { - String keyRingId = formData.path(KEY_RING_ID_FIELDNAME).asText(); + String keyRingId = formData.path(GcpKmsAuthConfigField.KEY_RING_ID.fieldName).asText(); log.info("validateKMSProviderConfigFormData: Checked all required fields exist."); // If given a custom key ring, validate its permissions if (testGcpPermissions(formData, minRequiredPermissionsList) == false) { diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/util/HashicorpEARServiceUtil.java b/managed/src/main/java/com/yugabyte/yw/common/kms/util/HashicorpEARServiceUtil.java index bfcd48243070..daa5c04acb6e 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/util/HashicorpEARServiceUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/util/HashicorpEARServiceUtil.java @@ -11,6 +11,7 @@ package com.yugabyte.yw.common.kms.util; +import java.util.Arrays; import java.util.List; import java.util.UUID; import javax.crypto.KeyGenerator; @@ -261,4 +262,12 @@ public static void refreshServiceUtil(UUID configUUID, ObjectNode authConfig) th VaultSecretEngineBuilder.getVaultSecretEngine(authConfig); updateAuthConfigObj(configUUID, vaultSecretEngine, authConfig); } + + public static List getMetadataFields() { + return Arrays.asList( + HashicorpVaultConfigParams.HC_VAULT_ADDRESS, + HashicorpVaultConfigParams.HC_VAULT_ENGINE, + HashicorpVaultConfigParams.HC_VAULT_MOUNT_PATH, + HashicorpVaultConfigParams.HC_VAULT_KEY_NAME); + } } diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/EncryptionAtRestController.java b/managed/src/main/java/com/yugabyte/yw/controllers/EncryptionAtRestController.java index 86fc4885b50c..c839ed7991f5 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/EncryptionAtRestController.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/EncryptionAtRestController.java @@ -27,6 +27,8 @@ import com.yugabyte.yw.common.kms.util.HashicorpEARServiceUtil; import com.yugabyte.yw.common.kms.util.KeyProvider; import com.yugabyte.yw.common.kms.util.AwsEARServiceUtil.AwsKmsAuthConfigField; +import com.yugabyte.yw.common.kms.util.AzuEARServiceUtil.AzuKmsAuthConfigField; +import com.yugabyte.yw.common.kms.util.GcpEARServiceUtil.GcpKmsAuthConfigField; import com.yugabyte.yw.forms.PlatformResults; import com.yugabyte.yw.forms.PlatformResults.YBPSuccess; import com.yugabyte.yw.forms.PlatformResults.YBPTask; @@ -151,10 +153,10 @@ private void validateKMSProviderConfigFormData( LOG.info( "Finished validating AZU provider config form data for key vault = " + azuEARServiceUtil.getConfigFieldValue( - formData, AzuEARServiceUtil.AZU_VAULT_URL_FIELDNAME) + formData, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName) + ", key name = " + azuEARServiceUtil.getConfigFieldValue( - formData, AzuEARServiceUtil.AZU_KEY_NAME_FIELDNAME)); + formData, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName)); } catch (Exception e) { LOG.warn("Could not finish validating AZU provider config form data."); throw new PlatformServiceException(BAD_REQUEST, e.toString()); @@ -208,11 +210,11 @@ private void checkEditableFields(ObjectNode formData, KeyProvider keyProvider, U // All the below fields are non editable List nonEditableFields = Arrays.asList( - GcpEARServiceUtil.LOCATION_ID_FIELDNAME, - GcpEARServiceUtil.PROTECTION_LEVEL_FIELDNAME, - GcpEARServiceUtil.GCP_KMS_ENDPOINT_FIELDNAME, - GcpEARServiceUtil.KEY_RING_ID_FIELDNAME, - GcpEARServiceUtil.CRYPTO_KEY_ID_FIELDNAME); + GcpKmsAuthConfigField.LOCATION_ID.fieldName, + GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName, + GcpKmsAuthConfigField.GCP_KMS_ENDPOINT.fieldName, + GcpKmsAuthConfigField.KEY_RING_ID.fieldName, + GcpKmsAuthConfigField.CRYPTO_KEY_ID.fieldName); for (String field : nonEditableFields) { if (formData.has(field)) { if (!authconfig.has(field) @@ -228,10 +230,10 @@ private void checkEditableFields(ObjectNode formData, KeyProvider keyProvider, U // All the below fields are non editable in AZU List nonEditableFieldsAzu = Arrays.asList( - AzuEARServiceUtil.AZU_VAULT_URL_FIELDNAME, - AzuEARServiceUtil.AZU_KEY_NAME_FIELDNAME, - AzuEARServiceUtil.AZU_KEY_ALGORITHM_FIELDNAME, - AzuEARServiceUtil.AZU_KEY_SIZE_FIELDNAME); + AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName, + AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName, + AzuKmsAuthConfigField.AZU_KEY_ALGORITHM.fieldName, + AzuKmsAuthConfigField.AZU_KEY_SIZE.fieldName); for (String field : nonEditableFieldsAzu) { if (formData.has(field)) { if (!authconfig.has(field) @@ -289,22 +291,22 @@ private ObjectNode addNonEditableFieldsData( // All these fields must be kept the same from the old authConfig (if it has) List nonEditableFields = Arrays.asList( - GcpEARServiceUtil.LOCATION_ID_FIELDNAME, - GcpEARServiceUtil.PROTECTION_LEVEL_FIELDNAME, - GcpEARServiceUtil.GCP_KMS_ENDPOINT_FIELDNAME, - GcpEARServiceUtil.KEY_RING_ID_FIELDNAME, - GcpEARServiceUtil.CRYPTO_KEY_ID_FIELDNAME); + GcpKmsAuthConfigField.LOCATION_ID.fieldName, + GcpKmsAuthConfigField.PROTECTION_LEVEL.fieldName, + GcpKmsAuthConfigField.GCP_KMS_ENDPOINT.fieldName, + GcpKmsAuthConfigField.KEY_RING_ID.fieldName, + GcpKmsAuthConfigField.CRYPTO_KEY_ID.fieldName); for (String field : nonEditableFields) { if (authConfig.has(field)) { formData.set(field, authConfig.get(field)); } } // GCP_CONFIG field can change. If no config is specified, use the same old one. - if (!formData.has(GcpEARServiceUtil.GCP_CONFIG_FIELDNAME) - && authConfig.has(GcpEARServiceUtil.GCP_CONFIG_FIELDNAME)) { + if (!formData.has(GcpKmsAuthConfigField.GCP_CONFIG.fieldName) + && authConfig.has(GcpKmsAuthConfigField.GCP_CONFIG.fieldName)) { formData.set( - GcpEARServiceUtil.GCP_CONFIG_FIELDNAME, - authConfig.get(GcpEARServiceUtil.GCP_CONFIG_FIELDNAME)); + GcpKmsAuthConfigField.GCP_CONFIG.fieldName, + authConfig.get(GcpKmsAuthConfigField.GCP_CONFIG.fieldName)); } LOG.info("Added all required fields to the formData to be edited"); break; @@ -312,10 +314,10 @@ private ObjectNode addNonEditableFieldsData( // All these fields must be kept the same from the old authConfig (if it has) List nonEditableFieldsAzu = Arrays.asList( - AzuEARServiceUtil.AZU_VAULT_URL_FIELDNAME, - AzuEARServiceUtil.AZU_KEY_NAME_FIELDNAME, - AzuEARServiceUtil.AZU_KEY_ALGORITHM_FIELDNAME, - AzuEARServiceUtil.AZU_KEY_SIZE_FIELDNAME); + AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName, + AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName, + AzuKmsAuthConfigField.AZU_KEY_ALGORITHM.fieldName, + AzuKmsAuthConfigField.AZU_KEY_SIZE.fieldName); for (String field : nonEditableFieldsAzu) { if (authConfig.has(field)) { formData.set(field, authConfig.get(field)); @@ -324,9 +326,9 @@ private ObjectNode addNonEditableFieldsData( // Below fields can change. If no new field is specified, use the same old one. List editableFieldsAzu = Arrays.asList( - AzuEARServiceUtil.CLIENT_ID_FIELDNAME, - AzuEARServiceUtil.CLIENT_SECRET_FIELDNAME, - AzuEARServiceUtil.TENANT_ID_FIELDNAME); + AzuKmsAuthConfigField.CLIENT_ID.fieldName, + AzuKmsAuthConfigField.CLIENT_SECRET.fieldName, + AzuKmsAuthConfigField.TENANT_ID.fieldName); for (String field : editableFieldsAzu) { if (!formData.has(field) && authConfig.has(field)) { formData.set(field, authConfig.get(field)); diff --git a/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java b/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java index e9d1dc9853bb..2f5782783365 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java +++ b/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java @@ -238,6 +238,18 @@ public static Set getUniverses(UUID configUUID, KmsHistoryId.TargetTyp return Universe.getAllPresent(universeUUIDs); } + public static Set getDistinctKmsConfigUUIDs(UUID targetUUID) { + Set KmsConfigUUIDs = new HashSet<>(); + KmsHistory.find + .query() + .where() + .eq("target_uuid", targetUUID) + .eq("type", KmsHistoryId.TargetType.UNIVERSE_KEY) + .findList() + .forEach(kh -> KmsConfigUUIDs.add(kh.configUuid)); + return KmsConfigUUIDs; + } + @Override public String toString() { return Json.newObject() diff --git a/managed/src/test/java/com/yugabyte/yw/common/TestHelper.java b/managed/src/test/java/com/yugabyte/yw/common/TestHelper.java index e8dbf2d9a32f..f50dea6e6d77 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/TestHelper.java +++ b/managed/src/test/java/com/yugabyte/yw/common/TestHelper.java @@ -5,9 +5,14 @@ import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.yugabyte.yw.common.ha.PlatformReplicationManager; +import com.yugabyte.yw.common.kms.util.KeyProvider; + import io.ebean.Ebean; import io.ebean.EbeanServer; import java.io.BufferedOutputStream; @@ -113,4 +118,53 @@ public static void createTarGzipFiles(List paths, Path output) throws IOEx tOut.finish(); } } + + public static ObjectNode getFakeKmsAuthConfig(KeyProvider keyProvider) { + ObjectNode fakeAuthConfig = new ObjectMapper().createObjectNode(); + switch (keyProvider) { + case AWS: + fakeAuthConfig.put("name", "fake-aws-kms-config"); + fakeAuthConfig.put("AWS_ACCESS_KEY_ID", "fake-access-key"); + fakeAuthConfig.put("AWS_SECRET_ACCESS_KEY", "fake-secret-access"); + fakeAuthConfig.put("AWS_KMS_ENDPOINT", "fake-kms-endpoint"); + fakeAuthConfig.put("cmk_policy", "fake-cmk-policy"); + fakeAuthConfig.put("AWS_REGION", "us-west-1"); + fakeAuthConfig.put("cmk_id", "fake-cmk-id"); + break; + case GCP: + fakeAuthConfig.put("name", "fake-gcp-kms-config"); + fakeAuthConfig.put("LOCATION_ID", "global"); + fakeAuthConfig.put("PROTECTION_LEVEL", "HSM"); + fakeAuthConfig.put("GCP_KMS_ENDPOINT", "fake-kms-endpoint"); + fakeAuthConfig.put("KEY_RING_ID", "yb-kr"); + fakeAuthConfig.put("CRYPTO_KEY_ID", "yb-ck"); + + // Populate GCP config + ObjectNode fakeGcpConfig = fakeAuthConfig.putObject("GCP_CONFIG"); + fakeGcpConfig.put("type", "service_account"); + fakeGcpConfig.put("project_id", "yugabyte"); + break; + case AZU: + fakeAuthConfig.put("name", "fake-azu-kms-config"); + fakeAuthConfig.put("CLIENT_ID", "fake-client-id"); + fakeAuthConfig.put("CLIENT_SECRET", "fake-client-secret"); + fakeAuthConfig.put("TENANT_ID", "fake-tenant-id"); + fakeAuthConfig.put("AZU_VAULT_URL", "fake-vault-url"); + fakeAuthConfig.put("AZU_KEY_NAME", "fake-key-name"); + fakeAuthConfig.put("AZU_KEY_ALGORITHM", "RSA"); + fakeAuthConfig.put("AZU_KEY_SIZE", 2048); + break; + case HASHICORP: + fakeAuthConfig.put("name", "fake-hc-kms-config"); + fakeAuthConfig.put("HC_VAULT_ADDRESS", "fake-vault-address"); + fakeAuthConfig.put("HC_VAULT_TOKEN", "fake-vault-token"); + fakeAuthConfig.put("HC_VAULT_MOUNT_PATH", "fake-mount-path"); + fakeAuthConfig.put("HC_VAULT_ENGINE", "fake-vault-engine"); + break; + case SMARTKEY: + break; + } + + return fakeAuthConfig; + } } diff --git a/managed/src/test/java/com/yugabyte/yw/common/kms/services/AwsEARServiceTest.java b/managed/src/test/java/com/yugabyte/yw/common/kms/services/AwsEARServiceTest.java index a05546fe27e0..18016841729a 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/kms/services/AwsEARServiceTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/kms/services/AwsEARServiceTest.java @@ -14,10 +14,10 @@ import static org.junit.Assert.assertNotNull; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; import com.amazonaws.services.kms.AWSKMS; import com.amazonaws.services.kms.model.AliasListEntry; @@ -32,16 +32,20 @@ import com.amazonaws.services.kms.model.ListAliasesRequest; import com.amazonaws.services.kms.model.ListAliasesResult; import com.amazonaws.services.kms.model.UpdateAliasRequest; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.yugabyte.yw.common.ApiHelper; import com.yugabyte.yw.common.FakeDBApplication; +import com.yugabyte.yw.common.TestHelper; import com.yugabyte.yw.common.config.RuntimeConfGetter; import com.yugabyte.yw.common.config.UniverseConfKeys; import com.yugabyte.yw.common.kms.util.KeyProvider; import com.yugabyte.yw.forms.EncryptionAtRestConfig; +import com.yugabyte.yw.models.KmsConfig; import com.yugabyte.yw.models.Universe; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.UUID; import org.junit.Before; @@ -56,11 +60,11 @@ public class AwsEARServiceTest extends FakeDBApplication { @Mock RuntimeConfGetter mockConfGetter; ApiHelper mockApiHelper; - AwsEARService encryptionService; + AwsEARService awsEARService; KeyProvider testKeyProvider = KeyProvider.AWS; - String testCmkId = "some_cmk_id"; + String testCmkId = "fake-cmk-id"; AWSKMS mockClient; ListAliasesResult mockAliasList; @@ -90,22 +94,31 @@ public void setUp() { mockKeyMetadata = mock(KeyMetadata.class); mockDataKeyResult = mock(GenerateDataKeyWithoutPlaintextResult.class); mockDecryptResult = mock(DecryptResult.class); - when(mockAlias.getAliasName()).thenReturn(String.format("alias/%s", testUniUUID.toString())); - when(mockAliasList.getAliases()).thenReturn(mockAliases); - when(mockClient.listAliases(any(ListAliasesRequest.class))).thenReturn(mockAliasList); - when(mockClient.createKey(any(CreateKeyRequest.class))).thenReturn(mockCreateKeyResult); - when(mockCreateKeyResult.getKeyMetadata()).thenReturn(mockKeyMetadata); - when(mockKeyMetadata.getKeyId()).thenReturn(testCmkId); - when(mockClient.generateDataKeyWithoutPlaintext( - any(GenerateDataKeyWithoutPlaintextRequest.class))) + lenient() + .when(mockAlias.getAliasName()) + .thenReturn(String.format("alias/%s", testUniUUID.toString())); + lenient().when(mockAliasList.getAliases()).thenReturn(mockAliases); + lenient().when(mockClient.listAliases(any(ListAliasesRequest.class))).thenReturn(mockAliasList); + lenient() + .when(mockClient.createKey(any(CreateKeyRequest.class))) + .thenReturn(mockCreateKeyResult); + lenient().when(mockCreateKeyResult.getKeyMetadata()).thenReturn(mockKeyMetadata); + lenient().when(mockKeyMetadata.getKeyId()).thenReturn(testCmkId); + lenient() + .when( + mockClient.generateDataKeyWithoutPlaintext( + any(GenerateDataKeyWithoutPlaintextRequest.class))) .thenReturn(mockDataKeyResult); - when(mockDataKeyResult.getCiphertextBlob()) + lenient() + .when(mockDataKeyResult.getCiphertextBlob()) .thenReturn(ByteBuffer.wrap(new String("some_universe_key_value_encrypted").getBytes())); - when(mockClient.decrypt(any(DecryptRequest.class))).thenReturn(mockDecryptResult); - when(mockDecryptResult.getPlaintext()).thenReturn(decryptedKeyBuffer); - when(mockConfGetter.getConfForScope(any(Universe.class), eq(UniverseConfKeys.cloudEnabled))) + lenient().when(mockClient.decrypt(any(DecryptRequest.class))).thenReturn(mockDecryptResult); + lenient().when(mockDecryptResult.getPlaintext()).thenReturn(decryptedKeyBuffer); + lenient() + .when( + mockConfGetter.getConfForScope(any(Universe.class), eq(UniverseConfKeys.cloudEnabled))) .thenReturn(false); - encryptionService = new AwsEARService(mockConfGetter); + awsEARService = new AwsEARService(mockConfGetter); config = new EncryptionAtRestConfig(); // TODO: (Daniel) - Create KMS Config and link to here config.kmsConfigUUID = null; @@ -119,7 +132,7 @@ public void testCreateAndRetrieveEncryptionKeyCreateAlias() { .withAliasName(String.format("alias/%s", testUniUUID.toString())) .withTargetKeyId(testCmkId); ListAliasesRequest listAliasReq = new ListAliasesRequest().withLimit(100); - byte[] encryptionKey = encryptionService.createKey(testUniUUID, testCustomerUUID, config); + byte[] encryptionKey = awsEARService.createKey(testUniUUID, testCustomerUUID, config); verify(mockClient, times(1)).createKey(any(CreateKeyRequest.class)); verify(mockClient, times(1)).listAliases(listAliasReq); verify(mockClient, times(1)).createAlias(createAliasReq); @@ -136,11 +149,27 @@ public void testCreateAndRetrieveEncryptionKeyUpdateAlias() { .withAliasName("alias/" + testUniUUID.toString()) .withTargetKeyId(testCmkId); ListAliasesRequest listAliasReq = new ListAliasesRequest().withLimit(100); - byte[] encryptionKey = encryptionService.createKey(testUniUUID, testCustomerUUID, config); + byte[] encryptionKey = awsEARService.createKey(testUniUUID, testCustomerUUID, config); verify(mockClient, times(1)).createKey(any(CreateKeyRequest.class)); verify(mockClient, times(1)).listAliases(listAliasReq); verify(mockClient, times(1)).updateAlias(updateAliasReq); assertNotNull(encryptionKey); assertEquals(new String(encryptionKey), new String(mockEncryptionKey)); } + + @Test + public void testGetKeyMetadata() { + // Form the expected key metadata. + ObjectNode fakeAuthConfig = TestHelper.getFakeKmsAuthConfig(KeyProvider.AWS); + ObjectNode expectedKeyMetadata = + fakeAuthConfig.deepCopy().retain(Arrays.asList("AWS_REGION", "cmk_id")); + expectedKeyMetadata.put("key_provider", KeyProvider.AWS.name()); + + // Get the key metadata from the service and compare. + KmsConfig fakeKmsConfig = + KmsConfig.createKMSConfig( + testCustomerUUID, testKeyProvider, fakeAuthConfig, fakeAuthConfig.get("name").asText()); + ObjectNode retrievedKeyMetadata = awsEARService.getKeyMetadata(fakeKmsConfig.configUUID); + assertEquals(expectedKeyMetadata, retrievedKeyMetadata); + } } diff --git a/managed/src/test/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestServiceTest.java b/managed/src/test/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestServiceTest.java index 28da2d3d0aab..23c5b242154e 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestServiceTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/kms/services/EncryptionAtRestServiceTest.java @@ -90,6 +90,12 @@ public byte[] validateRetrieveKeyWithService( this.createRequest = !this.createRequest; return this.createRequest ? null : "some_key_value".getBytes(); } + + @Override + public ObjectNode getKeyMetadata(UUID configUUID) { + // Does nothing here because we test for individual KMS providers. + return null; + } } @RunWith(MockitoJUnitRunner.class) diff --git a/managed/src/test/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtilTest.java b/managed/src/test/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtilTest.java index 6d9060e47906..e7efdc750fe4 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtilTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/kms/util/AzuEARServiceUtilTest.java @@ -30,6 +30,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yugabyte.yw.common.FakeDBApplication; +import com.yugabyte.yw.common.kms.util.AzuEARServiceUtil.AzuKmsAuthConfigField; + import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -112,7 +114,7 @@ public void setUp() { public void testGetConfigClientId() { String clientId = mockAzuEARServiceUtil.getConfigFieldValue( - fakeAuthConfig, AzuEARServiceUtil.CLIENT_ID_FIELDNAME); + fakeAuthConfig, AzuKmsAuthConfigField.CLIENT_ID.fieldName); assertEquals(clientId, authConfigClientId); } @@ -120,7 +122,7 @@ public void testGetConfigClientId() { public void testGetConfigClientSecret() { String clientSecret = mockAzuEARServiceUtil.getConfigFieldValue( - fakeAuthConfig, AzuEARServiceUtil.CLIENT_SECRET_FIELDNAME); + fakeAuthConfig, AzuKmsAuthConfigField.CLIENT_SECRET.fieldName); assertEquals(clientSecret, authConfigClientSecret); } @@ -128,7 +130,7 @@ public void testGetConfigClientSecret() { public void testGetConfigTenantId() { String tenantId = mockAzuEARServiceUtil.getConfigFieldValue( - fakeAuthConfig, AzuEARServiceUtil.TENANT_ID_FIELDNAME); + fakeAuthConfig, AzuKmsAuthConfigField.TENANT_ID.fieldName); assertEquals(tenantId, authConfigTenantId); } @@ -136,7 +138,7 @@ public void testGetConfigTenantId() { public void testGetConfigVaultUrl() { String vaultUrl = mockAzuEARServiceUtil.getConfigFieldValue( - fakeAuthConfig, AzuEARServiceUtil.AZU_VAULT_URL_FIELDNAME); + fakeAuthConfig, AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName); assertEquals(vaultUrl, authConfigAzuVaultUrl); } @@ -144,7 +146,7 @@ public void testGetConfigVaultUrl() { public void testGetConfigKeyName() { String keyName = mockAzuEARServiceUtil.getConfigFieldValue( - fakeAuthConfig, AzuEARServiceUtil.AZU_KEY_NAME_FIELDNAME); + fakeAuthConfig, AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName); assertEquals(keyName, authConfigAzuKeyName); } @@ -152,7 +154,7 @@ public void testGetConfigKeyName() { public void testGetConfigKeyAlgorithm() { String keyAlgorithm = mockAzuEARServiceUtil.getConfigFieldValue( - fakeAuthConfig, AzuEARServiceUtil.AZU_KEY_ALGORITHM_FIELDNAME); + fakeAuthConfig, AzuKmsAuthConfigField.AZU_KEY_ALGORITHM.fieldName); assertEquals(keyAlgorithm, authConfigAzuKeyAlgorithm); } diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/EncryptionAtRestControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/EncryptionAtRestControllerTest.java index c6f36bed6109..e529c757fd23 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/EncryptionAtRestControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/EncryptionAtRestControllerTest.java @@ -38,6 +38,7 @@ import com.yugabyte.yw.common.kms.util.AzuEARServiceUtil; import com.yugabyte.yw.common.kms.util.KeyProvider; import com.yugabyte.yw.common.kms.util.AwsEARServiceUtil.AwsKmsAuthConfigField; +import com.yugabyte.yw.common.kms.util.AzuEARServiceUtil.AzuKmsAuthConfigField; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.KmsConfig; import com.yugabyte.yw.models.Universe; @@ -379,13 +380,13 @@ public void testEditAZUKMSConfigKeyName() { ObjectNode kmsConfigReq = Json.newObject() .put("name", configName) - .put(AzuEARServiceUtil.CLIENT_ID_FIELDNAME, "test-client-id") - .put(AzuEARServiceUtil.CLIENT_SECRET_FIELDNAME, "test-client-secret") - .put(AzuEARServiceUtil.TENANT_ID_FIELDNAME, "test-tenant-id") - .put(AzuEARServiceUtil.AZU_VAULT_URL_FIELDNAME, "test-vault-url") - .put(AzuEARServiceUtil.AZU_KEY_NAME_FIELDNAME, "test-key-name") - .put(AzuEARServiceUtil.AZU_KEY_ALGORITHM_FIELDNAME, "RSA") - .put(AzuEARServiceUtil.AZU_KEY_SIZE_FIELDNAME, 2048); + .put(AzuKmsAuthConfigField.CLIENT_ID.fieldName, "test-client-id") + .put(AzuKmsAuthConfigField.CLIENT_SECRET.fieldName, "test-client-secret") + .put(AzuKmsAuthConfigField.TENANT_ID.fieldName, "test-tenant-id") + .put(AzuKmsAuthConfigField.AZU_VAULT_URL.fieldName, "test-vault-url") + .put(AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName, "test-key-name") + .put(AzuKmsAuthConfigField.AZU_KEY_ALGORITHM.fieldName, "RSA") + .put(AzuKmsAuthConfigField.AZU_KEY_SIZE.fieldName, 2048); KmsConfig result = KmsConfig.createKMSConfig(customer.uuid, keyProvider, kmsConfigReq, configName); @@ -393,7 +394,7 @@ public void testEditAZUKMSConfigKeyName() { // Edit the key name field in the request body String kmsConfigUrl = String.format("/api/customers/%s/kms_configs/%s/edit", customer.uuid, result.configUUID); - kmsConfigReq.put(AzuEARServiceUtil.AZU_KEY_NAME_FIELDNAME, "test-key-name-2"); + kmsConfigReq.put(AzuKmsAuthConfigField.AZU_KEY_NAME.fieldName, "test-key-name-2"); // Call the API and assert that the POST request throws exception Result updateKMSResult = From d748e570a35712b1e2e530cd0d0a170dca2460bb Mon Sep 17 00:00:00 2001 From: Hari Krishna Sunder Date: Thu, 23 Feb 2023 00:20:26 -0800 Subject: [PATCH 10/81] [#16102] DocDb: Fix index backfill mem leak Summary: The backfill job needs to be added to the table so that we abort and clean it up on shutdown. BackfillTable has a self reference to itself via its backfill_job_ On successful completion of the job we call SetState which in turn calls MarkDone which clears this self reference. With this change we perform the same sequence on AbortAndReturnPrevState as well. The job is added to the table for AbortAndReturnPrevState to get called on shutdown and LoadSysCatalog (master leader election) Fixes #16102 Test Plan: PgIndexBackfillTest.MasterLeaderStepdown PgIndexBackfillTest.DropAfterFail Reviewers: bogdan, amitanand Reviewed By: amitanand Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D23101 --- src/yb/master/backfill_index.cc | 11 +++++++++++ src/yb/master/backfill_index.h | 7 +++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/yb/master/backfill_index.cc b/src/yb/master/backfill_index.cc index 35db0a4c5d52..ce7212ce8d76 100644 --- a/src/yb/master/backfill_index.cc +++ b/src/yb/master/backfill_index.cc @@ -608,6 +608,7 @@ MonitoredTaskState BackfillTableJob::AbortAndReturnPrevState(const Status& statu while (!IsStateTerminal(old_state)) { if (state_.compare_exchange_strong(old_state, MonitoredTaskState::kAborted)) { + MarkDone(); return old_state; } old_state = state(); @@ -623,6 +624,15 @@ void BackfillTableJob::SetState(MonitoredTaskState new_state) { } } } + +void BackfillTableJob::MarkDone() { + completion_timestamp_ = MonoTime::Now(); + if (backfill_table_) { + backfill_table_->table()->RemoveTask(shared_from_this()); + backfill_table_.reset(); + } +} + // ----------------------------------------------------------------------------------------------- // BackfillTable // ----------------------------------------------------------------------------------------------- @@ -720,6 +730,7 @@ const std::unordered_set BackfillTable::indexes_to_build() const { Status BackfillTable::Launch() { backfill_job_ = std::make_shared(shared_from_this()); backfill_job_->SetState(MonitoredTaskState::kRunning); + table()->AddTask(backfill_job_); master_->catalog_manager_impl()->jobs_tracker_->AddTask(backfill_job_); { diff --git a/src/yb/master/backfill_index.h b/src/yb/master/backfill_index.h index 7d4728d085aa..3e1982e38f93 100644 --- a/src/yb/master/backfill_index.h +++ b/src/yb/master/backfill_index.h @@ -151,6 +151,8 @@ class BackfillTable : public std::enable_shared_from_this { const TableId& indexed_table_id() const { return indexed_table_->id(); } + scoped_refptr table() { return indexed_table_; } + Status UpdateRowsProcessedForIndexTable(const uint64_t number_rows_processed); const uint64_t number_rows_processed() const { return number_rows_processed_; } @@ -242,10 +244,7 @@ class BackfillTableJob : public server::MonitoredTask { server::MonitoredTaskState AbortAndReturnPrevState(const Status& status) override; - void MarkDone() { - completion_timestamp_ = MonoTime::Now(); - backfill_table_ = nullptr; - } + void MarkDone(); private: MonoTime start_timestamp_, completion_timestamp_; From b4d1452c3094beab16c217c329458229ba016343 Mon Sep 17 00:00:00 2001 From: Sanskar Garg Date: Mon, 27 Feb 2023 17:39:31 +0000 Subject: [PATCH 11/81] [#11032][yugabyted] Stay connected to ysqlsh/ycqlsh shell on pressing `Ctrl+C` while running `yugabyted connect` command Summary: Pressing `Ctrl+C` while running a `yugabyted connect ysql` or `yugabyted connect ycql` command causes a KeyboardInterrupt error to be raised. Handled the error properly to match the behaviour of `Ctrl+C` inside a ysql/ycql shell started through ysqlsh/ycqlsh. Test Plan: Manualy Tested on a insecure and secure cluster. Reviewers: nikhil Reviewed By: nikhil Subscribers: sgarg-yb Differential Revision: https://phabricator.dev.yugabyte.com/D23187 --- bin/yugabyted | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/yugabyted b/bin/yugabyted index 27fd7ae92892..8d6c17d4944d 100755 --- a/bin/yugabyted +++ b/bin/yugabyted @@ -4156,7 +4156,12 @@ class YsqlProxy(object): env_var = os.environ.copy() env_var["PGUSER"] = self.username shell = subprocess.Popen(cmd, env=env_var) - shell.communicate() + while True: + try: + shell.communicate() + except KeyboardInterrupt: + continue + break # Checks if db exists. # Note that this will return false if ysqlsh can't connect, even if db exists. @@ -4270,7 +4275,12 @@ class YcqlProxy(object): if self.keyspace is not None: cmd.extend(["-k", self.keyspace]) shell = subprocess.Popen(cmd) - shell.communicate() + while True: + try: + shell.communicate() + except KeyboardInterrupt: + continue + break # Check user exists # Note that this will return false if ycqlsh can't connect, even if user exists. From 59261232781ad70e9881c32ea2124401b6700382 Mon Sep 17 00:00:00 2001 From: Sanskar Garg Date: Mon, 27 Feb 2023 15:38:44 +0000 Subject: [PATCH 12/81] [#16245][yugabyted] Fixing `yugabyted configure data_placement` for nodes started with DNS name as `--advertise_address` Summary: `yugabyted configure data_placement` was giving an error when running with a node started with DNS name as it's advertise_address. Test Plan: Manual Testing Reviewers: nikhil Reviewed By: nikhil Subscribers: sgarg-yb Differential Revision: https://phabricator.dev.yugabyte.com/D23189 --- bin/yugabyted | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/yugabyted b/bin/yugabyted index 8d6c17d4944d..6f31b5e76cbe 100755 --- a/bin/yugabyted +++ b/bin/yugabyted @@ -2035,7 +2035,7 @@ class ControlScript(object): placementInfoOfEveryNode = {} for node in dictOfAllNodes: - hostname = re.split('[-:]', node)[0] + hostname = node.split(":")[0] cloudLocationOfHost = "{}.{}.{}".format(dictOfAllNodes[node]["cloud"], dictOfAllNodes[node]["region"], dictOfAllNodes[node]["zone"]) placementInfoOfEveryNode[hostname] = cloudLocationOfHost From 0647506a6c745d2b22f526aca5066ad1d99072a7 Mon Sep 17 00:00:00 2001 From: tnayak Date: Sun, 5 Feb 2023 00:03:54 -0800 Subject: [PATCH 13/81] [#15871] YSQL: Adjust ScanChoices to not unnecessarily read unfiltered suffix columns Summary: This diff fixes up unnecessary copy creations of KeyEntryValues in ScanChoices and makes sure not to read doc key column values that are not filtered. This optimization is particularly helpful in the case of a secondary index only scan where we always impose a dummy filter of [kLowest, kHighest] on the baseybctid column even though there is no reason to read this column. On the following DML query we have an index only scan that goes from 2500ms to 1700ms (~30% improvement) ``` create table t1(a int, b int, primary key(a hash)); insert into t1 select i,i from generate_series(1,1000000) i; create index on t1(a asc); explain analyze select a from t1 where a <= 10000000; ``` Test Plan: Jenkins Reviewers: timur, lnguyen, kpopali Reviewed By: lnguyen, kpopali Subscribers: lnguyen, yql Differential Revision: https://phabricator.dev.yugabyte.com/D22721 --- .../src/backend/access/yb_access/yb_lsm.c | 6 +- .../src/backend/access/yb_access/yb_scan.c | 36 ++++---- src/yb/docdb/doc_reader.cc | 2 - src/yb/docdb/doc_rowwise_iterator.cc | 9 +- src/yb/docdb/scan_choices-test.cc | 31 +++++-- src/yb/docdb/scan_choices.cc | 84 +++++++++---------- src/yb/docdb/scan_choices.h | 2 + 7 files changed, 90 insertions(+), 80 deletions(-) diff --git a/src/postgres/src/backend/access/yb_access/yb_lsm.c b/src/postgres/src/backend/access/yb_access/yb_lsm.c index 9080936f0c98..c15a7eea8f1f 100644 --- a/src/postgres/src/backend/access/yb_access/yb_lsm.c +++ b/src/postgres/src/backend/access/yb_access/yb_lsm.c @@ -456,14 +456,15 @@ ybcingettuple(IndexScanDesc scan, ScanDirection dir) * IndexScan(SysTable, Index) --> HeapTuple. */ scan->xs_ctup.t_ybctid = 0; + bool has_tuple = false; if (ybscan->prepare_params.index_only_scan) { IndexTuple tuple = ybc_getnext_indextuple(ybscan, is_forward_scan, &scan->xs_recheck); if (tuple) { - scan->xs_ctup.t_ybctid = tuple->t_ybctid; scan->xs_itup = tuple; scan->xs_itupdesc = RelationGetDescr(scan->indexRelation); + has_tuple = true; } } else @@ -474,10 +475,11 @@ ybcingettuple(IndexScanDesc scan, ScanDirection dir) scan->xs_ctup.t_ybctid = tuple->t_ybctid; scan->xs_hitup = tuple; scan->xs_hitupdesc = RelationGetDescr(scan->heapRelation); + has_tuple = true; } } - return scan->xs_ctup.t_ybctid != 0; + return has_tuple; } void diff --git a/src/postgres/src/backend/access/yb_access/yb_scan.c b/src/postgres/src/backend/access/yb_access/yb_scan.c index 4c51848fa048..2136c473ae32 100644 --- a/src/postgres/src/backend/access/yb_access/yb_scan.c +++ b/src/postgres/src/backend/access/yb_access/yb_scan.c @@ -1807,33 +1807,25 @@ ybcSetupTargets(YbScanDesc ybScan, YbScanPlan scan_plan, Scan *pg_scan_plan) ybcAddTargetColumn(ybScan, ObjectIdAttributeNumber); if (is_index_only_scan) + return; + + /* Two cases: + * - Primary Scan (Key or sequential) + * SELECT data, ybctid FROM table [ WHERE primary-key-condition ] + * - Secondary IndexScan + * SELECT data, ybctid FROM table WHERE ybctid IN + * ( SELECT base_ybctid FROM IndexTable ) + */ + ybcAddTargetColumn(ybScan, YBTupleIdAttributeNumber); + if (index && !index->rd_index->indisprimary) { /* - * IndexOnlyScan: - * SELECT [ data, ] ybbasectid (ROWID of UserTable, relation) FROM secondary-index-table - * In this case, Postgres requests base_ctid and maybe also data from IndexTable and then uses - * them for further processing. + * IndexScan: Postgres layer sends both actual-query and + * index-scan to PgGate, who will select and immediately use + * base_ctid to query data before responding. */ ybcAddTargetColumn(ybScan, YBIdxBaseTupleIdAttributeNumber); } - else - { - /* Two cases: - * - Primary Scan (Key or sequential) - * SELECT data, ybctid FROM table [ WHERE primary-key-condition ] - * - Secondary IndexScan - * SELECT data, ybctid FROM table WHERE ybctid IN ( SELECT base_ybctid FROM IndexTable ) - */ - ybcAddTargetColumn(ybScan, YBTupleIdAttributeNumber); - if (index && !index->rd_index->indisprimary) - { - /* - * IndexScan: Postgres layer sends both actual-query and index-scan to PgGate, who will - * select and immediately use base_ctid to query data before responding. - */ - ybcAddTargetColumn(ybScan, YBIdxBaseTupleIdAttributeNumber); - } - } } /* diff --git a/src/yb/docdb/doc_reader.cc b/src/yb/docdb/doc_reader.cc index b62c95ef475c..78580cdd7dc8 100644 --- a/src/yb/docdb/doc_reader.cc +++ b/src/yb/docdb/doc_reader.cc @@ -258,7 +258,6 @@ class DocDBTableReader::GetHelperBase { RETURN_NOT_OK(Scan(CheckExistOnly::kTrue)); if (Found()) { EmptyDocFound(); - reader_.iter_->SeekOutOfSubDoc(root_doc_key_); return true; } @@ -374,7 +373,6 @@ class DocDBTableReader::GetHelperBase { } ++column_index_; if (column_index_ == reader_.projection_->size()) { - reader_.iter_->SeekOutOfSubDoc(root_doc_key_); return false; } UpdatePackedColumnData(); diff --git a/src/yb/docdb/doc_rowwise_iterator.cc b/src/yb/docdb/doc_rowwise_iterator.cc index 9fcc2a1890ae..dadcf300ff62 100644 --- a/src/yb/docdb/doc_rowwise_iterator.cc +++ b/src/yb/docdb/doc_rowwise_iterator.cc @@ -306,11 +306,12 @@ Status DocRowwiseIterator::AdvanceIteratorToNextDesiredRow() const { && !scan_choices_->CurrentTargetMatchesKey(row_key_)) { return scan_choices_->SeekToCurrentTarget(db_iter_.get()); } + } + if (!is_forward_scan_) { + VLOG(4) << __PRETTY_FUNCTION__ << " setting as PrevDocKey"; + db_iter_->PrevDocKey(row_key_); } else { - if (!is_forward_scan_) { - VLOG(4) << __PRETTY_FUNCTION__ << " setting as PrevDocKey"; - db_iter_->PrevDocKey(row_key_); - } + db_iter_->SeekOutOfSubDoc(row_key_); } return Status::OK(); diff --git a/src/yb/docdb/scan_choices-test.cc b/src/yb/docdb/scan_choices-test.cc index 765f4010cc2e..4ac05c7d8c4f 100644 --- a/src/yb/docdb/scan_choices-test.cc +++ b/src/yb/docdb/scan_choices-test.cc @@ -239,18 +239,37 @@ void ScanChoicesTest::AdjustForRangeConstraints() { // Size of the dockey we have read so far size_t prev_size = 0; + auto cur_opts = choices_->TEST_GetCurrentOptions(); for (size_t i = 0; i < current_schema_->num_range_key_columns(); i++) { - EXPECT_OK(decoder.DecodeKeyEntryValue(&cur_val)); - if (cur_val.IsInfinity()) { + if (i < cur_opts.size()) { + EXPECT_OK(decoder.DecodeKeyEntryValue(&cur_val)); + } + + if (i == cur_opts.size() || cur_val.IsInfinity()) { KeyBytes new_target; new_target.Reset(Slice(cur_target.data().AsSlice().data(), cur_target.data().AsSlice().data() + valid_size)); ASSERT_GE(i, 1); - auto cur_opts = choices_->TEST_GetCurrentOptions(); auto is_inclusive = cur_opts[i - 1].upper_inclusive(); - for (size_t j = i - 1; j < current_schema_->num_range_key_columns(); j++) { - cur_opts[j].upper().AppendToKey(&new_target); + auto sorttype = current_schema_->column(i - 1).sorting_type(); + auto sortorder = (sorttype == SortingType::kAscending || + sorttype == SortingType::kAscendingNullsLast) ? SortOrder::kAscending : + SortOrder::kDescending; + + auto j = i - 1; + if (is_inclusive) { + // If column i - 1 was inclusive, we move that column up by one to move + // the previous OptionRange for column i - 1. + KeyEntryValue::Int32(cur_opts[j].upper().GetInt32() + 1, + sortorder).AppendToKey(&new_target); + j++; + } + + for (; j < current_schema_->num_range_key_columns(); j++) { + auto upper = + j < cur_opts.size() ? cur_opts[j].upper() : KeyEntryValue(KeyEntryType::kHighest); + upper.AppendToKey(&new_target); } EXPECT_OK(choices_->SkipTargetsUpTo(new_target)); @@ -360,7 +379,7 @@ TEST_F(ScanChoicesTest, SimplePartialFilterHybridScan) { {{{10_ColId}, QL_OP_IN, {{5}, {6}}}}; const Schema &schema = test_range_schema; - TestOptionIteration(schema, conds, {{{5}, {}}, {{6}, {}}}); + TestOptionIteration(schema, conds, {{{5}}, {{6}}}); } TEST_F(ScanChoicesTest, SimpleMixedFilterHybridScan) { diff --git a/src/yb/docdb/scan_choices.cc b/src/yb/docdb/scan_choices.cc index 25ea82de2a60..8d232a01cb16 100644 --- a/src/yb/docdb/scan_choices.cc +++ b/src/yb/docdb/scan_choices.cc @@ -33,10 +33,11 @@ namespace docdb { bool ScanChoices::CurrentTargetMatchesKey(const Slice& curr) { VLOG(3) << __PRETTY_FUNCTION__ << " checking if acceptable ? " - << (curr == current_scan_target_ ? "YEP" : "NOPE") << ": " - << DocKey::DebugSliceToString(curr) << " vs " + << (!current_scan_target_.empty() && + curr.starts_with(current_scan_target_) ? "YEP" : "NOPE") + << ": " << DocKey::DebugSliceToString(curr) << " vs " << DocKey::DebugSliceToString(current_scan_target_.AsSlice()); - return curr == current_scan_target_; + return !current_scan_target_.empty() && curr.starts_with(current_scan_target_); } HybridScanChoices::HybridScanChoices( @@ -55,6 +56,10 @@ HybridScanChoices::HybridScanChoices( prefix_length_(prefix_length) { size_t num_hash_cols = schema.num_hash_key_columns(); + // Number of dockey columns with specified filters. [kLowest, kHighest] does not + // count as a specified filter. + size_t last_filtered_idx = num_hash_cols - 1; + for (size_t idx = num_hash_cols; idx < schema.num_key_columns(); idx++) { const ColumnId col_id = schema.column_id(idx); std::vector current_options; @@ -82,6 +87,8 @@ HybridScanChoices::HybridScanChoices( col_groups_.BeginNewGroup(); col_groups_.AddToLatestGroup(idx - num_hash_cols); + if (!upper.IsInfinity() || !lower.IsInfinity()) + last_filtered_idx = idx; } else if (col_has_range_option) { auto& options = (*range_options)[idx - num_hash_cols]; current_options.reserve(options.size()); @@ -136,6 +143,7 @@ HybridScanChoices::HybridScanChoices( current_options.emplace_back(last_option, true, last_option, true, begin, current_ind); } + last_filtered_idx = idx; } else { // If no filter is specified, we just impose an artificial range // filter [kLowest, kHighest] @@ -152,6 +160,11 @@ HybridScanChoices::HybridScanChoices( range_cols_scan_options_.push_back(current_options); } + size_t filter_length = std::max(last_filtered_idx - num_hash_cols + 1, prefix_length); + DCHECK_LE(filter_length, range_cols_scan_options_.size()); + + range_cols_scan_options_.resize(filter_length); + current_scan_target_ranges_.resize(range_cols_scan_options_.size()); current_scan_target_.Clear(); for (size_t i = 0; i < range_cols_scan_options_.size(); i++) { @@ -166,6 +179,7 @@ HybridScanChoices::HybridScanChoices( if (!is_forward_scan_) { KeyEntryValue(KeyEntryType::kHighest).AppendToKey(¤t_scan_target_); } + schema_num_keys_ = schema.num_range_key_columns(); } HybridScanChoices::HybridScanChoices( @@ -349,8 +363,8 @@ Result HybridScanChoices::SkipTargetsUpTo(const Slice& new_target) { return false; } - auto lower = current_it->lower(); - auto upper = current_it->upper(); + const auto *lower = ¤t_it->lower(); + const auto *upper = ¤t_it->upper(); bool lower_incl = current_it->lower_inclusive(); bool upper_incl = current_it->upper_inclusive(); @@ -370,7 +384,7 @@ Result HybridScanChoices::SkipTargetsUpTo(const Slice& new_target) { // If it's in range then good, continue after appending the target value // column. - if (lower_cmp_fn(target_value, lower) && upper_cmp_fn(target_value, upper)) { + if (lower_cmp_fn(target_value, *lower) && upper_cmp_fn(target_value, *upper)) { target_value.AppendToKey(¤t_scan_target_); continue; } @@ -412,8 +426,8 @@ Result HybridScanChoices::SkipTargetsUpTo(const Slice& new_target) { // If we are within a range then target_value itself should work - lower = it->lower(); - upper = it->upper(); + lower = &it->lower(); + upper = &it->upper(); lower_incl = it->lower_inclusive(); upper_incl = it->upper_inclusive(); @@ -427,9 +441,9 @@ Result HybridScanChoices::SkipTargetsUpTo(const Slice& new_target) { : [](const KeyEntryValue& t1, const KeyEntryValue& t2) { return t1 < t2; }; - if (target_value >= lower && target_value <= upper) { + if (target_value >= *lower && target_value <= *upper) { target_value.AppendToKey(¤t_scan_target_); - if (lower_cmp_fn(target_value, lower) && upper_cmp_fn(target_value, upper)) { + if (lower_cmp_fn(target_value, *lower) && upper_cmp_fn(target_value, *upper)) { // target_value satisfies the current range condition. // Let's move on. continue; @@ -440,15 +454,15 @@ Result HybridScanChoices::SkipTargetsUpTo(const Slice& new_target) { // If a strict upper bound is broken then we can increment // and move on to the next target - DCHECK(target_value == upper || target_value == lower); + DCHECK(target_value == *upper || target_value == *lower); - if (is_forward_scan_ && target_value == upper) { + if (is_forward_scan_ && target_value == *upper) { RETURN_NOT_OK(IncrementScanTargetAtOptionList(static_cast(option_list_idx))); option_list_idx = current_scan_target_ranges_.size(); break; } - if (!is_forward_scan_ && target_value == lower) { + if (!is_forward_scan_ && target_value == *lower) { RETURN_NOT_OK(IncrementScanTargetAtOptionList(static_cast(option_list_idx))); option_list_idx = current_scan_target_ranges_.size(); break; @@ -470,16 +484,16 @@ Result HybridScanChoices::SkipTargetsUpTo(const Slice& new_target) { // This only works as we are assuming all given ranges are // disjoint. - DCHECK( - (is_forward_scan_ && lower > target_value) || (!is_forward_scan_ && upper < target_value)); + DCHECK((is_forward_scan_ && *lower > target_value) || + (!is_forward_scan_ && *upper < target_value)); if (is_forward_scan_) { - lower.AppendToKey(¤t_scan_target_); + lower->AppendToKey(¤t_scan_target_); if (!lower_incl) { KeyEntryValue(KeyEntryType::kHighest).AppendToKey(¤t_scan_target_); } } else { - upper.AppendToKey(¤t_scan_target_); + upper->AppendToKey(¤t_scan_target_); if (!upper_incl) { KeyEntryValue(KeyEntryType::kLowest).AppendToKey(¤t_scan_target_); } @@ -511,7 +525,6 @@ Result HybridScanChoices::SkipTargetsUpTo(const Slice& new_target) { } } - current_scan_target_.AppendKeyEntryType(KeyEntryType::kGroupEnd); VLOG(2) << "After " << __PRETTY_FUNCTION__ << " current_scan_target_ is " << DocKey::DebugSliceToString(current_scan_target_); return true; @@ -634,7 +647,7 @@ Status HybridScanChoices::IncrementScanTargetAtOptionList(int start_option_list_ decoder.left_input().cdata() - current_scan_target_.AsSlice().cdata()); if (start_with_infinity && - (option_list_idx < static_cast(current_scan_target_ranges_.size()))) { + (option_list_idx < static_cast(schema_num_keys_))) { if (is_forward_scan_) { KeyEntryValue(KeyEntryType::kHighest).AppendToKey(¤t_scan_target_); } else { @@ -685,12 +698,14 @@ std::vector HybridScanChoices::TEST_GetCurrentOptions() { // Method called when the scan target is done being used Status HybridScanChoices::DoneWithCurrentTarget() { - // prev_scan_target_ is necessary for backwards scans - prev_scan_target_ = current_scan_target_; - int incr_idx = - static_cast(prefix_length_ ? prefix_length_ : current_scan_target_ranges_.size()) - 1; - RETURN_NOT_OK(IncrementScanTargetAtOptionList(incr_idx)); - current_scan_target_.AppendKeyEntryType(KeyEntryType::kGroupEnd); + if (schema_num_keys_ == range_cols_scan_options_.size() || + prefix_length_ > 0) { + + int incr_idx = + static_cast(prefix_length_ ? prefix_length_ : current_scan_target_ranges_.size()) - 1; + RETURN_NOT_OK(IncrementScanTargetAtOptionList(incr_idx)); + current_scan_target_.AppendKeyEntryType(KeyEntryType::kGroupEnd); + } // if we we incremented the last index then // if this is a forward scan it doesn't matter what we do @@ -719,21 +734,6 @@ Status HybridScanChoices::DoneWithCurrentTarget() { << " and prev_scan_target_ is " << DocKey::DebugSliceToString(prev_scan_target_); - // The below condition is either indicative of the special case - // where IncrementScanTargetAtOptionList didn't change the target due - // to the case specified in the last section of the - // documentation for IncrementScanTargetAtOptionList or we have exhausted - // all available range keys for the given hash key (indicated - // by is_options_done_) - // We clear the scan target in these cases to indicate that the - // current_scan_target_ has been used and is invalid - // In all other cases, IncrementScanTargetAtOptionList has updated - // current_scan_target_ to the new value that we want to seek to. - // Hence, we shouldn't clear it in those cases - if (prev_scan_target_ == current_scan_target_ || is_options_done_) { - current_scan_target_.Clear(); - } - return Status::OK(); } @@ -763,10 +763,6 @@ Status HybridScanChoices::SeekToCurrentTarget(IntentAwareIteratorIf* db_iter) { VLOG(3) << __PRETTY_FUNCTION__ << " Going to PrevDocKey " << tmp; db_iter->PrevDocKey(tmp); } - } else { - if (!is_forward_scan_ && !prev_scan_target_.empty()) { - db_iter->PrevDocKey(prev_scan_target_); - } } } diff --git a/src/yb/docdb/scan_choices.h b/src/yb/docdb/scan_choices.h index 85ec28829a61..c856d48298ff 100644 --- a/src/yb/docdb/scan_choices.h +++ b/src/yb/docdb/scan_choices.h @@ -342,6 +342,8 @@ class HybridScanChoices : public ScanChoices { ColGroupHolder col_groups_; size_t prefix_length_ = 0; + + size_t schema_num_keys_; }; } // namespace docdb From 0bfe8c444db4c8c7e1207288627f881a3f15701d Mon Sep 17 00:00:00 2001 From: yifanguan Date: Mon, 27 Feb 2023 11:40:03 -0800 Subject: [PATCH 14/81] [#16029] YSQL: Change yb-admin list_tables table type output of colocated parent tables from catalog to parent_table Summary: Before this diff, the table type of colocated parent tables in `yb-admin list_tables include_table_type` command is marked as `catalog`. This is kind of confusing. In this diff, the output table type is changed to `parent_table` for colocated parent tables. Test Plan: Manually test `yb-admin` command with `include_table_type` option. Example: ``` CREATE DATABASE col_db WITH COLOCATION = true; \c col_db CREATE TABLE tbl (k INT); yb-admin --master_addresses=127.0.0.1,127.0.0.2,127.0.0.3 list_tables include_table_type include_db_type include_table_id | grep "colocation.parent.tablename" ysql.col_db.00004000000030008000000000004004.colocation.parent.tablename 00004000000030008000000000004004.colocation.parent.uuid parent_table ``` Reviewers: tverona, fizaa Reviewed By: fizaa Subscribers: yql, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D23194 --- src/yb/master/catalog_manager.cc | 8 ++++++++ src/yb/master/master-test.cc | 7 +++++++ src/yb/master/master_types.proto | 1 + src/yb/tools/yb-admin_client.cc | 5 ++++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/yb/master/catalog_manager.cc b/src/yb/master/catalog_manager.cc index 5ed1ee42a244..afa10f38317e 100644 --- a/src/yb/master/catalog_manager.cc +++ b/src/yb/master/catalog_manager.cc @@ -7266,6 +7266,7 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req, bool include_user_table = has_rel_filter ? false : true; bool include_user_index = has_rel_filter ? false : true; bool include_user_matview = has_rel_filter ? false : true; + bool include_colocated_parent_table = has_rel_filter ? false : true; bool include_system_table = req->exclude_system_tables() ? false : (has_rel_filter ? false : true); @@ -7278,6 +7279,8 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req, include_user_index = true; } else if (relation == MATVIEW_TABLE_RELATION) { include_user_matview = true; + } else if (relation == COLOCATED_PARENT_TABLE_RELATION) { + include_colocated_parent_table = true; } } @@ -7317,6 +7320,11 @@ Status CatalogManager::ListTables(const ListTablesRequestPB* req, continue; } relation_type = USER_TABLE_RELATION; + } else if (table_info->IsColocationParentTable()) { + if (!include_colocated_parent_table || !include_system_table) { + continue; + } + relation_type = COLOCATED_PARENT_TABLE_RELATION; } else { if (!include_system_table) { continue; diff --git a/src/yb/master/master-test.cc b/src/yb/master/master-test.cc index a92ab6ebb4f4..75514fbd4370 100644 --- a/src/yb/master/master-test.cc +++ b/src/yb/master/master-test.cc @@ -455,6 +455,13 @@ TEST_F(MasterTest, TestCatalog) { ASSERT_EQ(0, tables.tables_size()); } + { + ListTablesRequestPB req; + req.add_relation_type_filter(COLOCATED_PARENT_TABLE_RELATION); + DoListTables(req, &tables); + ASSERT_EQ(0, tables.tables_size()); + } + { ListTablesRequestPB req; req.add_relation_type_filter(SYSTEM_TABLE_RELATION); diff --git a/src/yb/master/master_types.proto b/src/yb/master/master_types.proto index ede2787a63f7..56526df96623 100644 --- a/src/yb/master/master_types.proto +++ b/src/yb/master/master_types.proto @@ -24,6 +24,7 @@ enum RelationType { USER_TABLE_RELATION = 2; INDEX_TABLE_RELATION = 3; MATVIEW_TABLE_RELATION = 4; + COLOCATED_PARENT_TABLE_RELATION = 5; } enum SysRowEntryType { diff --git a/src/yb/tools/yb-admin_client.cc b/src/yb/tools/yb-admin_client.cc index 6aaaa4fe5995..eef23432d70f 100644 --- a/src/yb/tools/yb-admin_client.cc +++ b/src/yb/tools/yb-admin_client.cc @@ -1275,7 +1275,10 @@ Status ClusterAdminClient::ListTables(bool include_db_type, str << " index"; break; case master::MATVIEW_TABLE_RELATION: - str << "matview"; + str << " matview"; + break; + case master::COLOCATED_PARENT_TABLE_RELATION: + str << " parent_table"; break; default: str << " other"; From 2df87482588da23652f36fc239ad486d33b57b0b Mon Sep 17 00:00:00 2001 From: Yury Shchetinin Date: Tue, 27 Dec 2022 07:33:35 +0300 Subject: [PATCH 15/81] [PLAT-6626] Add Region to existing provider Summary: 1) Modified bootstrap region code to have an ability to add region with bootstraping 2) Re-organizing addRegion method Test Plan: TODO Reviewers: sb-yb, svarshney Reviewed By: sb-yb, svarshney Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D22149 --- .../devops/opscli/ybops/cloud/aws/cloud.py | 21 ++- .../devops/opscli/ybops/cloud/aws/utils.py | 5 +- .../devops/opscli/ybops/cloud/azure/cloud.py | 20 ++- .../yw/commissioner/tasks/CloudBootstrap.java | 58 ++++--- .../subtasks/cloud/CloudAccessKeySetup.java | 2 - .../com/yugabyte/yw/common/AccessManager.java | 30 +++- .../yugabyte/yw/common/CloudQueryHelper.java | 61 +++++-- .../com/yugabyte/yw/common/DevopsBase.java | 102 ++++------- .../com/yugabyte/yw/common/DnsManager.java | 17 +- .../yugabyte/yw/common/NetworkManager.java | 29 +++- .../com/yugabyte/yw/common/NodeManager.java | 29 ++-- .../yugabyte/yw/common/TemplateManager.java | 7 +- .../handlers/CloudProviderHandler.java | 161 ++++++++---------- .../tasks/CloudBootstrapTest.java | 8 +- .../CloudProviderApiControllerTest.java | 18 -- 15 files changed, 300 insertions(+), 268 deletions(-) diff --git a/managed/devops/opscli/ybops/cloud/aws/cloud.py b/managed/devops/opscli/ybops/cloud/aws/cloud.py index bb86912a2f76..56ef20432696 100644 --- a/managed/devops/opscli/ybops/cloud/aws/cloud.py +++ b/managed/devops/opscli/ybops/cloud/aws/cloud.py @@ -208,7 +208,6 @@ def network_cleanup(self, args): return output def network_bootstrap(self, args): - result = {} # Generate region subset, based on passed in JSON. custom_payload = json.loads(args.custom_payload) per_region_meta = custom_payload.get("perRegionMetadata") @@ -230,11 +229,13 @@ def network_bootstrap(self, args): # per-item creation and x-region connectivity/glue. # # For now, let's leave it as a top-level knob to "do everything" vs "do nothing"... - user_provided_vpc_ids = 0 - for r in per_region_meta.values(): - if r.get("vpcId") is not None: - user_provided_vpc_ids += 1 - if user_provided_vpc_ids > 0 and user_provided_vpc_ids != len(per_region_meta): + added_region_codes = custom_payload.get("addedRegionCodes") + if added_region_codes is None: + added_region_codes = per_region_meta.keys() + + user_provided_vpc_ids = len([r for k, r in per_region_meta.items() + if k in added_region_codes and r.get("vpcId") is not None]) + if user_provided_vpc_ids > 0 and user_provided_vpc_ids != len(added_region_codes): raise YBOpsRuntimeError("Either no regions or all regions must have vpcId specified.") components = {} @@ -245,9 +246,13 @@ def network_bootstrap(self, args): else: # Bootstrap the individual region items standalone (vpc, subnet, sg, RT, etc). for region in metadata_subset: - components[region] = client.bootstrap_individual_region(region) + if region in added_region_codes: + components[region] = client.bootstrap_individual_region(region) + else: + components[region] = YbVpcComponents.from_user_json( + region, per_region_meta.get(region)) # Cross link all the regions together. - client.cross_link_regions(components) + client.cross_link_regions(components, added_region_codes) return {region: c.as_json() for region, c in components.items()} def query_vpc(self, args): diff --git a/managed/devops/opscli/ybops/cloud/aws/utils.py b/managed/devops/opscli/ybops/cloud/aws/utils.py index ec362cbf0e09..c8ab9b8179a3 100644 --- a/managed/devops/opscli/ybops/cloud/aws/utils.py +++ b/managed/devops/opscli/ybops/cloud/aws/utils.py @@ -231,7 +231,7 @@ def bootstrap_individual_region(self, region): region, client.vpc.id, client.sg_yugabyte.id, client.route_table.id, {az: s.id for az, s in client.subnets.items()}) - def cross_link_regions(self, components): + def cross_link_regions(self, components, added_region_codes): # Do the cross linking, adding CIDR entries to RTs and SGs, as well as doing vpc peerings. region_and_vpc_tuples = [(r, c.vpc) for r, c in components.items()] host_vpc = None @@ -243,6 +243,9 @@ def cross_link_regions(self, components): for i in range(len(region_and_vpc_tuples) - 1): i_region, i_vpc = region_and_vpc_tuples[i] for j in range(i + 1, len(region_and_vpc_tuples)): + # skip linking existing regions + if i_region not in added_region_codes and j_region not in added_region_codes: + continue j_region, j_vpc = region_and_vpc_tuples[j] peerings = create_vpc_peering( # i is the host, j is the target. diff --git a/managed/devops/opscli/ybops/cloud/azure/cloud.py b/managed/devops/opscli/ybops/cloud/azure/cloud.py index be1a16d067cc..09666b72484e 100644 --- a/managed/devops/opscli/ybops/cloud/azure/cloud.py +++ b/managed/devops/opscli/ybops/cloud/azure/cloud.py @@ -49,17 +49,20 @@ def network_bootstrap(self, args): # "azToSubnetIds": Dict mapping zones to subnet name # "customSecurityGroupId": (Optional) String representing Azure SG name # if provided by the user - perRegionMetadata = json.loads(args.custom_payload).get("perRegionMetadata") + custom_payload = json.loads(args.custom_payload) + perRegionMetadata = custom_payload.get("perRegionMetadata") # First, make sure the resource group exists. # If not, place it in arbitrary Azure region about to be bootstrapped. create_resource_group(next(iter(perRegionMetadata.keys()))) - user_provided_vnets = 0 + added_region_codes = custom_payload.get("addedRegionCodes") + if added_region_codes is None: + added_region_codes = perRegionMetadata.keys() # Verify that the user provided data - user_provided_vnets = len([r for r in perRegionMetadata.values() - if r.get("vpcId") is not None]) - if user_provided_vnets > 0 and user_provided_vnets != len(perRegionMetadata): + user_provided_vnets = len([r for k, r in perRegionMetadata.items() + if k in added_region_codes and r.get("vpcId") is not None]) + if user_provided_vnets > 0 and user_provided_vnets != len(added_region_codes): raise YBOpsRuntimeError("Either no regions or all regions must have vpcId specified.") components = {} @@ -72,8 +75,11 @@ def network_bootstrap(self, args): logging.info("Bootstrapping individual regions.") # Bootstrap the individual region items standalone (vnet, subnet, sg, RT, etc). for region, metadata in perRegionMetadata.items(): - components[region] = self.get_admin().network(metadata) \ - .bootstrap(region).to_components() + if region in added_region_codes: + components[region] = self.get_admin().network(metadata) \ + .bootstrap(region).to_components() + else: + components[region] = self.get_admin().network(metadata).to_components() self.get_admin().network().peer(components) print(json.dumps(components)) diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrap.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrap.java index 016e3d122adc..d3d445c8bf88 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrap.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrap.java @@ -34,8 +34,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import play.libs.Json; public class CloudBootstrap extends CloudTaskBase { @@ -52,9 +54,10 @@ public static Params fromProvider(Provider provider) { public static Params fromProvider(Provider provider, List regions) { Params taskParams = new Params(); - AccessKey accessKey = null; + // This is the case of initial provider creation. + // If user provides his own access keys, we should take the first one in the list. if (provider.allAccessKeys != null && provider.allAccessKeys.size() > 0) { - accessKey = provider.allAccessKeys.get(0); + AccessKey accessKey = provider.allAccessKeys.get(0); taskParams.keyPairName = accessKey.getKeyInfo().keyPairName; taskParams.sshPrivateKeyContent = accessKey.getKeyInfo().sshPrivateKeyContent; } @@ -244,13 +247,12 @@ public static PerRegionMetadata fromRegion(Region region) { // Dictates whether or not to show the set up NTP option in the provider UI. public boolean showSetUpChrony = true; - // Whether or not task is a pure region add. // This dictates whether the task skips the initialization and bootstrapping of the cloud. - public boolean regionAddOnly = false; - } + public boolean skipBootstrapRegion = false; - // TODO: these fields should probably be persisted with provider but currently these are lost - public static class ProviderTransientData {} + // Whether or not task is a pure region add. + public Set addedRegionCodes = null; + } @Override protected Params taskParams() { @@ -260,29 +262,29 @@ protected Params taskParams() { @Override public void run() { Provider p = Provider.get(taskParams().providerUUID); - if (!taskParams().regionAddOnly) { - if (p.code.equals(Common.CloudType.gcp.toString()) - || p.code.equals(Common.CloudType.aws.toString()) - || p.code.equals(Common.CloudType.azu.toString())) { - createCloudSetupTask() - .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.BootstrappingCloud); - } + Common.CloudType cloudType = Common.CloudType.valueOf(p.code); + if (cloudType.isRequiresBootstrap() + && cloudType != Common.CloudType.onprem + && !taskParams().skipBootstrapRegion) { + createCloudSetupTask() + .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.BootstrappingCloud); + } + Map regionsToInit = + new HashMap<>(taskParams().perRegionMetadata); + if (!CollectionUtils.isEmpty(taskParams().addedRegionCodes)) { + regionsToInit.keySet().retainAll(taskParams().addedRegionCodes); } - taskParams() - .perRegionMetadata - .forEach( - (regionCode, metadata) -> { - createRegionSetupTask(regionCode, metadata) - .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.BootstrappingRegion); - }); - taskParams() - .perRegionMetadata - .forEach( - (regionCode, metadata) -> { - createAccessKeySetupTask(regionCode) - .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.CreateAccessKey); - }); + regionsToInit.forEach( + (regionCode, metadata) -> { + createRegionSetupTask(regionCode, metadata) + .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.BootstrappingRegion); + }); + regionsToInit.forEach( + (regionCode, metadata) -> { + createAccessKeySetupTask(regionCode) + .setSubTaskGroupType(UserTaskDetails.SubTaskGroupType.CreateAccessKey); + }); // Need not to init CloudInitializer task for onprem provider. if (!p.getCloudCode().equals(Common.CloudType.onprem)) { diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/cloud/CloudAccessKeySetup.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/cloud/CloudAccessKeySetup.java index 11865cb00f88..0dbb2eccaa82 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/cloud/CloudAccessKeySetup.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/cloud/CloudAccessKeySetup.java @@ -52,8 +52,6 @@ public void run() { throw new RuntimeException("Region " + regionCode + " not setup."); } AccessManager accessManager = Play.current().injector().instanceOf(AccessManager.class); - - // TODO(bogdan): validation at higher level? String accessKeyCode = Strings.isNullOrEmpty(taskParams().keyPairName) ? AccessKey.getDefaultKeyCode(provider) diff --git a/managed/src/main/java/com/yugabyte/yw/common/AccessManager.java b/managed/src/main/java/com/yugabyte/yw/common/AccessManager.java index 551c57886e63..061c8861a3e0 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/AccessManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/AccessManager.java @@ -396,7 +396,13 @@ public AccessKey addKey( commandArgs.add(privateKeyFilePath); } - JsonNode response = execAndParseCommandRegion(regionUUID, "add-key", commandArgs); + JsonNode response = + execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("add-key") + .commandArgs(commandArgs) + .build()); if (response.has("error")) { throw new PlatformServiceException( INTERNAL_SERVER_ERROR, @@ -465,11 +471,21 @@ public JsonNode createVault(UUID regionUUID, String privateKeyFile) { } commandArgs.add("--private_key_file"); commandArgs.add(privateKeyFile); - return execAndParseCommandRegion(regionUUID, "create-vault", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("create-vault") + .commandArgs(commandArgs) + .build()); } public JsonNode listKeys(UUID regionUUID) { - return execAndParseCommandRegion(regionUUID, "list-keys", Collections.emptyList()); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("list-keys") + .commandArgs(Collections.emptyList()) + .build()); } public JsonNode deleteKey(UUID regionUUID, String keyCode) { @@ -526,7 +542,13 @@ private JsonNode deleteKey( commandArgs.add("--delete_remote"); } commandArgs.add("--ignore_auth_failure"); - JsonNode response = execAndParseCommandRegion(regionUUID, "delete-key", commandArgs); + JsonNode response = + execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("delete-key") + .commandArgs(commandArgs) + .build()); if (response.has("error")) { throw new RuntimeException(response.get("error").asText()); } diff --git a/managed/src/main/java/com/yugabyte/yw/common/CloudQueryHelper.java b/managed/src/main/java/com/yugabyte/yw/common/CloudQueryHelper.java index ef3c12a33eb3..48aa9949ae14 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/CloudQueryHelper.java +++ b/managed/src/main/java/com/yugabyte/yw/common/CloudQueryHelper.java @@ -27,10 +27,10 @@ import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nullable; import javax.inject.Singleton; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import play.libs.Json; -import org.apache.commons.lang3.StringUtils; @Singleton public class CloudQueryHelper extends DevopsBase { @@ -62,9 +62,12 @@ private JsonNode currentHostInfo( commandArgs.add("--metadata_types"); commandArgs.addAll(metadataTypes); } - return parseShellResponse( - execCommand(null, null, cloudType, "current-host", commandArgs, new ArrayList<>()), - "current-host"); + return execAndParseShellResponse( + DevopsCommand.builder() + .cloudType(cloudType) + .command("current-host") + .commandArgs(commandArgs) + .build()); } public List getRegionCodes(Provider p) { @@ -79,7 +82,13 @@ public List getRegionCodes(Provider p) { commandArgs.add(potentialGcpNetwork); } } - JsonNode regionInfo = execAndParseCommandCloud(p.uuid, "regions", commandArgs); + JsonNode regionInfo = + execAndParseShellResponse( + DevopsCommand.builder() + .providerUUID(p.uuid) + .command("regions") + .commandArgs(commandArgs) + .build()); List regionCodes = ImmutableList.of(); if (regionInfo instanceof ArrayNode) { regionCodes = Json.fromJson(regionInfo, List.class); @@ -106,7 +115,12 @@ public JsonNode getZones(UUID regionUUID, String destVpcId, String customPayload commandArgs.add("--custom_payload"); commandArgs.add(customPayload); } - return execAndParseCommandRegion(region.uuid, "zones", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(region.uuid) + .command("zones") + .commandArgs(commandArgs) + .build()); } /* @@ -143,14 +157,24 @@ public JsonNode getInstanceTypes(List regionList, String customPayload) commandArgs.add("--custom_payload"); commandArgs.add(customPayload); } - return execAndParseCommandRegion(regionList.get(0).uuid, "instance_types", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionList.get(0).uuid) + .command("instance_types") + .commandArgs(commandArgs) + .build()); } public JsonNode getMachineImages(UUID providerUUID, Region region) { List commandArgs = new ArrayList<>(); commandArgs.add("--regions"); commandArgs.add(region.code); - return execAndParseCommandCloud(providerUUID, "ami", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .providerUUID(providerUUID) + .command("ami") + .commandArgs(commandArgs) + .build()); } public JsonNode queryVpcs(UUID regionUUID, String arch) { @@ -159,7 +183,12 @@ public JsonNode queryVpcs(UUID regionUUID, String arch) { commandArgs.add("--architecture"); commandArgs.add(arch); } - return execAndParseCommandRegion(regionUUID, "vpc", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("vpc") + .commandArgs(commandArgs) + .build()); } public String getDefaultImage(Region region, String architecture) { @@ -180,7 +209,12 @@ public JsonNode queryImage(UUID regionUUID, String ybImage) { List commandArgs = new ArrayList<>(); commandArgs.add("--machine_image"); commandArgs.add(ybImage); - return execAndParseCommandRegion(regionUUID, "image", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("image") + .commandArgs(commandArgs) + .build()); } public String getImageArchitecture(Region region) { @@ -206,7 +240,12 @@ public String getImageArchitecture(Region region) { public JsonNode queryVnet(UUID regionUUID) { List commandArgs = new ArrayList<>(); - return execAndParseCommandRegion(regionUUID, "vnet", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("vnet") + .commandArgs(commandArgs) + .build()); } public String getVnetOrFail(Region region) { diff --git a/managed/src/main/java/com/yugabyte/yw/common/DevopsBase.java b/managed/src/main/java/com/yugabyte/yw/common/DevopsBase.java index e8e0ca4284ae..3818e7a124a5 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/DevopsBase.java +++ b/managed/src/main/java/com/yugabyte/yw/common/DevopsBase.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import lombok.Builder; import lombok.extern.slf4j.Slf4j; import play.libs.Json; @@ -48,85 +49,34 @@ protected NodeAgentClient getNodeAgentClient() { return nodeAgentClient; } - protected JsonNode parseShellResponse(ShellResponse response, String command) { + protected JsonNode execAndParseShellResponse(DevopsCommand devopsCommand) { + ShellResponse response = execCommand(devopsCommand); if (response.code == 0) { return Json.parse(response.message); } else { String errorMsg = String.format( "YBCloud command %s (%s) failed to execute. %s", - getCommandType(), command, response.message); + getCommandType(), devopsCommand.command, response.message); log.error(errorMsg); return ApiResponse.errorJSON(errorMsg); } } - protected JsonNode execAndParseCommandCloud( - UUID providerUUID, String command, List commandArgs) { - ShellResponse response = - execCommand(null, providerUUID, null, command, commandArgs, Collections.emptyList()); - return parseShellResponse(response, command); - } - - protected JsonNode execAndParseCommandRegion( - UUID regionUUID, String command, List commandArgs) { - ShellResponse response = - execCommand(regionUUID, null, null, command, commandArgs, Collections.emptyList()); - return parseShellResponse(response, command); - } - - protected ShellResponse execCommand( - UUID regionUUID, - UUID providerUUID, - String command, - List commandArgs, - List cloudArgs) { - return execCommand( - regionUUID, providerUUID, null /*cloudType*/, command, commandArgs, cloudArgs); - } - - protected ShellResponse execCommand( - UUID regionUUID, - UUID providerUUID, - Common.CloudType cloudType, - String command, - List commandArgs, - List cloudArgs) { - return execCommand(regionUUID, providerUUID, cloudType, command, commandArgs, cloudArgs, null); - } - - protected ShellResponse execCommand( - UUID regionUUID, - UUID providerUUID, - Common.CloudType cloudType, - String command, - List commandArgs, - List cloudArgs, - Map envVars) { - return execCommand( - regionUUID, providerUUID, cloudType, command, commandArgs, cloudArgs, envVars, null); - } - - protected ShellResponse execCommand( - UUID regionUUID, - UUID providerUUID, - Common.CloudType cloudType, - String command, - List commandArgs, - List cloudArgs, - Map envVars, - Map sensitiveData) { + protected ShellResponse execCommand(DevopsCommand devopsCommand) { List commandList = new ArrayList<>(); commandList.add(YBCLOUD_SCRIPT); Map extraVars = new HashMap<>(); - if (envVars != null) { - extraVars.putAll(envVars); + if (devopsCommand.envVars != null) { + extraVars.putAll(devopsCommand.envVars); } Region region = null; - if (regionUUID != null) { - region = Region.get(regionUUID); + if (devopsCommand.regionUUID != null) { + region = Region.get(devopsCommand.regionUUID); } + List commandArgs = devopsCommand.commandArgs; + if (runtimeConfigFactory.globalRuntimeConf().getBoolean("yb.security.ssh2_enabled")) { commandArgs.add("--ssh2_enabled"); } @@ -142,8 +92,8 @@ protected ShellResponse execCommand( } catch (Exception e) { throw new RuntimeException("Failed to retrieve env variables for the provider!", e); } - } else if (providerUUID != null) { - provider = Provider.get(providerUUID); + } else if (devopsCommand.providerUUID != null) { + provider = Provider.get(devopsCommand.providerUUID); commandList.add(provider.code); try { Map envConfig = CloudInfoInterface.fetchEnvVars(provider); @@ -151,24 +101,36 @@ protected ShellResponse execCommand( } catch (Exception e) { throw new RuntimeException("Failed to retrieve env variables for the provider!", e); } - } else if (cloudType != null) { - commandList.add(cloudType.toString()); + } else if (devopsCommand.cloudType != null) { + commandList.add(devopsCommand.cloudType.toString()); } else { throw new RuntimeException( "Invalid args provided for execCommand: region, provider or cloudType required!"); } String description = String.join(" ", commandList); - description += (" " + getCommandType().toLowerCase() + " " + command); + description += (" " + getCommandType().toLowerCase() + " " + devopsCommand.command); if (commandArgs.size() >= 1) { description += (" " + commandArgs.get(commandArgs.size() - 1)); } - commandList.addAll(cloudArgs); + commandList.addAll(devopsCommand.cloudArgs); commandList.add(getCommandType().toLowerCase()); - commandList.add(command); + commandList.add(devopsCommand.command); commandList.addAll(commandArgs); - return (sensitiveData != null && !sensitiveData.isEmpty()) - ? shellProcessHandler.run(commandList, extraVars, description, sensitiveData) + return (devopsCommand.sensitiveData != null && !devopsCommand.sensitiveData.isEmpty()) + ? shellProcessHandler.run(commandList, extraVars, description, devopsCommand.sensitiveData) : shellProcessHandler.run(commandList, extraVars, description); } + + @Builder + public static class DevopsCommand { + UUID regionUUID; + UUID providerUUID; + Common.CloudType cloudType; + String command; + List commandArgs; + @Builder.Default List cloudArgs = Collections.emptyList(); + Map envVars; + Map sensitiveData; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/DnsManager.java b/managed/src/main/java/com/yugabyte/yw/common/DnsManager.java index 4abdc6431a30..60859d12e751 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/DnsManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/DnsManager.java @@ -45,7 +45,11 @@ public ShellResponse manipulateDnsRecord( commandArgs.add("--node_ips"); commandArgs.add(nodeIpCsv); return execCommand( - null, providerUUID, null, type.toString().toLowerCase(), commandArgs, new ArrayList<>()); + DevopsCommand.builder() + .providerUUID(providerUUID) + .command(type.toString().toLowerCase()) + .commandArgs(commandArgs) + .build()); } public ShellResponse listDnsRecord(UUID providerUUID, String hostedZoneId) { @@ -53,11 +57,10 @@ public ShellResponse listDnsRecord(UUID providerUUID, String hostedZoneId) { commandArgs.add("--hosted_zone_id"); commandArgs.add(hostedZoneId); return execCommand( - null, - providerUUID, - null, - DnsCommandType.List.toString().toLowerCase(), - commandArgs, - new ArrayList<>()); + DevopsCommand.builder() + .providerUUID(providerUUID) + .command(DnsCommandType.List.toString().toLowerCase()) + .commandArgs(commandArgs) + .build()); } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/NetworkManager.java b/managed/src/main/java/com/yugabyte/yw/common/NetworkManager.java index 16a223b2a42c..50fea380eac8 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NetworkManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NetworkManager.java @@ -30,9 +30,19 @@ public JsonNode bootstrap(UUID regionUUID, UUID providerUUID, String customPaylo commandArgs.add(customPayload); if (regionUUID != null) { - return execAndParseCommandRegion(regionUUID, "bootstrap", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("bootstrap") + .commandArgs(commandArgs) + .build()); } else { - return execAndParseCommandCloud(providerUUID, "bootstrap", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .providerUUID(providerUUID) + .command("bootstrap") + .commandArgs(commandArgs) + .build()); } } @@ -40,11 +50,22 @@ public JsonNode query(UUID regionUUID, String customPayload) { List commandArgs = new ArrayList<>(); commandArgs.add("--custom_payload"); commandArgs.add(customPayload); - return execAndParseCommandRegion(regionUUID, "query", commandArgs); + return execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("query") + .commandArgs(commandArgs) + .build()); } public JsonNode cleanupOrFail(UUID regionUUID) { - JsonNode response = execAndParseCommandRegion(regionUUID, "cleanup", Collections.emptyList()); + JsonNode response = + execAndParseShellResponse( + DevopsCommand.builder() + .regionUUID(regionUUID) + .command("cleanup") + .commandArgs(Collections.emptyList()) + .build()); if (response.has("error")) { throw new PlatformServiceException(INTERNAL_SERVER_ERROR, response.get("error").asText()); } diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java index b3c703d32b4f..afc05c188f26 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java @@ -1280,13 +1280,12 @@ public ShellResponse detachedNodeCommand( List cloudArgs = Arrays.asList("--node_metadata", Json.stringify(nodeDetails)); return execCommand( - nodeTaskParam.getRegion().uuid, - null, - null, - type.toString().toLowerCase(), - commandArgs, - cloudArgs, - Collections.emptyMap()); + DevopsCommand.builder() + .regionUUID(nodeTaskParam.getRegion().uuid) + .command(type.toString().toLowerCase()) + .commandArgs(commandArgs) + .cloudArgs(cloudArgs) + .build()); } private Path addBootscript( @@ -2038,14 +2037,14 @@ public ShellResponse nodeCommand(NodeCommandType type, NodeTaskParams nodeTaskPa commandArgs.add(nodeTaskParam.nodeName); try { return execCommand( - nodeTaskParam.getRegion().uuid, - null, - null, - type.toString().toLowerCase(), - commandArgs, - getCloudArgs(nodeTaskParam), - getAnsibleEnvVars(nodeTaskParam.universeUUID), - sensitiveData); + DevopsCommand.builder() + .regionUUID(nodeTaskParam.getRegion().uuid) + .command(type.toString().toLowerCase()) + .commandArgs(commandArgs) + .cloudArgs(getCloudArgs(nodeTaskParam)) + .envVars(getAnsibleEnvVars(nodeTaskParam.universeUUID)) + .sensitiveData(sensitiveData) + .build()); } finally { if (bootScriptFile != null) { try { diff --git a/managed/src/main/java/com/yugabyte/yw/common/TemplateManager.java b/managed/src/main/java/com/yugabyte/yw/common/TemplateManager.java index 94e6315897fe..42de42206ac9 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/TemplateManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/TemplateManager.java @@ -107,7 +107,12 @@ public void createProvisionTemplate( } JsonNode result = - execAndParseCommandCloud(accessKey.getProviderUUID(), "template", commandArgs); + execAndParseShellResponse( + DevopsCommand.builder() + .providerUUID(accessKey.getProviderUUID()) + .command("template") + .commandArgs(commandArgs) + .build()); if (result.get("error") == null) { details.passwordlessSudoAccess = passwordlessSudoAccess; diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java index e0b10c86cd75..e5f3d484cfc1 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java @@ -13,6 +13,7 @@ import static com.yugabyte.yw.commissioner.Common.CloudType.aws; import static com.yugabyte.yw.commissioner.Common.CloudType.gcp; import static com.yugabyte.yw.commissioner.Common.CloudType.kubernetes; +import static com.yugabyte.yw.commissioner.Common.CloudType.onprem; import static com.yugabyte.yw.common.ConfigHelper.ConfigType.DockerInstanceTypeMetadata; import static com.yugabyte.yw.common.ConfigHelper.ConfigType.DockerRegionMetadata; import static play.mvc.Http.Status.BAD_REQUEST; @@ -20,7 +21,6 @@ import static play.mvc.Http.Status.INTERNAL_SERVER_ERROR; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Strings; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Multimap; @@ -69,6 +69,7 @@ import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Secret; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -654,56 +655,15 @@ private Set checkIfRegionsToAdd(Provider editProviderReq, Provider provi Set regionsToAdd = new HashSet<>(); if (provider.getCloudCode().canAddRegions()) { if (editProviderReq.regions != null && !editProviderReq.regions.isEmpty()) { - Set newRegionCodes = - editProviderReq.regions.stream().map(region -> region.code).collect(Collectors.toSet()); + Map newRegions = + editProviderReq.regions.stream().collect(Collectors.toMap(r -> r.code, r -> r)); Set existingRegionCodes = provider.regions.stream().map(region -> region.code).collect(Collectors.toSet()); - newRegionCodes.removeAll(existingRegionCodes); - if (!newRegionCodes.isEmpty()) { - regionsToAdd = - editProviderReq - .regions - .stream() - .filter(region -> newRegionCodes.contains(region.code)) - .collect(Collectors.toSet()); + newRegions.keySet().removeAll(existingRegionCodes); + if (!newRegions.isEmpty()) { + regionsToAdd = new HashSet<>(newRegions.values()); } } - if (!regionsToAdd.isEmpty()) { - // Perform validation for necessary fields - if (provider.getCloudCode() == gcp) { - if (editProviderReq.destVpcId == null) { - throw new PlatformServiceException(BAD_REQUEST, "Required field dest vpc id for GCP"); - } - } - // Verify access key exists. If no value provided, we use the default keycode. - String accessKeyCode; - if (Strings.isNullOrEmpty(editProviderReq.keyPairName)) { - LOG.debug("Access key not specified, using default key code..."); - accessKeyCode = AccessKey.getDefaultKeyCode(provider); - } else { - accessKeyCode = editProviderReq.keyPairName; - } - AccessKey.getOrBadRequest(provider.uuid, accessKeyCode); - - regionsToAdd.forEach( - region -> { - if (region.getVnetName() == null && provider.getCloudCode() == aws) { - throw new PlatformServiceException( - BAD_REQUEST, "Required field vnet name (VPC ID) for region: " + region.code); - } - if (region.zones == null || region.zones.isEmpty()) { - throw new PlatformServiceException( - BAD_REQUEST, "Zone info needs to be specified for region: " + region.code); - } - region.zones.forEach( - zone -> { - if (zone.subnet == null) { - throw new PlatformServiceException( - BAD_REQUEST, "Required field subnet for zone: " + zone.code); - } - }); - }); - } } return regionsToAdd; } @@ -724,9 +684,12 @@ private UUID doEditProvider( CloudInfoInterface.mergeSensitiveFields(provider, editProviderReq); // Check if region edit mode. Set regionsToAdd = checkIfRegionsToAdd(editProviderReq, provider); - boolean providerDataUpdated = - updateProviderData(customer, provider, editProviderReq, regionsToAdd, validate); - UUID taskUUID = maybeAddRegions(customer, editProviderReq, provider, regionsToAdd); + UUID taskUUID = null; + if (!regionsToAdd.isEmpty()) { + // TODO: PLAT-7258 allow adding region for auto-creating VPC case + taskUUID = addRegions(customer, provider, regionsToAdd, true); + } + boolean providerDataUpdated = updateProviderData(customer, provider, editProviderReq, validate); if (!providerDataUpdated && taskUUID == null) { throw new PlatformServiceException( BAD_REQUEST, "No changes to be made for provider type: " + provider.code); @@ -736,11 +699,7 @@ private UUID doEditProvider( @Transactional private boolean updateProviderData( - Customer customer, - Provider provider, - Provider editProviderReq, - Set regionsToAdd, - boolean validate) { + Customer customer, Provider provider, Provider editProviderReq, boolean validate) { Map providerConfig = CloudInfoInterface.fetchEnvVars(editProviderReq); Map existingConfigMap = CloudInfoInterface.fetchEnvVars(provider); boolean updatedProviderDetails = false; @@ -773,40 +732,66 @@ private boolean updateProviderData( return providerDataUpdated; } - private UUID maybeAddRegions( - Customer customer, Provider editProviderReq, Provider provider, Set regionsToAdd) { - if (!regionsToAdd.isEmpty()) { - // Validate regions to add. We only support providing custom VPCs for now. - // So the user must have entered the VPC Info for the regions, as well as - // the zone info. - CloudBootstrap.Params taskParams = new CloudBootstrap.Params(); - taskParams.keyPairName = - Strings.isNullOrEmpty(editProviderReq.keyPairName) - ? AccessKey.getDefaultKeyCode(provider) - : editProviderReq.keyPairName; - taskParams.skipKeyPairValidate = - runtimeConfigFactory.forProvider(provider).getBoolean(SKIP_KEYPAIR_VALIDATION_KEY); - taskParams.providerUUID = provider.uuid; - taskParams.destVpcId = editProviderReq.destVpcId; - taskParams.perRegionMetadata = - regionsToAdd - .stream() - .collect( - Collectors.toMap( - region -> region.name, CloudBootstrap.Params.PerRegionMetadata::fromRegion)); - // Only adding new regions in the bootstrap task. - taskParams.regionAddOnly = true; - UUID taskUUID = commissioner.submit(TaskType.CloudBootstrap, taskParams); - CustomerTask.create( - customer, - provider.uuid, - taskUUID, - CustomerTask.TargetType.Provider, - CustomerTask.TaskType.Update, - provider.name); - return taskUUID; + public UUID addRegions( + Customer customer, Provider provider, Set regionsToAdd, boolean skipBootstrap) { + // Perform validation for necessary fields + if (provider.getCloudCode() == gcp) { + // TODO: Remove once we allow vpc creation for added regions + if (skipBootstrap && provider.destVpcId == null) { + throw new PlatformServiceException(BAD_REQUEST, "Required field dest vpc id for GCP"); + } } - return null; + regionsToAdd.forEach( + region -> { + // TODO: Remove once we allow vpc creation for added regions + if (skipBootstrap && region.getVnetName() == null && provider.getCloudCode() == aws) { + throw new PlatformServiceException( + BAD_REQUEST, "Required field vnet name (VPC ID) for region: " + region.code); + } + if (region.zones == null || region.zones.isEmpty()) { + throw new PlatformServiceException( + BAD_REQUEST, "Zone info needs to be specified for region: " + region.code); + } + region.zones.forEach( + zone -> { + if (zone.subnet == null && provider.getCloudCode() != onprem) { + throw new PlatformServiceException( + BAD_REQUEST, "Required field subnet for zone: " + zone.code); + } + }); + }); + + // Validate regions to add. We only support providing custom VPCs for now. + // So the user must have entered the VPC Info for the regions, as well as + // the zone info. + CloudBootstrap.Params taskParams = new CloudBootstrap.Params(); + // Assuming that at that point we already have at least one AccessKey. + // And we can use actual one. + taskParams.keyPairName = AccessKey.getLatestKey(provider.uuid).getKeyCode(); + taskParams.skipKeyPairValidate = + runtimeConfigFactory.forProvider(provider).getBoolean(SKIP_KEYPAIR_VALIDATION_KEY); + taskParams.providerUUID = provider.uuid; + taskParams.destVpcId = provider.destVpcId; + List allRegions = new ArrayList<>(provider.regions); + allRegions.addAll(regionsToAdd); + taskParams.perRegionMetadata = + allRegions + .stream() + .collect( + Collectors.toMap( + region -> region.name, CloudBootstrap.Params.PerRegionMetadata::fromRegion)); + taskParams.addedRegionCodes = + regionsToAdd.stream().map(r -> r.code).collect(Collectors.toSet()); + taskParams.skipBootstrapRegion = skipBootstrap; + UUID taskUUID = commissioner.submit(TaskType.CloudBootstrap, taskParams); + CustomerTask.create( + customer, + provider.uuid, + taskUUID, + CustomerTask.TargetType.Provider, + CustomerTask.TaskType.Update, + provider.name); + return taskUUID; } private boolean maybeUpdateKubeConfig(Provider provider, Map providerConfig) { diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrapTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrapTest.java index 1bb514f60138..04d2bfc20042 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrapTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CloudBootstrapTest.java @@ -24,11 +24,11 @@ import com.yugabyte.yw.commissioner.Common; import com.yugabyte.yw.common.AccessManager.KeyType; import com.yugabyte.yw.models.AccessKey; +import com.yugabyte.yw.models.AccessKey.KeyInfo; import com.yugabyte.yw.models.AvailabilityZone; import com.yugabyte.yw.models.Provider; import com.yugabyte.yw.models.Region; import com.yugabyte.yw.models.TaskInfo; -import com.yugabyte.yw.models.AccessKey.KeyInfo; import com.yugabyte.yw.models.helpers.CloudInfoInterface; import com.yugabyte.yw.models.helpers.TaskType; import com.yugabyte.yw.models.helpers.provider.region.GCPRegionCloudInfo; @@ -288,9 +288,9 @@ private void validateCloudBootstrapSuccess( } // Check AMI info. if (customImageId) { - assertEquals(r.getYbImage(), metadata.customImageId); + assertEquals(metadata.customImageId, r.getYbImage()); } else { - assertEquals(r.getYbImage(), defaultImage); + assertEquals(defaultImage, r.getYbImage()); } // Check Arch info. assertEquals(r.getArchitecture(), metadata.architecture); @@ -550,7 +550,7 @@ public void testCloudBootstrapAddRegion() throws InterruptedException { eastRegion.subnetId = "us-east1-subnet1"; eastRegion.secondarySubnetId = "us-east1-subnet2"; taskParams.providerUUID = gcpProvider.uuid; - taskParams.regionAddOnly = true; + taskParams.skipBootstrapRegion = true; taskParams.perRegionMetadata.put("us-east1", eastRegion); validateCloudBootstrapSuccess( taskParams, diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderApiControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderApiControllerTest.java index 4dca703e9ab4..62ed008f4c06 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderApiControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderApiControllerTest.java @@ -34,7 +34,6 @@ import static play.mvc.Http.Status.BAD_REQUEST; import static play.test.Helpers.contentAsString; -import com.amazonaws.AmazonServiceException; import com.amazonaws.services.ec2.model.Image; import com.amazonaws.services.ec2.model.IpPermission; import com.amazonaws.services.ec2.model.SecurityGroup; @@ -745,23 +744,6 @@ public void testIncorrectFieldsForAddRegionFail() { assertBadRequest(result, "Required field vnet name (VPC ID) for region: us-west-1"); } - @Test - public void testAddRegionNoAccessKeyFail() { - when(mockCommissioner.submit(any(TaskType.class), any(CloudBootstrap.Params.class))) - .thenReturn(UUID.randomUUID()); - Provider provider = Provider.create(customer.uuid, Common.CloudType.aws, "test"); - String jsonString = - "{\"code\":\"aws\",\"name\":\"test\",\"regions\":[{\"name\":\"us-west-1\"" - + ",\"code\":\"us-west-1\", \"details\": {\"cloudInfo\": { \"aws\": {" - + "\"securityGroupId\":\"sg-foo\" }}}, " - + "\"zones\":[{\"code\":\"us-west-1a\",\"name\":\"us-west-1a\"," - + "\"secondarySubnet\":\"subnet-foo\",\"subnet\":\"subnet-foo\"}]}]}"; - - Result result = - assertPlatformException(() -> editProvider(Json.parse(jsonString), provider.uuid)); - assertBadRequest(result, "KeyCode not found: " + AccessKey.getDefaultKeyCode(provider)); - } - @Test public void testCreateAWSProviderWithInvalidAccessParams() { ObjectNode bodyJson = Json.newObject(); From df6efc876f41b5d7f3c9717acae87e0a17da3552 Mon Sep 17 00:00:00 2001 From: Lingeshwar S Date: Mon, 27 Feb 2023 08:56:08 +0530 Subject: [PATCH 16/81] [PLAT-7545] : Enable feature flag for new UI Summary: Enabling feature flag **//enbaleNewUI=true//** by default Test Plan: Tested Manually Reviewers: asathyan, kkannan Reviewed By: asathyan, kkannan Subscribers: jenkins-bot, ui, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23170 --- managed/ui/src/reducers/feature.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed/ui/src/reducers/feature.js b/managed/ui/src/reducers/feature.js index c290606f9f8e..9cbf4d0c6db3 100644 --- a/managed/ui/src/reducers/feature.js +++ b/managed/ui/src/reducers/feature.js @@ -25,7 +25,7 @@ const initialStateFeatureInTest = { enableNotificationTemplates: false, enableRestore: false, enablePrefillKubeConfig: false, - enableNewUI: false + enableNewUI: true // feature flag to enable new revamped UI }; const initialStateFeatureReleased = { From e090e329226d3326261de1c87a696c09ef48ab3a Mon Sep 17 00:00:00 2001 From: Timur Yusupov Date: Tue, 21 Feb 2023 22:53:52 +0300 Subject: [PATCH 17/81] [#16006] docdb: execute callbacks for local RPC calls the same way as for remote calls Summary: Callbacks for local calls are executed directly in the same thread right from inside the call processing code when it sets successful response. This leads to artificial chains of locks which could be in wrong order and can cause lock cycles and deadlocks. Solved by reusing the same mechanism of executing callbacks for local calls inside a thread pool that as we already have in place for remote calls. Test Plan: `ybd tsan --cxx-test integration-tests_cassandra_cpp_driver-test --gtest_filter CppCassandraDriverTest.ConcurrentIndexUpdate -n 50 -- -p 1` Reviewers: sergei, arybochkin Reviewed By: sergei, arybochkin Subscribers: ybase Differential Revision: https://phabricator.dev.yugabyte.com/D23067 --- src/yb/rpc/local_call.cc | 5 +++-- src/yb/rpc/local_call.h | 3 ++- src/yb/rpc/proxy.cc | 13 ++++++++----- src/yb/rpc/proxy.h | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/yb/rpc/local_call.cc b/src/yb/rpc/local_call.cc index 314d24df2f87..c07340e4716e 100644 --- a/src/yb/rpc/local_call.cc +++ b/src/yb/rpc/local_call.cc @@ -34,10 +34,11 @@ LocalOutboundCall::LocalOutboundCall( const RemoteMethod* remote_method, const shared_ptr& outbound_call_metrics, AnyMessagePtr response_storage, RpcController* controller, - std::shared_ptr rpc_metrics, ResponseCallback callback) + std::shared_ptr rpc_metrics, ResponseCallback callback, + ThreadPool* callback_thread_pool) : OutboundCall(remote_method, outbound_call_metrics, /* method_metrics= */ nullptr, response_storage, controller, std::move(rpc_metrics), std::move(callback), - /* callback_thread_pool= */ nullptr) { + callback_thread_pool) { TRACE_TO(trace_, "LocalOutboundCall"); } diff --git a/src/yb/rpc/local_call.h b/src/yb/rpc/local_call.h index a4e1e1899f65..97e3bdb40c38 100644 --- a/src/yb/rpc/local_call.h +++ b/src/yb/rpc/local_call.h @@ -32,7 +32,8 @@ class LocalOutboundCall : public OutboundCall { LocalOutboundCall(const RemoteMethod* remote_method, const std::shared_ptr& outbound_call_metrics, AnyMessagePtr response_storage, RpcController* controller, - std::shared_ptr rpc_metrics, ResponseCallback callback); + std::shared_ptr rpc_metrics, ResponseCallback callback, + ThreadPool* callback_thread_pool); Status SetRequestParam(AnyMessageConstPtr req, const MemTrackerPtr& mem_tracker) override; diff --git a/src/yb/rpc/proxy.cc b/src/yb/rpc/proxy.cc index 950ceb342c0f..3f05102e83f7 100644 --- a/src/yb/rpc/proxy.cc +++ b/src/yb/rpc/proxy.cc @@ -175,10 +175,12 @@ bool Proxy::PrepareCall(AnyMessageConstPtr req, RpcController* controller) { void Proxy::AsyncLocalCall( const RemoteMethod* method, AnyMessageConstPtr req, AnyMessagePtr resp, - RpcController* controller, ResponseCallback callback) { + RpcController* controller, ResponseCallback callback, + const bool force_run_callback_on_reactor) { controller->call_ = std::make_shared( method, outbound_call_metrics_, resp, controller, context_->rpc_metrics(), - std::move(callback)); + std::move(callback), + GetCallbackThreadPool(force_run_callback_on_reactor, controller->invoke_callback_mode())); if (!PrepareCall(req, controller)) { return; } @@ -204,7 +206,7 @@ void Proxy::AsyncLocalCall( void Proxy::AsyncRemoteCall( const RemoteMethod* method, std::shared_ptr method_metrics, AnyMessageConstPtr req, AnyMessagePtr resp, RpcController* controller, - ResponseCallback callback, bool force_run_callback_on_reactor) { + ResponseCallback callback, const bool force_run_callback_on_reactor) { controller->call_ = std::make_shared( method, outbound_call_metrics_, std::move(method_metrics), resp, controller, context_->rpc_metrics(), std::move(callback), @@ -227,11 +229,12 @@ void Proxy::DoAsyncRequest(const RemoteMethod* method, AnyMessagePtr resp, RpcController* controller, ResponseCallback callback, - bool force_run_callback_on_reactor) { + const bool force_run_callback_on_reactor) { CHECK(controller->call_.get() == nullptr) << "Controller should be reset"; if (call_local_service_) { - AsyncLocalCall(method, req, resp, controller, std::move(callback)); + AsyncLocalCall( + method, req, resp, controller, std::move(callback), force_run_callback_on_reactor); } else { AsyncRemoteCall( method, std::move(method_metrics), req, resp, controller, std::move(callback), diff --git a/src/yb/rpc/proxy.h b/src/yb/rpc/proxy.h index d72b8ac0992b..ea3a93127414 100644 --- a/src/yb/rpc/proxy.h +++ b/src/yb/rpc/proxy.h @@ -182,7 +182,7 @@ class Proxy { void AsyncLocalCall( const RemoteMethod* method, AnyMessageConstPtr req, AnyMessagePtr resp, - RpcController* controller, ResponseCallback callback); + RpcController* controller, ResponseCallback callback, bool force_run_callback_on_reactor); void AsyncRemoteCall( const RemoteMethod* method, std::shared_ptr method_metrics, From 8cec0b0ae3fb429f81ff2466d0468ad1a2dc560b Mon Sep 17 00:00:00 2001 From: Hari Krishna Sunder Date: Tue, 28 Feb 2023 23:07:44 -0800 Subject: [PATCH 18/81] [#15819] StatefulService framework to run services on tserver Summary: Introducing the framework to run StatefulServices on tservers. The ability to tag system tablets with a StatefulServiceKind (#15887) and resolve its location (#15888) have already been checked in. This change adds the base class for all StatefulServices. It registers a callback with ts_tablet_manager to get notified of consensus changes to the service tablet. The callback is processed in a background thread. The service will be activated if the tablet is leader and deactivated if the tablet is not leader. We wait for the leader to get a valid lease before activation. Leader change callbacks are async, and we can loose the lease while still remaining the leader. As a result, all rpc messages processed by this service will be subjected to term checks to ensure that it is leader with a valid lease. STATEFUL_SERVICE_IMPL_METHODS macro wraps the rpc method handler with the term check. Also adding StatefulServiceClientBase which helps create clients for StatefulService. The client handles service location resolution and retries on transient errors. TestEchoService is a sample service that is used for testing the service framework and client. Fixes #15819 Test Plan: stateful_service-itest Reviewers: jhe, pjain, burak_velioglu, slingam, asrivastava Reviewed By: asrivastava Subscribers: tverona, ybase, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D23017 --- src/yb/client/CMakeLists.txt | 3 + .../stateful_service_client_base.cc | 207 +++++++++++ .../stateful_service_client_base.h | 108 ++++++ .../test_echo_service_client.cc | 23 ++ .../test_echo_service_client.h | 30 ++ src/yb/integration-tests/CMakeLists.txt | 1 + src/yb/integration-tests/mini_cluster_base.cc | 5 + src/yb/integration-tests/mini_cluster_base.h | 9 + .../stateful_service-itest.cc | 128 ++++--- src/yb/master/catalog_manager_bg_tasks.cc | 6 + src/yb/server/server_base_options.cc | 2 +- src/yb/server/server_base_options.h | 2 +- src/yb/tserver/CMakeLists.txt | 20 ++ .../stateful_service_base.cc | 333 ++++++++++++++++++ .../stateful_services/stateful_service_base.h | 136 +++++++ .../stateful_services/test_echo_service.cc | 57 +++ .../stateful_services/test_echo_service.h | 38 ++ .../stateful_services/test_echo_service.proto | 34 ++ src/yb/tserver/tablet_server.cc | 14 + src/yb/tserver/ts_tablet_manager.cc | 43 ++- src/yb/tserver/ts_tablet_manager.h | 11 + 21 files changed, 1163 insertions(+), 47 deletions(-) create mode 100644 src/yb/client/stateful_services/stateful_service_client_base.cc create mode 100644 src/yb/client/stateful_services/stateful_service_client_base.h create mode 100644 src/yb/client/stateful_services/test_echo_service_client.cc create mode 100644 src/yb/client/stateful_services/test_echo_service_client.h create mode 100644 src/yb/tserver/stateful_services/stateful_service_base.cc create mode 100644 src/yb/tserver/stateful_services/stateful_service_base.h create mode 100644 src/yb/tserver/stateful_services/test_echo_service.cc create mode 100644 src/yb/tserver/stateful_services/test_echo_service.h create mode 100644 src/yb/tserver/stateful_services/test_echo_service.proto diff --git a/src/yb/client/CMakeLists.txt b/src/yb/client/CMakeLists.txt index 5edefdc58d55..35ed4e64d3ec 100644 --- a/src/yb/client/CMakeLists.txt +++ b/src/yb/client/CMakeLists.txt @@ -54,6 +54,8 @@ set(CLIENT_SRCS permissions.cc session.cc schema.cc + stateful_services/stateful_service_client_base.cc + stateful_services/test_echo_service_client.cc table.cc table_alterer.cc table_creator.cc @@ -80,6 +82,7 @@ set(CLIENT_LIBS server_process tserver_proto tserver_service_proto + test_echo_service_proto tserver_util yb_util gutil diff --git a/src/yb/client/stateful_services/stateful_service_client_base.cc b/src/yb/client/stateful_services/stateful_service_client_base.cc new file mode 100644 index 000000000000..ed7a70f90b4b --- /dev/null +++ b/src/yb/client/stateful_services/stateful_service_client_base.cc @@ -0,0 +1,207 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#include "yb/client/stateful_services/stateful_service_client_base.h" + +#include + +#include "yb/master/master_client.proxy.h" +#include "yb/tserver/tablet_server.h" +#include "yb/rpc/secure_stream.h" +#include "yb/client/client-internal.h" +#include "yb/util/backoff_waiter.h" +#include "yb/util/status_format.h" + +DECLARE_bool(TEST_running_test); +DECLARE_string(certs_dir); + +DEFINE_RUNTIME_int32(stateful_service_operation_timeout_sec, 120, + "The number of seconds after which stateful service operations should timeout."); + +namespace yb { + +using namespace std::chrono_literals; +namespace client { + +StatefulServiceClientBase::StatefulServiceClientBase(StatefulServiceKind service_kind) + : service_kind_(service_kind), service_name_(StatefulServiceKind_Name(service_kind)) {} + +StatefulServiceClientBase::~StatefulServiceClientBase() { Shutdown(); } + +Status StatefulServiceClientBase::Init(tserver::TabletServer* server) { + std::vector addresses; + for (const auto& address : *server->options().GetMasterAddresses()) { + for (const auto& host_port : address) { + addresses.push_back(host_port.ToString()); + } + } + SCHECK(!addresses.empty(), InvalidArgument, "No master address found to StatefulServiceClient."); + + const auto master_addresses = JoinStrings(addresses, ","); + auto local_hosts = server->options().HostsString(); + + std::lock_guard lock(mutex_); + rpc::MessengerBuilder messenger_builder(service_name_ + "_Client"); + secure_context_ = VERIFY_RESULT( + server::SetupInternalSecureContext(local_hosts, *server->fs_manager(), &messenger_builder)); + + messenger_ = VERIFY_RESULT(messenger_builder.Build()); + + if (PREDICT_FALSE(FLAGS_TEST_running_test)) { + std::vector host_ports; + RETURN_NOT_OK(HostPort::ParseStrings(local_hosts, 0 /* default_port */, &host_ports)); + messenger_->TEST_SetOutboundIpBase(VERIFY_RESULT(HostToAddress(host_ports[0].host()))); + } + + master_client_ = VERIFY_RESULT( + yb::client::YBClientBuilder() + .add_master_server_addr(master_addresses) + .default_admin_operation_timeout(FLAGS_stateful_service_operation_timeout_sec * 1s) + .Build(messenger_.get())); + + proxy_cache_ = std::make_unique(messenger_.get()); + + return Status::OK(); +} + +Status StatefulServiceClientBase::TESTInit( + const std::string& local_host, const std::string& master_addresses) { + std::lock_guard lock(mutex_); + rpc::MessengerBuilder messenger_builder(service_name_ + "Client"); + secure_context_ = VERIFY_RESULT(server::SetupSecureContext( + FLAGS_certs_dir, local_host, server::SecureContextType::kInternal, &messenger_builder)); + + messenger_ = VERIFY_RESULT(messenger_builder.Build()); + + if (PREDICT_FALSE(FLAGS_TEST_running_test)) { + messenger_->TEST_SetOutboundIpBase(VERIFY_RESULT(HostToAddress(local_host))); + } + + master_client_ = VERIFY_RESULT( + yb::client::YBClientBuilder() + .add_master_server_addr(master_addresses) + .default_admin_operation_timeout(FLAGS_stateful_service_operation_timeout_sec * 1s) + .Build(messenger_.get())); + + proxy_cache_ = std::make_unique(messenger_.get()); + + return Status::OK(); +} + +namespace { +bool IsRetryableStatus(const Status& status) { + return status.IsTryAgain() || status.IsNetworkError() || status.IsServiceUnavailable(); +} + +const HostPortPB* GetHostPort(master::StatefulServiceInfoPB* info) { + if (!info->broadcast_addresses().empty()) { + return info->mutable_broadcast_addresses(0); + } + if (!info->private_rpc_addresses().empty()) { + return info->mutable_private_rpc_addresses(0); + } + return nullptr; +} +} // namespace + +Status StatefulServiceClientBase::InvokeRpcSync( + const CoarseTimePoint& deadline, + std::function + make_proxy, + std::function + rpc_func) { + CoarseBackoffWaiter waiter(deadline, 1s /* max_wait */, 100ms /* base_delay */); + bool first_run = true; + while (true) { + if (!first_run) { + SCHECK( + waiter.Wait(), TimedOut, "RPC call to $0 timed out after $1 attempt(s)", service_name_, + waiter.attempt()); + } + first_run = false; + + auto proxy_result = GetProxy(make_proxy); + if (!proxy_result.ok()) { + if (IsRetryableStatus(proxy_result.status())) { + // Try again with wait. + VLOG(1) << "Retrying Proxy creation for: " << service_name_ + << " Error: " << proxy_result.status(); + continue; + } + return proxy_result.status(); + } + + rpc::RpcController rpc; + rpc.set_deadline(deadline); + + auto status = rpc_func(proxy_result->get(), &rpc); + if (!status.ok()) { + if (status.IsTryAgain()) { + VLOG(1) << "Retrying RPC call to " << service_name_ << ": " << status; + continue; + } + + const auto* rpc_error = rpc.error_response(); + if (IsRetryableStatus(status) || + (rpc_error && (rpc_error->code() == rpc::ErrorStatusPB::FATAL_SERVER_SHUTTING_DOWN || + rpc_error->code() == rpc::ErrorStatusPB::ERROR_NO_SUCH_SERVICE))) { + VLOG(1) << "Retrying RPC call to " << service_name_ << " with re-resolve: " << status + << (rpc_error ? Format(" Rpc error: $0", rpc_error->code()) : ""); + ResetServiceLocation(); + continue; + } + } + + return status; + } +} + +void StatefulServiceClientBase::ResetServiceLocation() { + std::lock_guard lock(mutex_); + service_hp_.reset(); + proxy_.reset(); +} + +Result> StatefulServiceClientBase::GetProxy( + std::function make_proxy) { + std::lock_guard lock(mutex_); + if (proxy_) { + return proxy_; + } + + if (!service_hp_) { + auto location = VERIFY_RESULT(master_client_->GetStatefulServiceLocation(service_kind_)); + auto* host_port = GetHostPort(&location); + SCHECK( + host_port && host_port->has_host(), IllegalState, "Service host is invalid: $0", + location.DebugString()); + service_hp_ = std::make_shared(HostPort::FromPB(*host_port)); + } + + proxy_.reset(make_proxy(proxy_cache_.get(), *service_hp_)); + return proxy_; +} + +void StatefulServiceClientBase::Shutdown() { + std::lock_guard lock(mutex_); + + if (master_client_) { + master_client_->Shutdown(); + } + + if (messenger_) { + messenger_->Shutdown(); + } +} +} // namespace client +} // namespace yb diff --git a/src/yb/client/stateful_services/stateful_service_client_base.h b/src/yb/client/stateful_services/stateful_service_client_base.h new file mode 100644 index 000000000000..409bbed4ca9d --- /dev/null +++ b/src/yb/client/stateful_services/stateful_service_client_base.h @@ -0,0 +1,108 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#pragma once + +#include +#include "yb/client/client.h" +#include "yb/rpc/messenger.h" +#include "yb/server/secure.h" +#include "yb/util/net/net_util.h" +#include "yb/util/status.h" +#include "yb/rpc/proxy.h" +#include "yb/rpc/rpc_context.h" +#include "yb/gutil/thread_annotations.h" +#include "yb/common/wire_protocol.h" + +using namespace std::placeholders; + +namespace yb { +namespace tserver { +class TabletServer; +} + +namespace client { + +#define STATEFUL_SERVICE_RPC(r, service, method_name) \ + inline Status method_name( \ + CoarseTimePoint deadline, \ + const stateful_service::BOOST_PP_CAT(method_name, RequestPB) & req, \ + stateful_service::BOOST_PP_CAT(method_name, ResponsePB) * resp) { \ + return InvokeRpcSync( \ + deadline, \ + [](rpc::ProxyCache* cache, const HostPort& hp) { \ + return new stateful_service::BOOST_PP_CAT(service, ServiceProxy)(cache, hp); \ + }, \ + [&req, &resp, this](rpc::ProxyBase* proxy, rpc::RpcController* controller) { \ + auto status = static_cast(proxy) \ + ->method_name(req, resp, controller); \ + return ExtractStatus(status, *resp); \ + }); \ + } + +#define STATEFUL_SERVICE_RPCS(service, methods) \ + BOOST_PP_SEQ_FOR_EACH(STATEFUL_SERVICE_RPC, service, methods) + +class StatefulServiceClientBase { + public: + explicit StatefulServiceClientBase(StatefulServiceKind service_kind); + + virtual ~StatefulServiceClientBase(); + + Status Init(tserver::TabletServer* server); + + Status TESTInit( + const std::string& local_host, const std::string& master_addresses); + + void Shutdown(); + + Status InvokeRpcSync( + const CoarseTimePoint& deadline, + std::function make_proxy, + std::function rpc_func); + + protected: + template + Status ExtractStatus(const Status& rpc_status, const RespClass& resp) { + RETURN_NOT_OK(rpc_status); + + if (!resp.has_error()) { + return Status::OK(); + } + + SCHECK_NE(resp.error().code(), AppStatusPB::NOT_FOUND, NotFound, resp.error().message()); + + return StatusFromPB(resp.error()); + } + + private: + void ResetServiceLocation() EXCLUDES(mutex_); + + Result> GetProxy( + std::function make_proxy) + EXCLUDES(mutex_); + + private: + const StatefulServiceKind service_kind_; + const std::string service_name_; + std::unique_ptr secure_context_; + std::unique_ptr messenger_; + std::shared_ptr master_client_; + std::unique_ptr proxy_cache_; + + mutable std::mutex mutex_; + std::shared_ptr service_hp_ GUARDED_BY(mutex_); + std::shared_ptr proxy_ GUARDED_BY(mutex_); +}; +} // namespace client +} // namespace yb diff --git a/src/yb/client/stateful_services/test_echo_service_client.cc b/src/yb/client/stateful_services/test_echo_service_client.cc new file mode 100644 index 000000000000..abbca9e5d278 --- /dev/null +++ b/src/yb/client/stateful_services/test_echo_service_client.cc @@ -0,0 +1,23 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#include "yb/client/stateful_services/test_echo_service_client.h" + +namespace yb { +namespace client { + +TestEchoServiceClient::TestEchoServiceClient() + : StatefulServiceClientBase(StatefulServiceKind::TEST_ECHO) {} + +} // namespace client +} // namespace yb diff --git a/src/yb/client/stateful_services/test_echo_service_client.h b/src/yb/client/stateful_services/test_echo_service_client.h new file mode 100644 index 000000000000..70063de1481f --- /dev/null +++ b/src/yb/client/stateful_services/test_echo_service_client.h @@ -0,0 +1,30 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#pragma once + +#include "yb/client/stateful_services/stateful_service_client_base.h" +#include "yb/tserver/stateful_services/test_echo_service.pb.h" +#include "yb/tserver/stateful_services/test_echo_service.proxy.h" + +namespace yb { +namespace client { + +class TestEchoServiceClient : public StatefulServiceClientBase { + public: + TestEchoServiceClient(); + + STATEFUL_SERVICE_RPCS(TestEcho, (GetEcho)); +}; +} // namespace client +} // namespace yb diff --git a/src/yb/integration-tests/CMakeLists.txt b/src/yb/integration-tests/CMakeLists.txt index 00ccd7f6cb61..4de8e5a85604 100644 --- a/src/yb/integration-tests/CMakeLists.txt +++ b/src/yb/integration-tests/CMakeLists.txt @@ -81,6 +81,7 @@ target_link_libraries(integration-tests yb_pgwrapper master_proto cdc_test_util + test_echo_service_proto ${INTEGRATION_TESTS_LIB_EXTENSIONS}) add_dependencies(integration-tests diff --git a/src/yb/integration-tests/mini_cluster_base.cc b/src/yb/integration-tests/mini_cluster_base.cc index 15109285af5f..e8c8138dad11 100644 --- a/src/yb/integration-tests/mini_cluster_base.cc +++ b/src/yb/integration-tests/mini_cluster_base.cc @@ -15,6 +15,7 @@ #include "yb/client/client.h" +#include "yb/client/stateful_services/stateful_service_client_base.h" #include "yb/server/secure.h" #include "yb/util/net/net_util.h" #include "yb/util/result.h" @@ -75,4 +76,8 @@ Result MiniClusterBase::GetLeaderMasterBoundRpcAddr() { return DoGetLeaderMasterBoundRpcAddr(); } +Status MiniClusterBase::InitStatefulServiceClient(client::StatefulServiceClientBase* client) { + auto host_port = VERIFY_RESULT(GetLeaderMasterBoundRpcAddr()); + return client->TESTInit("127.0.0.52", host_port.ToString()); +} } // namespace yb diff --git a/src/yb/integration-tests/mini_cluster_base.h b/src/yb/integration-tests/mini_cluster_base.h index b1004e7029a3..9a234863308c 100644 --- a/src/yb/integration-tests/mini_cluster_base.h +++ b/src/yb/integration-tests/mini_cluster_base.h @@ -24,6 +24,7 @@ namespace yb { namespace client { class YBClientBuilder; class YBClient; +class StatefulServiceClientBase; } // namespace client // Base class for ExternalMiniCluster and MiniCluster with common interface required by @@ -54,6 +55,13 @@ class MiniClusterBase { bool running() const { return running_.load(std::memory_order_acquire); } + template + Result> CreateStatefulServiceClient() { + auto client = std::make_unique(); + RETURN_NOT_OK(InitStatefulServiceClient(client.get())); + return client; + } + protected: virtual ~MiniClusterBase() = default; @@ -71,6 +79,7 @@ class MiniClusterBase { virtual void ConfigureClientBuilder(client::YBClientBuilder* builder) = 0; virtual Result DoGetLeaderMasterBoundRpcAddr() = 0; + Status InitStatefulServiceClient(client::StatefulServiceClientBase* client); }; } // namespace yb diff --git a/src/yb/integration-tests/stateful_services/stateful_service-itest.cc b/src/yb/integration-tests/stateful_services/stateful_service-itest.cc index 2cbcbf79206a..d7df09463c8d 100644 --- a/src/yb/integration-tests/stateful_services/stateful_service-itest.cc +++ b/src/yb/integration-tests/stateful_services/stateful_service-itest.cc @@ -12,6 +12,7 @@ // #include +#include "yb/client/stateful_services/test_echo_service_client.h" #include "yb/client/yb_table_name.h" #include "yb/client/client-internal.h" #include "yb/integration-tests/cluster_itest_util.h" @@ -31,6 +32,8 @@ #include "yb/tserver/service_util.h" DECLARE_int32(follower_unavailable_considered_failed_sec); +DECLARE_bool(TEST_echo_service_enabled); +DECLARE_string(vmodule); namespace yb { @@ -42,9 +45,11 @@ const client::YBTableName service_table_name( YQL_DATABASE_CQL, master::kSystemNamespaceName, StatefulServiceKind_Name(StatefulServiceKind::TEST_ECHO) + "_table"); -class StatefulServiceTest : public YBMiniClusterTestBase { +class StatefulServiceTest : public MiniClusterTestWithClient { public: void SetUp() override { + FLAGS_TEST_echo_service_enabled = true; + ASSERT_OK(SET_FLAG(vmodule, "stateful_service*=4")); YBMiniClusterTestBase::SetUp(); MiniClusterOptions opts; opts.num_tablet_servers = kNumTServers; @@ -53,12 +58,20 @@ class StatefulServiceTest : public YBMiniClusterTestBase { ASSERT_OK(cluster_->Start()); ASSERT_OK(cluster_->WaitForTabletServerCount(opts.num_tablet_servers)); + + ASSERT_OK(CreateClient()); + ASSERT_OK(client_->WaitForCreateTableToFinish(service_table_name)); + std::vector tablet_ids; + ASSERT_OK(client_->GetTablets(service_table_name, 0 /* max_tablets */, &tablet_ids, NULL)); + ASSERT_EQ(tablet_ids.size(), 1); + tablet_id_.swap(tablet_ids[0]); + ASSERT_OK(cluster_->WaitForLoadBalancerToStabilize(kTimeout)); } - Status VerifyEchoServiceHostedOnAllPeers(const TabletId& tablet_id) { + Status VerifyEchoServiceHostedOnAllPeers() { for (auto& tserver : cluster_->mini_tablet_servers()) { auto initial_peer_tablet = - VERIFY_RESULT(LookupTabletPeer(tserver->server()->tablet_peer_lookup(), tablet_id)); + VERIFY_RESULT(LookupTabletPeer(tserver->server()->tablet_peer_lookup(), tablet_id_)); auto hosted_service = initial_peer_tablet.tablet->metadata()->GetHostedServiceList(); SCHECK_EQ( hosted_service.size(), 1, IllegalState, @@ -70,6 +83,8 @@ class StatefulServiceTest : public YBMiniClusterTestBase { return Status::OK(); } + + TabletId tablet_id_; }; TEST_F(StatefulServiceTest, TestRemoteBootstrap) { @@ -77,18 +92,8 @@ TEST_F(StatefulServiceTest, TestRemoteBootstrap) { 5 * kTimeMultiplier; auto leader_master = ASSERT_RESULT(cluster_->GetLeaderMiniMaster()); - ASSERT_OK(leader_master->master()->catalog_manager_impl()->CreateTestEchoService()); - - auto client = ASSERT_RESULT(cluster_->CreateClient()); - ASSERT_OK(client->WaitForCreateTableToFinish(service_table_name)); - - master::MasterClusterProxy master_proxy(&client->proxy_cache(), leader_master->bound_rpc_addr()); - auto ts_map = ASSERT_RESULT(itest::CreateTabletServerMap(master_proxy, &client->proxy_cache())); - - std::vector tablet_ids; - ASSERT_OK(client->GetTablets(service_table_name, 0 /* max_tablets */, &tablet_ids, NULL)); - ASSERT_EQ(tablet_ids.size(), 1); - auto& tablet_id = tablet_ids[0]; + master::MasterClusterProxy master_proxy(&client_->proxy_cache(), leader_master->bound_rpc_addr()); + auto ts_map = ASSERT_RESULT(itest::CreateTabletServerMap(master_proxy, &client_->proxy_cache())); // Pick a random tserver and shut it down for for 2x the time it takes for a follower to be // considered failed. This will cause it to get remote bootstrapped. @@ -99,20 +104,20 @@ TEST_F(StatefulServiceTest, TestRemoteBootstrap) { // Wait till the peer is removed from quorum. itest::TServerDetails* leader_ts = nullptr; - ASSERT_OK(FindTabletLeader(ts_map, tablet_id, kTimeout, &leader_ts)); - ASSERT_OK( - itest::WaitUntilCommittedConfigNumVotersIs(kNumTServers - 1, leader_ts, tablet_id, kTimeout)); + ASSERT_OK(FindTabletLeader(ts_map, tablet_id_, kTimeout, &leader_ts)); + ASSERT_OK(itest::WaitUntilCommittedConfigNumVotersIs( + kNumTServers - 1, leader_ts, tablet_id_, kTimeout)); // Restart the server and wait for it bootstrap. ASSERT_OK(t_server->Start()); ASSERT_OK( - itest::WaitUntilCommittedConfigNumVotersIs(kNumTServers, leader_ts, tablet_id, kTimeout)); + itest::WaitUntilCommittedConfigNumVotersIs(kNumTServers, leader_ts, tablet_id_, kTimeout)); // Wait for new bootstrapped replica to catch up. ASSERT_OK(WaitFor( [&]() -> Result { auto op_ids = VERIFY_RESULT(itest::GetLastOpIdForEachReplica( - tablet_id, TServerDetailsVector(ts_map), consensus::OpIdType::COMMITTED_OPID, + tablet_id_, TServerDetailsVector(ts_map), consensus::OpIdType::COMMITTED_OPID, kTimeout)); SCHECK_EQ(op_ids.size(), 3, IllegalState, "Expected 3 replicas"); @@ -123,43 +128,31 @@ TEST_F(StatefulServiceTest, TestRemoteBootstrap) { ASSERT_OK(cluster_->WaitForLoadBalancerToStabilize(kTimeout)); // Failover to the rebootstrapped server. - ASSERT_OK(FindTabletLeader(ts_map, tablet_id, kTimeout, &leader_ts)); + ASSERT_OK(FindTabletLeader(ts_map, tablet_id_, kTimeout, &leader_ts)); auto* new_leader = ts_map[t_server->server()->permanent_uuid()].get(); if (leader_ts != new_leader) { - ASSERT_OK(itest::LeaderStepDown(leader_ts, tablet_id, new_leader, kTimeout)); + ASSERT_OK(itest::LeaderStepDown(leader_ts, tablet_id_, new_leader, kTimeout)); } - ASSERT_OK(itest::WaitUntilLeader(new_leader, tablet_id, kTimeout)); + ASSERT_OK(itest::WaitUntilLeader(new_leader, tablet_id_, kTimeout)); - ASSERT_OK(VerifyEchoServiceHostedOnAllPeers(tablet_id)); + ASSERT_OK(VerifyEchoServiceHostedOnAllPeers()); } TEST_F(StatefulServiceTest, TestGetStatefulServiceLocation) { - auto leader_master = ASSERT_RESULT(cluster_->GetLeaderMiniMaster()); - ASSERT_OK(leader_master->master()->catalog_manager_impl()->CreateTestEchoService()); - - auto cluster_client = ASSERT_RESULT(cluster_->CreateClient()); - ASSERT_OK(cluster_client->WaitForCreateTableToFinish(service_table_name)); - - std::vector producer_tablet_ids; - ASSERT_OK(cluster_client->GetTablets( - service_table_name, 0 /* max_tablets */, &producer_tablet_ids, NULL)); - ASSERT_EQ(producer_tablet_ids.size(), 1); - auto& tablet_id = producer_tablet_ids[0]; - // Verify the Hosted service is set on all the replicas. - ASSERT_OK(VerifyEchoServiceHostedOnAllPeers(tablet_id)); + ASSERT_OK(VerifyEchoServiceHostedOnAllPeers()); // Verify GetStatefulServiceLocation returns the correct location. - auto initial_leader = GetLeaderForTablet(cluster_.get(), tablet_id); + auto initial_leader = GetLeaderForTablet(cluster_.get(), tablet_id_); auto location = - ASSERT_RESULT(cluster_client->GetStatefulServiceLocation(StatefulServiceKind::TEST_ECHO)); + ASSERT_RESULT(client_->GetStatefulServiceLocation(StatefulServiceKind::TEST_ECHO)); ASSERT_EQ(location.permanent_uuid(), initial_leader->server()->permanent_uuid()); initial_leader->Shutdown(); ASSERT_OK(WaitFor( [&]() -> Result { - auto leader = GetLeaderForTablet(cluster_.get(), tablet_id); + auto leader = GetLeaderForTablet(cluster_.get(), tablet_id_); return leader != nullptr; }, kTimeout, "Wait for new leader")); @@ -167,14 +160,63 @@ TEST_F(StatefulServiceTest, TestGetStatefulServiceLocation) { ASSERT_OK(cluster_->WaitForLoadBalancerToStabilize(kTimeout)); // Verify GetStatefulServiceLocation returns the correct location again. - auto final_leader = GetLeaderForTablet(cluster_.get(), tablet_id); + auto final_leader = GetLeaderForTablet(cluster_.get(), tablet_id_); ASSERT_NE(final_leader, initial_leader); - location = - ASSERT_RESULT(cluster_client->GetStatefulServiceLocation(StatefulServiceKind::TEST_ECHO)); + location = ASSERT_RESULT(client_->GetStatefulServiceLocation(StatefulServiceKind::TEST_ECHO)); ASSERT_EQ(location.permanent_uuid(), final_leader->server()->permanent_uuid()); ASSERT_OK(initial_leader->Start()); } +TEST_F(StatefulServiceTest, TestEchoService) { + auto service_client = + ASSERT_RESULT(cluster_->CreateStatefulServiceClient()); + + stateful_service::GetEchoRequestPB req; + stateful_service::GetEchoResponsePB resp; + req.set_message("Hello World!"); + + ASSERT_OK(service_client->GetEcho(CoarseMonoClock::Now() + kTimeout, req, &resp)); + + ASSERT_EQ(resp.message(), "Hello World! World! World!"); + auto initial_node_id = resp.node_id(); + + // Make sure the tablet leader is the one serving the request. + auto initial_leader = GetLeaderForTablet(cluster_.get(), tablet_id_); + auto initial_leader_uuid = initial_leader->server()->permanent_uuid(); + ASSERT_EQ(resp.node_id(), initial_leader_uuid); + + initial_leader->Shutdown(); + + resp.Clear(); + req.set_message("Hungry shark doo"); + ASSERT_OK(service_client->GetEcho(CoarseMonoClock::Now() + kTimeout, req, &resp)); + + ASSERT_EQ(resp.message(), "Hungry shark doo doo doo"); + ASSERT_NE(resp.node_id(), initial_leader_uuid); + + ASSERT_OK(WaitFor( + [&]() -> Result { + auto leader = GetLeaderForTablet(cluster_.get(), tablet_id_); + return leader != nullptr; + }, + kTimeout, "Wait for new leader")); + + ASSERT_OK(cluster_->WaitForLoadBalancerToStabilize(kTimeout)); + + auto final_leader = GetLeaderForTablet(cluster_.get(), tablet_id_); + auto final_leader_uuid = final_leader->server()->permanent_uuid(); + ASSERT_NE(final_leader_uuid, initial_leader_uuid); + + resp.Clear(); + req.set_message("Anybody there?"); + ASSERT_OK(service_client->GetEcho(CoarseMonoClock::Now() + kTimeout, req, &resp)); + + // Make sure the new tablet leader is the one serving the request. + ASSERT_EQ(resp.message(), "Anybody there? there? there?"); + ASSERT_EQ(resp.node_id(), final_leader_uuid); + + ASSERT_OK(initial_leader->Start()); +} } // namespace yb diff --git a/src/yb/master/catalog_manager_bg_tasks.cc b/src/yb/master/catalog_manager_bg_tasks.cc index bc2011cd3313..a0fc6ad6e792 100644 --- a/src/yb/master/catalog_manager_bg_tasks.cc +++ b/src/yb/master/catalog_manager_bg_tasks.cc @@ -67,6 +67,7 @@ DEFINE_test_flag(bool, pause_catalog_manager_bg_loop_end, false, "Pause the bg tasks thread at the end of the loop."); DECLARE_bool(enable_ysql); +DECLARE_bool(TEST_echo_service_enabled); namespace yb { namespace master { @@ -167,6 +168,11 @@ void CatalogManagerBgTasks::Run() { } } + if (FLAGS_TEST_echo_service_enabled) { + WARN_NOT_OK( + catalog_manager_->CreateTestEchoService(), "Failed to create Test Echo service"); + } + // Report metrics. catalog_manager_->ReportMetrics(); diff --git a/src/yb/server/server_base_options.cc b/src/yb/server/server_base_options.cc index 7eb86e2c3282..f2888b91a618 100644 --- a/src/yb/server/server_base_options.cc +++ b/src/yb/server/server_base_options.cc @@ -166,7 +166,7 @@ WebserverOptions& ServerBaseOptions::CompleteWebserverOptions() { return webserver_opts; } -std::string ServerBaseOptions::HostsString() { +std::string ServerBaseOptions::HostsString() const { return !server_broadcast_addresses.empty() ? server_broadcast_addresses : rpc_opts.rpc_bind_addresses; } diff --git a/src/yb/server/server_base_options.h b/src/yb/server/server_base_options.h index 72158e38e6fe..3c4090e81566 100644 --- a/src/yb/server/server_base_options.h +++ b/src/yb/server/server_base_options.h @@ -102,7 +102,7 @@ class ServerBaseOptions { WebserverOptions& CompleteWebserverOptions(); - std::string HostsString(); + std::string HostsString() const; protected: explicit ServerBaseOptions(int default_port); diff --git a/src/yb/tserver/CMakeLists.txt b/src/yb/tserver/CMakeLists.txt index 3e5695b7a815..fa3aa5e2a214 100644 --- a/src/yb/tserver/CMakeLists.txt +++ b/src/yb/tserver/CMakeLists.txt @@ -150,6 +150,24 @@ ADD_YB_LIBRARY(tserver_service_proto DEPS ${TSERVER_YRPC_LIBS} NONLINK_DEPS ${TSERVER_YRPC_TGTS}) +######################################### +# test_echo_service_proto +######################################### + +YRPC_GENERATE( + TSERVER_YRPC_SRCS TSERVER_YRPC_HDRS TSERVER_YRPC_TGTS + SOURCE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../.. + BINARY_ROOT ${CMAKE_CURRENT_BINARY_DIR}/../.. + PROTO_FILES stateful_services/test_echo_service.proto) +set(TSERVER_YRPC_LIBS + yrpc + yb_common_proto + protobuf) +ADD_YB_LIBRARY(test_echo_service_proto + SRCS ${TSERVER_YRPC_SRCS} + DEPS ${TSERVER_YRPC_LIBS} + NONLINK_DEPS ${TSERVER_YRPC_TGTS}) + ######################################### # tserver_util ######################################### @@ -198,6 +216,8 @@ set(TSERVER_SRCS tserver-path-handlers.cc tserver_metrics_heartbeat_data_provider.cc server_main_util.cc + stateful_services/stateful_service_base.cc + stateful_services/test_echo_service.cc xcluster_safe_time_map.cc cdc_consumer.cc cdc_poller.cc diff --git a/src/yb/tserver/stateful_services/stateful_service_base.cc b/src/yb/tserver/stateful_services/stateful_service_base.cc new file mode 100644 index 000000000000..10f9b14de754 --- /dev/null +++ b/src/yb/tserver/stateful_services/stateful_service_base.cc @@ -0,0 +1,333 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#include "yb/tserver/stateful_services/stateful_service_base.h" + +#include + +#include "yb/consensus/consensus.h" +#include "yb/gutil/bind.h" +#include "yb/gutil/bind_helpers.h" +#include "yb/util/backoff_waiter.h" +#include "yb/tserver/ts_tablet_manager.h" +#include "yb/util/logging.h" +#include "yb/util/unique_lock.h" + +using namespace std::chrono_literals; + +DEFINE_RUNTIME_uint32(stateful_service_periodic_task_interval_ms, 10000, + "Frequency of the stateful service periodic task. 0 indicates that the task should not run."); + +namespace yb { +using tablet::TabletPeerPtr; + +namespace stateful_service { + +#define RETURN_IF_SHUTDOWN \ + do { \ + if (shutdown_) { \ + return; \ + } \ + } while (0) + +#define LOG_TASK_IDLE_AND_RETURN_IF(condition, reason) \ + do { \ + if (condition) { \ + VLOG(1) << name_ << " periodic task entering idle mode due to: " << reason; \ + return; \ + } \ + } while (0) + +#define LOG_TASK_IDLE_AND_RETURN_IF_SHUTDOWN LOG_TASK_IDLE_AND_RETURN_IF(shutdown_, "Shutdown") + +StatefulServiceBase::StatefulServiceBase(const StatefulServiceKind service_kind) + : name_(StatefulServiceKind_Name(service_kind) + "_Service"), service_kind_(service_kind) {} + +StatefulServiceBase::~StatefulServiceBase() { + LOG_IF(DFATAL, !shutdown_) << "Shutdown was not called for " << name_; +} + +Status StatefulServiceBase::Init(tserver::TSTabletManager* ts_manager) { + std::lock_guard lock(task_enqueue_mutex_); + auto thread_pool_builder = ThreadPoolBuilder(name_ + "PeriodicTask"); + thread_pool_builder.set_max_threads(1); + + RETURN_NOT_OK_PREPEND( + thread_pool_builder.Build(&thread_pool_), + Format("Failed to create thread pool for $0", name_)); + + thread_pool_token_ = thread_pool_->NewToken(ThreadPool::ExecutionMode::SERIAL); + + task_interval_ms_flag_callback_reg_ = VERIFY_RESULT(RegisterFlagUpdateCallback( + &FLAGS_stateful_service_periodic_task_interval_ms, name_, + [this]() { StartPeriodicTaskIfNeeded(); })); + + return ts_manager->RegisterServiceCallback( + service_kind(), Bind(&StatefulServiceBase::RaftConfigChangeCallback, Unretained(this))); +} + +void StatefulServiceBase::Shutdown() { + { + std::lock_guard lock(service_state_mutex_); + if (shutdown_) { + return; + } + + // ProcessTaskPeriodically checks for shutdown under this lock before waiting on + // task_wait_cond_. + shutdown_ = true; + } + + task_wait_cond_.notify_all(); + + { + std::lock_guard lock(task_enqueue_mutex_); + if (thread_pool_token_) { + thread_pool_token_->Shutdown(); + } + + if (thread_pool_) { + thread_pool_->Shutdown(); + } + } + + task_interval_ms_flag_callback_reg_.Deregister(); + + // Wait for any activation tasks to finish before deactivating. + DoDeactivate(); +} + +void StatefulServiceBase::RaftConfigChangeCallback(TabletPeerPtr peer) { + CHECK_NOTNULL(peer); + RETURN_IF_SHUTDOWN; + + // We cannot perform intensive work or waits in the callback thread. Store the necessary info and + // schedule task in our own thread pool. + { + std::lock_guard lock(service_state_mutex_); + new_tablet_peer_ = peer; + raft_config_changed_ = true; + } + + // Schedule task and run it immediately. + StartPeriodicTaskIfNeeded(); +} + +bool StatefulServiceBase::IsActive() const { + SharedLock lock(service_state_mutex_); + return leader_term_ != OpId::kUnknownTerm; +} + +int64_t StatefulServiceBase::GetLeaderTerm() { + TabletPeerPtr tablet_peer; + int64_t leader_term; + { + SharedLock lock(service_state_mutex_); + if (leader_term_ == OpId::kUnknownTerm) { + return leader_term_; + } + tablet_peer = tablet_peer_; + leader_term = leader_term_; + } + + // Make sure term and lease are still valid. + if (tablet_peer->LeaderTerm() == leader_term) { + return leader_term; + } + + return OpId::kUnknownTerm; +} + +void StatefulServiceBase::DoDeactivate() { + { + std::lock_guard lock(service_state_mutex_); + if (leader_term_ == OpId::kUnknownTerm) { + return; + } + leader_term_ = OpId::kUnknownTerm; + } + + LOG(INFO) << "Deactivating " << name(); + Deactivate(); +} + +void StatefulServiceBase::ActivateOrDeactivateServiceIfNeeded() { + RETURN_IF_SHUTDOWN; + + TabletPeerPtr new_tablet_peer; + { + std::lock_guard lock(service_state_mutex_); + if (!raft_config_changed_) { + return; + } + raft_config_changed_ = false; + new_tablet_peer.swap(new_tablet_peer_); + } + + // If we lost the lease, or new leader was picked during the call to WaitForLeaderLeaseAndGetTerm + // then we may get kUnknownTerm. RaftConfigChangeCallback will scheduled the next work if we + // become leader again. + int64_t new_leader_term = WaitForLeaderLeaseAndGetTerm(new_tablet_peer); + + RETURN_IF_SHUTDOWN; + + { + SharedLock lock(service_state_mutex_); + if (leader_term_ == new_leader_term && tablet_peer_ == new_tablet_peer) { + // No change in local peers state. + return; + } + } + + // Always deactivate even if we are reactivating. Since the term changed a different node could + // have become active and changed the state. + DoDeactivate(); + + if (new_leader_term != OpId::kUnknownTerm) { + LOG(INFO) << "Activating " << name() << " on term " << new_leader_term; + Activate(new_leader_term); + + // Only after Activate completes we can set the leader_term_ and serve user requests. + std::lock_guard lock(service_state_mutex_); + leader_term_ = new_leader_term; + tablet_peer_ = new_tablet_peer; + } +} + +int64_t StatefulServiceBase::WaitForLeaderLeaseAndGetTerm(TabletPeerPtr tablet_peer) { + if (!tablet_peer) { + return OpId::kUnknownTerm; + } + + const auto& tablet_id = tablet_peer->tablet_id(); + auto consensus = tablet_peer->shared_consensus(); + if (!consensus) { + VLOG(1) << name() << " Received notification of tablet leader change " + << "but tablet no longer running. Tablet ID: " << tablet_id; + return OpId::kUnknownTerm; + } + + auto leader_status = consensus->CheckIsActiveLeaderAndHasLease(); + // The possible outcomes are: + // OK: We are leader and have a lease. + // LeaderNotReadyToServe, LeaderHasNoLease: We are leader but don't have a lease yet. + // IllegalState: We are not leader. + if (leader_status.ok() || leader_status.IsLeaderHasNoLease() || + leader_status.IsLeaderNotReadyToServe()) { + VLOG_WITH_FUNC(1) << name() << " started waiting for leader lease of tablet " << tablet_id; + CoarseBackoffWaiter waiter(CoarseTimePoint::max(), 1s /* max_wait */, 100ms /* base_delay */); + while (!shutdown_) { + auto status = + consensus->WaitForLeaderLeaseImprecise(CoarseMonoClock::now() + waiter.DelayForNow()); + waiter.NextAttempt(); + + // The possible outcomes are: + // OK: Peer acquired the lease. + // TimedOut: Peer is still the leader but still waiting for a lease. + // IllegalState/Other errors: Peer is no longer the leader. + if (status.ok()) { + auto term = consensus->LeaderTerm(); + if (term != OpId::kUnknownTerm) { + VLOG_WITH_FUNC(1) << name() << " completed waiting for leader lease of tablet " + << tablet_id << " with term " << term; + return term; + } + // We either lost the lease or the leader changed. Go back to + // WaitForLeaderLeaseImprecise to see which one it was. + } else if (!status.IsTimedOut()) { + LOG_WITH_FUNC(WARNING) << name() << " failed waiting for leader lease of tablet " + << tablet_id << ": " << status; + return OpId::kUnknownTerm; + } + + YB_LOG_EVERY_N_SECS(INFO, 10) + << name() << " waiting for new leader of tablet " << tablet_id << " to acquire the lease"; + } + VLOG_WITH_FUNC(1) << name() << " completed waiting for leader lease of tablet " << tablet_id + << " due to shutdown"; + } else { + VLOG_WITH_FUNC(1) << name() << " tablet " << tablet_id << " is a follower"; + } + + return OpId::kUnknownTerm; +} + +void StatefulServiceBase::StartPeriodicTaskIfNeeded() { + LOG_TASK_IDLE_AND_RETURN_IF_SHUTDOWN; + + { + std::lock_guard lock(task_enqueue_mutex_); + if (task_enqueued_) { + return; + } + + CHECK(thread_pool_token_) << "Init must be called before starting the task"; + + // It is ok to schedule a new task even when we have a running task. The thread pool token uses + // serial execution, and the task will sleep before returning. So it is always guaranteed that + // we only run one task at a time and that it will wait the required amount before running + // again. The advantage of this model is that we dont hold on to the thread when there is no + // work to do. + task_enqueued_ = true; + Status s = thread_pool_token_->SubmitFunc( + std::bind(&StatefulServiceBase::ProcessTaskPeriodically, this)); + if (!s.ok()) { + task_enqueued_ = false; + LOG(ERROR) << "Failed to schedule " << name_ << " periodic task :" << s; + } + } + + // Wake up the thread if its sleeping and process the task immediately. + task_wait_cond_.notify_all(); +} + +uint32 StatefulServiceBase::PeriodicTaskIntervalMs() const { + return GetAtomicFlag(&FLAGS_stateful_service_periodic_task_interval_ms); +} + +void StatefulServiceBase::ProcessTaskPeriodically() { + { + std::lock_guard lock(task_enqueue_mutex_); + task_enqueued_ = false; + } + + LOG_TASK_IDLE_AND_RETURN_IF_SHUTDOWN; + + ActivateOrDeactivateServiceIfNeeded(); + + LOG_TASK_IDLE_AND_RETURN_IF(!IsActive(), "Service is no longer active on this node"); + + auto wait_time_ms = PeriodicTaskIntervalMs(); + LOG_TASK_IDLE_AND_RETURN_IF(wait_time_ms == 0, "Task interval is 0"); + + auto further_computation_needed_result = RunPeriodicTask(); + + if (!further_computation_needed_result.ok()) { + LOG_WITH_FUNC(WARNING) << name_ << " periodic task failed: " + << further_computation_needed_result.status(); + } else { + LOG_TASK_IDLE_AND_RETURN_IF(!further_computation_needed_result.get(), "No more work left"); + } + + // Delay before running the task again. + { + UniqueLock lock(service_state_mutex_); + LOG_TASK_IDLE_AND_RETURN_IF_SHUTDOWN; + task_wait_cond_.wait_for(GetLockForCondition(&lock), wait_time_ms * 1ms); + } + + StartPeriodicTaskIfNeeded(); +} + +} // namespace stateful_service +} // namespace yb diff --git a/src/yb/tserver/stateful_services/stateful_service_base.h b/src/yb/tserver/stateful_services/stateful_service_base.h new file mode 100644 index 000000000000..40b80b4c36ca --- /dev/null +++ b/src/yb/tserver/stateful_services/stateful_service_base.h @@ -0,0 +1,136 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#pragma once + +#include +#include +#include "yb/tablet/metadata.pb.h" +#include "yb/tablet/tablet_peer.h" +#include "yb/util/result.h" +#include "yb/rpc/rpc_context.h" +#include "yb/common/wire_protocol.h" + +namespace yb { + +namespace tserver { +class TSTabletManager; +} + +namespace stateful_service { + +// Macro to ensure stateful service rpc methods are only served when the service is ready. +// This requires the service to be in active mode and the serving tablet peer to be a leader and +// have the lease. resp->mutable_error() will be set if the service is not ready. +#define STATEFUL_SERVICE_IMPL_METHODS(methods) \ + BOOST_PP_SEQ_FOR_EACH(STATEFUL_SERVICE_IMPL_METHOD_HELPER, ~, methods) + +#define STATEFUL_SERVICE_IMPL_METHOD_HELPER(i, data, method_name) \ + Status BOOST_PP_CAT(method_name, Impl)( \ + const BOOST_PP_CAT(method_name, RequestPB) & req, \ + BOOST_PP_CAT(method_name, ResponsePB) * resp); \ + void method_name( \ + const BOOST_PP_CAT(method_name, RequestPB) * req, \ + BOOST_PP_CAT(method_name, ResponsePB) * resp, rpc::RpcContext rpc) override { \ + HandleMessageWithTermCheck( \ + resp, &rpc, [req, resp, this]() { return BOOST_PP_CAT(method_name, Impl)(*req, resp); }); \ + } + +class StatefulServiceBase { + public: + explicit StatefulServiceBase(const StatefulServiceKind service_kind); + + virtual ~StatefulServiceBase(); + + Status Init(tserver::TSTabletManager* ts_manager); + + void Shutdown() EXCLUDES(service_state_mutex_); + + void RaftConfigChangeCallback(tablet::TabletPeerPtr peer) EXCLUDES(service_state_mutex_); + + const std::string& name() const { return name_; } + + StatefulServiceKind service_kind() const { return service_kind_; } + + bool IsActive() const EXCLUDES(service_state_mutex_); + + void StartPeriodicTaskIfNeeded() EXCLUDES(service_state_mutex_); + + protected: + // Hosting tablet peer has become a leader. RPC messages will only be processed after this + // function completes. + virtual void Activate(const int64_t leader_term) = 0; + + // Hosting tablet peer has stepped down to a follower. Release all resources acquired by the + // stateful services and clear in-mem data. + virtual void Deactivate() = 0; + + // Periodic task to be executed by the stateful service. Return true if the task should be rerun + // after PeriodicTaskIntervalMs. + virtual Result RunPeriodicTask() { return false; } + + // Interval in milliseconds between periodic tasks. 0 indicates that the task should not run + // again. + virtual uint32 PeriodicTaskIntervalMs() const; + + // Get the term when we last activated and make sure we still have a valid lease. + int64_t GetLeaderTerm() EXCLUDES(service_state_mutex_); + + template + void HandleMessageWithTermCheck( + ResponsePB* resp, rpc::RpcContext* rpc, std::function method_impl) { + // Will return a valid term only if we are active and have a valid lease. + const auto term = GetLeaderTerm(); + Status status; + if (term != OpId::kUnknownTerm) { + status = method_impl(); + } + + // Term should still be valid after the method_impl() call. + if (term == OpId::kUnknownTerm || term != GetLeaderTerm()) { + status = STATUS(ServiceUnavailable, Format(name() + " is not active on this server")); + } + + if (!status.ok()) { + resp->Clear(); + StatusToPB(status, resp->mutable_error()); + } + rpc->RespondSuccess(); + } + + private: + void ProcessTaskPeriodically() EXCLUDES(service_state_mutex_); + void ActivateOrDeactivateServiceIfNeeded() EXCLUDES(service_state_mutex_); + int64_t WaitForLeaderLeaseAndGetTerm(tablet::TabletPeerPtr tablet_peer); + void DoDeactivate(); + + const std::string name_; + const StatefulServiceKind service_kind_; + + std::atomic_bool shutdown_ = false; + mutable std::shared_mutex service_state_mutex_; + int64_t leader_term_ GUARDED_BY(service_state_mutex_) = OpId::kUnknownTerm; + tablet::TabletPeerPtr tablet_peer_ GUARDED_BY(service_state_mutex_); + tablet::TabletPeerPtr new_tablet_peer_ GUARDED_BY(service_state_mutex_); + bool raft_config_changed_ GUARDED_BY(service_state_mutex_) = false; + + std::mutex task_enqueue_mutex_; + bool task_enqueued_ GUARDED_BY(task_enqueue_mutex_) = false; + std::condition_variable_any task_wait_cond_; + std::unique_ptr thread_pool_; + std::unique_ptr thread_pool_token_; + + FlagCallbackRegistration task_interval_ms_flag_callback_reg_; +}; +} // namespace stateful_service +} // namespace yb diff --git a/src/yb/tserver/stateful_services/test_echo_service.cc b/src/yb/tserver/stateful_services/test_echo_service.cc new file mode 100644 index 000000000000..048d851e85b1 --- /dev/null +++ b/src/yb/tserver/stateful_services/test_echo_service.cc @@ -0,0 +1,57 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#include "yb/tserver/stateful_services/test_echo_service.h" + +namespace yb { + +namespace stateful_service { +TestEchoService::TestEchoService( + const std::string& node_uuid, const scoped_refptr& metric_entity) + : StatefulServiceBase(StatefulServiceKind::TEST_ECHO), + TestEchoServiceIf(metric_entity), + node_uuid_(node_uuid) {} + +void TestEchoService::Activate(const int64_t leader_term) { + LOG(INFO) << "Test Echo service activated on term: " << leader_term; +} + +void TestEchoService::Deactivate() { LOG(INFO) << "Test Echo service de-activated"; } + +Result TestEchoService::RunPeriodicTask() { + LOG(INFO) << "Test Echo service Running"; + return true; +} + +void TestEchoService::Shutdown() { + TestEchoServiceIf::Shutdown(); + StatefulServiceBase::Shutdown(); +} + +Status TestEchoService::GetEchoImpl(const GetEchoRequestPB& req, GetEchoResponsePB* resp) { + std::string echo = req.message(); + // For a string to bounce back and make an echo, there has to be a lot of latency between the + // string source and the thing (wall or mountain or service) that it hits and bounces back. Since + // latency in Yugabyte is very low we need to do some string manipulation instead. + auto loc = echo.find_last_of(' ') + 1; + auto last_word = " " + echo.substr(loc, echo.size() - loc); + echo.append(last_word).append(last_word); + + resp->set_message(std::move(echo)); + resp->set_node_id(node_uuid_); + + return Status::OK(); +} + +} // namespace stateful_service +} // namespace yb diff --git a/src/yb/tserver/stateful_services/test_echo_service.h b/src/yb/tserver/stateful_services/test_echo_service.h new file mode 100644 index 000000000000..bc11a9c4f184 --- /dev/null +++ b/src/yb/tserver/stateful_services/test_echo_service.h @@ -0,0 +1,38 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#pragma once + +#include "yb/tserver/stateful_services/stateful_service_base.h" +#include "yb/tserver/stateful_services/test_echo_service.service.h" + +namespace yb { +namespace stateful_service { +class TestEchoService : public StatefulServiceBase, public TestEchoServiceIf { + public: + TestEchoService(const std::string& node_uuid, const scoped_refptr& metric_entity); + void Shutdown() override; + + private: + void Activate(const int64_t leader_term) override; + void Deactivate() override; + virtual Result RunPeriodicTask() override; + + STATEFUL_SERVICE_IMPL_METHODS((GetEcho)); + + private: + const std::string node_uuid_; +}; + +} // namespace stateful_service +} // namespace yb diff --git a/src/yb/tserver/stateful_services/test_echo_service.proto b/src/yb/tserver/stateful_services/test_echo_service.proto new file mode 100644 index 000000000000..fe72579b474a --- /dev/null +++ b/src/yb/tserver/stateful_services/test_echo_service.proto @@ -0,0 +1,34 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +// + +syntax = "proto3"; + +package yb.stateful_service; + +option java_package = "org.yb.stateful_service"; + +import "yb/common/wire_protocol.proto"; + +service TestEchoService { + rpc GetEcho(GetEchoRequestPB) returns (GetEchoResponsePB); +} + +message GetEchoRequestPB { string message = 1; } + +message GetEchoResponsePB { + AppStatusPB error = 1; + string message = 2; + string node_id = 3; +} diff --git a/src/yb/tserver/tablet_server.cc b/src/yb/tserver/tablet_server.cc index e9711b0706f5..b26a47e6bee8 100644 --- a/src/yb/tserver/tablet_server.cc +++ b/src/yb/tserver/tablet_server.cc @@ -86,6 +86,7 @@ #include "yb/cdc/cdc_service.h" #include "yb/cdc/cdc_service_context.h" +#include "yb/tserver/stateful_services/test_echo_service.h" #include "yb/util/flags.h" #include "yb/util/logging.h" @@ -175,6 +176,9 @@ DEFINE_UNKNOWN_int32(tserver_yb_client_default_timeout_ms, kTServerYbClientDefau "Default timeout for the YBClient embedded into the tablet server that is used " "for distributed transactions."); +DEFINE_test_flag(bool, echo_service_enabled, false, "Enable the Test Echo service"); +DEFINE_test_flag(int32, echo_svc_queue_length, 50, "RPC queue length for the Test Echo service"); + DEFINE_test_flag(bool, select_all_status_tablets, false, ""); DEFINE_UNKNOWN_int32(ts_backup_svc_num_threads, 4, @@ -513,6 +517,16 @@ Status TabletServer::RegisterServices() { RETURN_NOT_OK(RpcAndWebServerBase::RegisterService( FLAGS_pg_client_svc_queue_length, std::move(pg_client_service))); + if (FLAGS_TEST_echo_service_enabled) { + auto test_echo_service = + std::make_shared(permanent_uuid(), metric_entity()); + LOG(INFO) << "yb::tserver::stateful_service::TestEchoService created at " + << test_echo_service.get(); + RETURN_NOT_OK(test_echo_service->Init(tablet_manager_.get())); + RETURN_NOT_OK(RpcAndWebServerBase::RegisterService( + FLAGS_TEST_echo_svc_queue_length, std::move(test_echo_service))); + } + return Status::OK(); } diff --git a/src/yb/tserver/ts_tablet_manager.cc b/src/yb/tserver/ts_tablet_manager.cc index fe579b0499c7..c029ffcd135c 100644 --- a/src/yb/tserver/ts_tablet_manager.cc +++ b/src/yb/tserver/ts_tablet_manager.cc @@ -66,6 +66,7 @@ #include "yb/fs/fs_manager.h" #include "yb/gutil/bind.h" +#include "yb/gutil/callback.h" #include "yb/gutil/stl_util.h" #include "yb/gutil/strings/substitute.h" #include "yb/gutil/sysinfo.h" @@ -600,6 +601,18 @@ Status TSTabletManager::Init() { return Status::OK(); } +Status TSTabletManager::RegisterServiceCallback( + StatefulServiceKind service_kind, ConsensusChangeCallback callback) { + std::lock_guard lock(service_registration_mutex_); + SCHECK( + !service_consensus_change_cb_.contains(service_kind), AlreadyPresent, + "Service of kind $0 is already registered", StatefulServiceKind_Name(service_kind)); + + service_consensus_change_cb_[service_kind] = callback; + + return Status::OK(); +} + void TSTabletManager::CleanupCheckpoints() { for (const auto& data_root : fs_manager_->GetDataRootDirs()) { auto tables_dir = JoinPathSegments(data_root, FsManager::kRocksDBDirName); @@ -1676,6 +1689,11 @@ void TSTabletManager::StartShutdown() { } } + { + std::lock_guard lock(service_registration_mutex_); + service_consensus_change_cb_.clear(); + } + tablets_cleaner_->Shutdown(); verify_tablet_data_poller_->Shutdown(); @@ -1954,8 +1972,29 @@ void TSTabletManager::ApplyChange(const string& tablet_id, void TSTabletManager::MarkTabletDirty(const TabletId& tablet_id, std::shared_ptr context) { - std::lock_guard lock(mutex_); - MarkDirtyUnlocked(tablet_id, context); + { + std::lock_guard lock(mutex_); + MarkDirtyUnlocked(tablet_id, context); + } + NotifyConfigChangeToStatefulServices(tablet_id); +} + +void TSTabletManager::NotifyConfigChangeToStatefulServices(const TabletId& tablet_id) { + auto tablet_peer = GetServingTablet(tablet_id); + if (!tablet_peer.ok()) { + VLOG_WITH_FUNC(1) << "Received notification of tablet config change " + << "but tablet peer is not ready. Tablet ID: " << tablet_id + << " Error: " << tablet_peer.status(); + return; + } + + const auto service_list = tablet_peer.get()->tablet_metadata()->GetHostedServiceList(); + for (auto& service_kind : service_list) { + std::shared_lock lock(service_registration_mutex_); + if (service_consensus_change_cb_.contains(service_kind)) { + service_consensus_change_cb_[service_kind].Run(tablet_peer.get()); + } + } } void TSTabletManager::MarkTabletBeingRemoteBootstrapped( diff --git a/src/yb/tserver/ts_tablet_manager.h b/src/yb/tserver/ts_tablet_manager.h index e14a215153a8..1daf7eb27bf0 100644 --- a/src/yb/tserver/ts_tablet_manager.h +++ b/src/yb/tserver/ts_tablet_manager.h @@ -52,6 +52,7 @@ #include "yb/docdb/local_waiting_txn_registry.h" +#include "yb/gutil/callback.h" #include "yb/gutil/macros.h" #include "yb/gutil/ref_counted.h" #include "yb/gutil/stl_util.h" @@ -108,6 +109,8 @@ struct TabletCreationMetaData; typedef boost::container::static_vector SplitTabletsCreationMetaData; +typedef Callback ConsensusChangeCallback; + // If 'expr' fails, log a message, tombstone the given tablet, and return the // error status. #define TOMBSTONE_NOT_OK(expr, meta, uuid, msg, ts_manager_ptr) \ @@ -146,6 +149,9 @@ class TSTabletManager : public tserver::TabletPeerLookupIf, public tablet::Table Status Init(); Status Start(); + Status RegisterServiceCallback( + StatefulServiceKind service_kind, ConsensusChangeCallback callback); + // Waits for all the bootstraps to complete. // Returns Status::OK if all tablets bootstrapped successfully. If // the bootstrap of any tablet failed returns the failure reason for @@ -283,6 +289,8 @@ class TSTabletManager : public tserver::TabletPeerLookupIf, public tablet::Table void ApplyChange(const TabletId& tablet_id, std::shared_ptr context); + void NotifyConfigChangeToStatefulServices(const TabletId& tablet_id) EXCLUDES(mutex_); + // Marks tablet with 'tablet_id' dirty. // Used for state changes outside of the control of TsTabletManager, such as consensus role // changes. @@ -661,6 +669,9 @@ class TSTabletManager : public tserver::TabletPeerLookupIf, public tablet::Table std::unique_ptr full_compaction_manager_; + std::shared_mutex service_registration_mutex_; + std::unordered_map service_consensus_change_cb_; + DISALLOW_COPY_AND_ASSIGN(TSTabletManager); }; From 480604f364403cd78b1da2d222d5a4fd992c942c Mon Sep 17 00:00:00 2001 From: saratkumar-yb Date: Tue, 28 Feb 2023 23:10:55 +0530 Subject: [PATCH 19/81] Break release build_test scripts to two different scripts Summary: Break release build_test scripts to two different scripts (build & testing) Test Plan: Created a new release alma8 test job and phab db build test job on both old branches and master Reviewers: jharveysmith, steve.varnau Reviewed By: steve.varnau Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D22071 --- .../jenkins/{build-and-test.sh => build.sh} | 222 +--------- build-support/jenkins/common-lto.sh | 90 ++++ build-support/jenkins/test.sh | 402 ++++++++++++++++++ build-support/jenkins/yb-jenkins-build.sh | 13 +- build-support/jenkins/yb-jenkins-test.sh | 89 ++++ 5 files changed, 584 insertions(+), 232 deletions(-) rename build-support/jenkins/{build-and-test.sh => build.sh} (72%) create mode 100755 build-support/jenkins/common-lto.sh create mode 100755 build-support/jenkins/test.sh create mode 100755 build-support/jenkins/yb-jenkins-test.sh diff --git a/build-support/jenkins/build-and-test.sh b/build-support/jenkins/build.sh similarity index 72% rename from build-support/jenkins/build-and-test.sh rename to build-support/jenkins/build.sh index 14bad957c1fa..c8a5aeaee388 100755 --- a/build-support/jenkins/build-and-test.sh +++ b/build-support/jenkins/build.sh @@ -131,13 +131,6 @@ build_cpp_code() { set_yb_src_root "$old_yb_src_root" } -cleanup() { - if [[ -n ${BUILD_ROOT:-} && ${DONT_DELETE_BUILD_ROOT} == "0" ]]; then - log "Running the script to clean up build artifacts..." - "$YB_BUILD_SUPPORT_DIR/jenkins/post-build-clean.sh" - fi -} - # ================================================================================================= # Main script # ================================================================================================= @@ -159,64 +152,7 @@ log "Removing old JSON-based test report files" activate_virtualenv set_pythonpath -# We change YB_RUN_JAVA_TEST_METHODS_SEPARATELY in a subshell in a few places and that is OK. -# shellcheck disable=SC2031 -export YB_RUN_JAVA_TEST_METHODS_SEPARATELY=1 - -export TSAN_OPTIONS="" - -if is_mac; then - # This is needed to make sure we're using Homebrew-installed CMake on Mac OS X. - export PATH=/usr/local/bin:$PATH -fi - -# gather core dumps -ulimit -c unlimited - -detect_architecture - -BUILD_TYPE=${BUILD_TYPE:-debug} -build_type=$BUILD_TYPE -normalize_build_type -readonly build_type - -BUILD_TYPE=$build_type -readonly BUILD_TYPE -export BUILD_TYPE - -export YB_USE_NINJA=1 - -set_cmake_build_type_and_compiler_type - -if [[ ${YB_DOWNLOAD_THIRDPARTY:-auto} == "auto" ]]; then - log "Setting YB_DOWNLOAD_THIRDPARTY=1 automatically" - export YB_DOWNLOAD_THIRDPARTY=1 -fi -log "YB_DOWNLOAD_THIRDPARTY=$YB_DOWNLOAD_THIRDPARTY" - -# This is normally done in set_build_root, but we need to decide earlier because this is factored -# into the decision of whether to use LTO. -decide_whether_to_use_linuxbrew - -if [[ -z ${YB_LINKING_TYPE:-} ]]; then - if ! is_mac && [[ - ${YB_COMPILER_TYPE} =~ ^clang[0-9]+$ && - ${BUILD_TYPE} == "release" - ]]; then - export YB_LINKING_TYPE=full-lto - else - export YB_LINKING_TYPE=dynamic - fi - log "Automatically decided to set YB_LINKING_TYPE to ${YB_LINKING_TYPE} based on:" \ - "YB_COMPILER_TYPE=${YB_COMPILER_TYPE}," \ - "BUILD_TYPE=${BUILD_TYPE}," \ - "YB_USE_LINUXBREW=${YB_USE_LINUXBREW}," \ - "YB_LINUXBREW_DIR=${YB_LINUXBREW_DIR:-undefined}." -else - log "YB_LINKING_TYPE is already set to ${YB_LINKING_TYPE}" -fi -log "YB_LINKING_TYPE=${YB_LINKING_TYPE}" -export YB_LINKING_TYPE +. "${BASH_SOURCE%/*}/common-lto.sh" # ------------------------------------------------------------------------------------------------- # Build root setup and build directory cleanup @@ -353,12 +289,6 @@ CTEST_FULL_OUTPUT_PATH="${BUILD_ROOT}"/ctest-full.log TEST_LOG_DIR="${BUILD_ROOT}/test-logs" -# If we're running inside Jenkins (the BUILD_ID is set), then install an exit handler which will -# clean up all of our build results. -if is_jenkins; then - trap cleanup EXIT -fi - configure_remote_compilation export NO_REBUILD_THIRDPARTY=1 @@ -367,7 +297,6 @@ THIRDPARTY_BIN=$YB_SRC_ROOT/thirdparty/installed/bin export PPROF_PATH=$THIRDPARTY_BIN/pprof # Configure the build -# cd "$BUILD_ROOT" @@ -489,31 +418,6 @@ if [[ ${BUILD_TYPE} != "tsan" ]]; then fi fi -# ------------------------------------------------------------------------------------------------- -# Dependency graph analysis allowing to determine what tests to run. -# ------------------------------------------------------------------------------------------------- - -if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then - if ! ( set -x - "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ - --build-root "${BUILD_ROOT}" \ - self-test \ - --rebuild-graph ); then - # Trying to diagnose this error: - # https://gist.githubusercontent.com/mbautin/c5c6f14714f7655c10620d8e658e1f5b/raw - log "dependency_graph.py failed, listing all pb.{h,cc} files in the build directory" - ( set -x; find "$BUILD_ROOT" -name "*.pb.h" -or -name "*.pb.cc" ) - fatal "Dependency graph construction failed" - fi -fi - -# Save the current HEAD commit in case we build Java below and add a new commit. This is used for -# the following purposes: -# - So we can upload the release under the correct commit, from Jenkins, to then be picked up from -# itest, from the snapshots bucket. -# - For picking up the changeset corresponding the the current diff being tested and detecting what -# tests to run in Phabricator builds. If we just diff with origin/master, we'll always pick up -# pom.xml changes we've just made, forcing us to always run Java tests. current_git_commit=$(git rev-parse HEAD) # ------------------------------------------------------------------------------------------------- @@ -703,126 +607,4 @@ else time "${YB_SRC_ROOT}/yb_build.sh" "${BUILD_TYPE}" --build-yugabyted-ui --skip-java fi -# ------------------------------------------------------------------------------------------------- -# Run tests, either on Spark or locally. -# If YB_COMPILE_ONLY is set to 1, we skip running all tests (Java and C++). - -set_sanitizer_runtime_options - -# To reduce Jenkins archive size, let's gzip Java logs and delete per-test-method logs in case -# of no test failures. -export YB_GZIP_PER_TEST_METHOD_LOGS=1 -export YB_GZIP_TEST_LOGS=1 -export YB_DELETE_SUCCESSFUL_PER_TEST_METHOD_LOGS=1 - -if [[ ${YB_COMPILE_ONLY} != "1" ]]; then - if spark_available; then - if [[ ${YB_BUILD_CPP} == "1" || ${YB_BUILD_JAVA} == "1" ]]; then - log "Will run tests on Spark" - run_tests_extra_args=() - if [[ ${YB_BUILD_JAVA} == "1" ]]; then - run_tests_extra_args+=( "--java" ) - fi - if [[ ${YB_BUILD_CPP} == "1" ]]; then - run_tests_extra_args+=( "--cpp" ) - fi - if [[ ${YB_RUN_AFFECTED_TESTS_ONLY} == "1" ]]; then - test_conf_path="${BUILD_ROOT}/test_conf.json" - # YB_GIT_COMMIT_FOR_DETECTING_TESTS allows overriding the commit to use to detect the set - # of tests to run. Useful when testing this script. - ( - set -x - "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ - --build-root "${BUILD_ROOT}" \ - --git-commit "${YB_GIT_COMMIT_FOR_DETECTING_TESTS:-$current_git_commit}" \ - --output-test-config "${test_conf_path}" \ - affected - ) - run_tests_extra_args+=( "--test_conf" "${test_conf_path}" ) - unset test_conf_path - fi - if is_linux || (is_mac && ! is_src_root_on_nfs); then - log "Will create an archive for Spark workers with all the code instead of using NFS." - run_tests_extra_args+=( "--send_archive_to_workers" ) - fi - # Workers use /private path, which caused mis-match when check is done by yb_dist_tests that - # YB_MVN_LOCAL_REPO is in source tree. So unsetting value here to allow default. - if is_mac; then - unset YB_MVN_LOCAL_REPO - fi - - NUM_REPETITIONS="${YB_NUM_REPETITIONS:-1}" - log "NUM_REPETITIONS is set to ${NUM_REPETITIONS}" - if [[ ${NUM_REPETITIONS} -gt 1 ]]; then - log "Repeating each test ${NUM_REPETITIONS} times" - run_tests_extra_args+=( "--num_repetitions" "${NUM_REPETITIONS}" ) - fi - - set +u # because extra_args can be empty - if ! run_tests_on_spark "${run_tests_extra_args[@]}"; then - set -u - EXIT_STATUS=1 - FAILURES+=$'Distributed tests on Spark (C++ and/or Java) failed\n' - log "Some tests that were run on Spark failed" - fi - set -u - unset extra_args - else - log "Neither C++ or Java tests are enabled, nothing to run on Spark." - fi - else - # A single-node way of running tests (without Spark). - - if [[ ${YB_BUILD_CPP} == "1" ]]; then - log "Run C++ tests in a non-distributed way" - export GTEST_OUTPUT="xml:${TEST_LOG_DIR}/" # Enable JUnit-compatible XML output. - - if ! spark_available; then - log "Did not find Spark on the system, falling back to a ctest-based way of running tests" - set +e - # We don't double-quote EXTRA_TEST_FLAGS on purpose, to allow specifying multiple flags. - # shellcheck disable=SC2086 - time ctest "-j${NUM_PARALLEL_TESTS}" ${EXTRA_TEST_FLAGS:-} \ - --output-log "${CTEST_FULL_OUTPUT_PATH}" \ - --output-on-failure 2>&1 | tee "${CTEST_OUTPUT_PATH}" - ctest_exit_code=$? - set -e - if [[ $ctest_exit_code -ne 0 ]]; then - EXIT_STATUS=1 - FAILURES+=$'C++ tests failed with exit code ${ctest_exit_code}\n' - fi - fi - log "Finished running C++ tests (see timing information above)" - fi - - if [[ ${YB_BUILD_JAVA} == "1" ]]; then - set_test_invocation_id - log "Running Java tests in a non-distributed way" - if ! time run_all_java_test_methods_separately; then - EXIT_STATUS=1 - FAILURES+=$'Java tests failed\n' - fi - log "Finished running Java tests (see timing information above)" - # shellcheck disable=SC2119 - kill_stuck_processes - fi - fi -fi - -# Finished running tests. -remove_latest_symlink - -log "Aggregating test reports" -"$YB_SRC_ROOT/python/yb/aggregate_test_reports.py" \ - --yb-src-root "${YB_SRC_ROOT}" \ - --output-dir "${YB_SRC_ROOT}" \ - --build-type "${build_type}" \ - --compiler-type "${YB_COMPILER_TYPE}" \ - --build-root "${BUILD_ROOT}" - -if [[ -n ${FAILURES} ]]; then - heading "Failure summary" - echo >&2 "${FAILURES}" -fi - -exit ${EXIT_STATUS} +exit ${EXIT_STATUS} \ No newline at end of file diff --git a/build-support/jenkins/common-lto.sh b/build-support/jenkins/common-lto.sh new file mode 100755 index 000000000000..46db94a819d4 --- /dev/null +++ b/build-support/jenkins/common-lto.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# The following only applies to changes made to this file as part of YugaByte development. +# +# Portions Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +# This script is invoked from the Jenkins builds to build YB and run all the unit tests. +# +# Environment variables may be used to customize operation: +# BUILD_TYPE: Default: debug +# Maybe be one of asan|tsan|debug|release|coverage|lint +# +# Portions Copyright (c) YugaByte, Inc. + +set -euo pipefail + +# We change YB_RUN_JAVA_TEST_METHODS_SEPARATELY in a subshell in a few places and that is OK. +# shellcheck disable=SC2031 +export YB_RUN_JAVA_TEST_METHODS_SEPARATELY=1 +export TSAN_OPTIONS="" +if is_mac; then + # This is needed to make sure we're using Homebrew-installed CMake on Mac OS X. + export PATH=/usr/local/bin:$PATH +fi +# gather core dumps +ulimit -c unlimited +detect_architecture +BUILD_TYPE=${BUILD_TYPE:-debug} +build_type=$BUILD_TYPE +normalize_build_type +readonly build_type +BUILD_TYPE=$build_type +readonly BUILD_TYPE +export BUILD_TYPE +export YB_USE_NINJA=1 +set_cmake_build_type_and_compiler_type +if [[ ${YB_DOWNLOAD_THIRDPARTY:-auto} == "auto" ]]; then + log "Setting YB_DOWNLOAD_THIRDPARTY=1 automatically" + export YB_DOWNLOAD_THIRDPARTY=1 +fi +log "YB_DOWNLOAD_THIRDPARTY=$YB_DOWNLOAD_THIRDPARTY" +# This is normally done in set_build_root, but we need to decide earlier because this is factored +# into the decision of whether to use LTO. +decide_whether_to_use_linuxbrew +if [[ -z ${YB_LINKING_TYPE:-} ]]; then + if ! is_mac && [[ + ${YB_COMPILER_TYPE} =~ ^clang[0-9]+$ && + ${BUILD_TYPE} == "release" + ]]; then + export YB_LINKING_TYPE=full-lto + else + export YB_LINKING_TYPE=dynamic + fi + log "Automatically decided to set YB_LINKING_TYPE to ${YB_LINKING_TYPE} based on:" \ + "YB_COMPILER_TYPE=${YB_COMPILER_TYPE}," \ + "BUILD_TYPE=${BUILD_TYPE}," \ + "YB_USE_LINUXBREW=${YB_USE_LINUXBREW}," \ + "YB_LINUXBREW_DIR=${YB_LINUXBREW_DIR:-undefined}." +else + log "YB_LINKING_TYPE is already set to ${YB_LINKING_TYPE}" +fi +log "YB_LINKING_TYPE=${YB_LINKING_TYPE}" +export YB_LINKING_TYPE \ No newline at end of file diff --git a/build-support/jenkins/test.sh b/build-support/jenkins/test.sh new file mode 100755 index 000000000000..856bd71d3d25 --- /dev/null +++ b/build-support/jenkins/test.sh @@ -0,0 +1,402 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# The following only applies to changes made to this file as part of YugaByte development. +# +# Portions Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +# This script is invoked from the Jenkins builds to build YB and run all the unit tests. +# +# Environment variables may be used to customize operation: +# BUILD_TYPE: Default: debug +# Maybe be one of asan|tsan|debug|release|coverage|lint +# +# YB_BUILD_CPP +# Default: 1 +# Build and test C++ code if this is set to 1. +# +# YB_SKIP_BUILD +# Default: 0 +# Skip building C++ and Java code, only run tests if this is set to 1 (useful for debugging). +# This option is actually handled by yb_build.sh. +# +# YB_BUILD_JAVA +# Default: 1 +# Build and test java code if this is set to 1. +# +# DONT_DELETE_BUILD_ROOT +# Default: 0 (meaning build root will be deleted) on Jenkins, 1 (don't delete) locally. +# Skip deleting BUILD_ROOT (useful for debugging). +# +# YB_COMPILE_ONLY +# Default: 0 +# Compile the code and build a package, but don't run tests. +# +# YB_RUN_AFFECTED_TESTS_ONLY +# Default: 0 +# Try to auto-detect the set of C++ tests to run for the current set of changes relative to +# origin/master. + +# +# Portions Copyright (c) YugaByte, Inc. + +set -euo pipefail + +echo "Build script ${BASH_SOURCE[0]} is running" + +# shellcheck source=build-support/common-test-env.sh +. "${BASH_SOURCE%/*}/../common-test-env.sh" + +# ------------------------------------------------------------------------------------------------- +# Functions + +cleanup() { + if [[ -n ${BUILD_ROOT:-} && ${DONT_DELETE_BUILD_ROOT} == "0" ]]; then + log "Running the script to clean up build artifacts..." + "$YB_BUILD_SUPPORT_DIR/jenkins/post-build-clean.sh" + fi +} + +# ================================================================================================= +# Main script +# ================================================================================================= + +log "Running with Bash version $BASH_VERSION" + +cd "$YB_SRC_ROOT" +if ! "$YB_BUILD_SUPPORT_DIR/common-build-env-test.sh"; then + fatal "Test of the common build environment failed, cannot proceed." +fi + +activate_virtualenv +set_pythonpath + +. "${BASH_SOURCE%/*}/common-lto.sh" + +# ------------------------------------------------------------------------------------------------- +# Build root setup and build directory cleanup +# ------------------------------------------------------------------------------------------------- + +# shellcheck disable=SC2119 +set_build_root + +set_common_test_paths + +# As soon as we know build root, we need to do the necessary workspace cleanup. +if is_jenkins; then + # Delete the build root by default on Jenkins. + DONT_DELETE_BUILD_ROOT=${DONT_DELETE_BUILD_ROOT:-0} +else + log "Not running on Jenkins, not deleting the build root by default." + # Don't delete the build root by default. + DONT_DELETE_BUILD_ROOT=${DONT_DELETE_BUILD_ROOT:-1} +fi + +export BUILD_ROOT + +# ------------------------------------------------------------------------------------------------- +# End of build root setup and build directory cleanup +# ------------------------------------------------------------------------------------------------- + +# We need to set this prior to the first invocation of yb_build.sh. +export YB_SKIP_FINAL_LTO_LINK=1 + +find_or_download_ysql_snapshots +find_or_download_thirdparty +validate_thirdparty_dir +detect_toolchain +log_thirdparty_and_toolchain_details +find_make_or_ninja_and_update_cmake_opts + +log "YB_USE_NINJA=$YB_USE_NINJA" +log "YB_NINJA_PATH=${YB_NINJA_PATH:-undefined}" + +set_java_home + +export YB_DISABLE_LATEST_SYMLINK=1 +remove_latest_symlink + +if is_jenkins; then + log "Running on Jenkins, will re-create the Python virtualenv" + # YB_RECREATE_VIRTUALENV is used in common-build-env.sh. + # shellcheck disable=SC2034 + YB_RECREATE_VIRTUALENV=1 +fi + +log "Running with PATH: ${PATH}" + +set +e +for python_command in python python2 python2.7 python3; do + log "Location of $python_command: $( which "$python_command" )" +done +set -e + +log "Running Python tests" +time run_python_tests +log "Finished running Python tests (see timing information above)" + +log "Running a light-weight lint script on our Java code" +time lint_java_code +log "Finished running a light-weight lint script on the Java code" + +# TODO: deduplicate this with similar logic in yb-jenkins-build.sh. +YB_BUILD_JAVA=${YB_BUILD_JAVA:-1} +YB_BUILD_CPP=${YB_BUILD_CPP:-1} + +if [[ -z ${YB_RUN_AFFECTED_TESTS_ONLY:-} ]] && is_jenkins_phabricator_build; then + log "YB_RUN_AFFECTED_TESTS_ONLY is not set, and this is a Jenkins Phabricator test." \ + "Setting YB_RUN_AFFECTED_TESTS_ONLY=1 automatically." + export YB_RUN_AFFECTED_TESTS_ONLY=1 +fi +export YB_RUN_AFFECTED_TESTS_ONLY=${YB_RUN_AFFECTED_TESTS_ONLY:-0} +log "YB_RUN_AFFECTED_TESTS_ONLY=${YB_RUN_AFFECTED_TESTS_ONLY}" + +export YB_SKIP_BUILD=${YB_SKIP_BUILD:-0} +if [[ ${YB_SKIP_BUILD} == "1" ]]; then + export NO_REBUILD_THIRDPARTY=1 +fi + +YB_SKIP_CPP_COMPILATION=${YB_SKIP_CPP_COMPILATION:-0} +YB_COMPILE_ONLY=${YB_COMPILE_ONLY:-0} + +CTEST_OUTPUT_PATH="${BUILD_ROOT}"/ctest.log +CTEST_FULL_OUTPUT_PATH="${BUILD_ROOT}"/ctest-full.log + +TEST_LOG_DIR="${BUILD_ROOT}/test-logs" + +# If we're running inside Jenkins (the BUILD_ID is set), then install an exit handler which will +# clean up all of our build results. +if is_jenkins; then + trap cleanup EXIT +fi + +configure_remote_compilation + +export NO_REBUILD_THIRDPARTY=1 + +THIRDPARTY_BIN=$YB_SRC_ROOT/thirdparty/installed/bin +export PPROF_PATH=$THIRDPARTY_BIN/pprof + +# Configure the build +# + +cd "$BUILD_ROOT" + +if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then + ( + set -x + # Remove the compilation command file, even if we have not deleted the build root. + rm -f "$BUILD_ROOT/compile_commands.json" + ) +fi + +# Only enable test core dumps for certain build types. +if [[ ${BUILD_TYPE} != "asan" ]]; then + # TODO: actually make this take effect. The issue is that we might not be able to set ulimit + # unless the OS configuration enables us to. + export YB_TEST_ULIMIT_CORE=unlimited +fi + +detect_num_cpus + +declare -i EXIT_STATUS=0 + +set +e +if [[ -d /tmp/yb-port-locks ]]; then + # Allow other users to also run minicluster tests on this machine. + chmod a+rwx /tmp/yb-port-locks +fi +set -e + +FAILURES="" + +if [[ ${YB_BUILD_CPP} == "1" ]] && ! which ctest >/dev/null; then + fatal "ctest not found, won't be able to run C++ tests" +fi + +export YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT=1 + +# ------------------------------------------------------------------------------------------------- +# Running initdb +# ------------------------------------------------------------------------------------------------- + +export YB_SKIP_INITIAL_SYS_CATALOG_SNAPSHOT=0 + +# ------------------------------------------------------------------------------------------------- +# Dependency graph analysis allowing to determine what tests to run. +# ------------------------------------------------------------------------------------------------- + +if [[ $YB_RUN_AFFECTED_TESTS_ONLY == "1" ]]; then + if ! ( set -x + "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ + --build-root "${BUILD_ROOT}" \ + self-test \ + --rebuild-graph ); then + # Trying to diagnose this error: + # https://gist.githubusercontent.com/mbautin/c5c6f14714f7655c10620d8e658e1f5b/raw + log "dependency_graph.py failed, listing all pb.{h,cc} files in the build directory" + ( set -x; find "$BUILD_ROOT" -name "*.pb.h" -or -name "*.pb.cc" ) + fatal "Dependency graph construction failed" + fi +fi + +# Save the current HEAD commit in case we build Java below and add a new commit. This is used for +# the following purposes: +# - So we can upload the release under the correct commit, from Jenkins, to then be picked up from +# itest, from the snapshots bucket. +# - For picking up the changeset corresponding the the current diff being tested and detecting what +# tests to run in Phabricator builds. If we just diff with origin/master, we'll always pick up +# pom.xml changes we've just made, forcing us to always run Java tests. +current_git_commit=$(git rev-parse HEAD) + +export YB_MVN_LOCAL_REPO=$BUILD_ROOT/m2_repository + +# ------------------------------------------------------------------------------------------------- +# Run tests, either on Spark or locally. +# If YB_COMPILE_ONLY is set to 1, we skip running all tests (Java and C++). + +set_sanitizer_runtime_options + +# To reduce Jenkins archive size, let's gzip Java logs and delete per-test-method logs in case +# of no test failures. +export YB_GZIP_PER_TEST_METHOD_LOGS=1 +export YB_GZIP_TEST_LOGS=1 +export YB_DELETE_SUCCESSFUL_PER_TEST_METHOD_LOGS=1 + +if [[ ${YB_COMPILE_ONLY} != "1" ]]; then + if spark_available; then + if [[ ${YB_BUILD_CPP} == "1" || ${YB_BUILD_JAVA} == "1" ]]; then + log "Will run tests on Spark" + run_tests_extra_args=() + if [[ ${YB_BUILD_JAVA} == "1" ]]; then + run_tests_extra_args+=( "--java" ) + fi + if [[ ${YB_BUILD_CPP} == "1" ]]; then + run_tests_extra_args+=( "--cpp" ) + fi + if [[ ${YB_RUN_AFFECTED_TESTS_ONLY} == "1" ]]; then + test_conf_path="${BUILD_ROOT}/test_conf.json" + # YB_GIT_COMMIT_FOR_DETECTING_TESTS allows overriding the commit to use to detect the set + # of tests to run. Useful when testing this script. + ( + set -x + "${YB_SRC_ROOT}/python/yb/dependency_graph.py" \ + --build-root "${BUILD_ROOT}" \ + --git-commit "${YB_GIT_COMMIT_FOR_DETECTING_TESTS:-$current_git_commit}" \ + --output-test-config "${test_conf_path}" \ + affected + ) + run_tests_extra_args+=( "--test_conf" "${test_conf_path}" ) + unset test_conf_path + fi + if is_linux || (is_mac && ! is_src_root_on_nfs); then + log "Will create an archive for Spark workers with all the code instead of using NFS." + run_tests_extra_args+=( "--send_archive_to_workers" ) + fi + # Workers use /private path, which caused mis-match when check is done by yb_dist_tests that + # YB_MVN_LOCAL_REPO is in source tree. So unsetting value here to allow default. + if is_mac; then + unset YB_MVN_LOCAL_REPO + fi + + NUM_REPETITIONS="${YB_NUM_REPETITIONS:-1}" + log "NUM_REPETITIONS is set to ${NUM_REPETITIONS}" + if [[ ${NUM_REPETITIONS} -gt 1 ]]; then + log "Repeating each test ${NUM_REPETITIONS} times" + run_tests_extra_args+=( "--num_repetitions" "${NUM_REPETITIONS}" ) + fi + + set +u # because extra_args can be empty + if ! run_tests_on_spark "${run_tests_extra_args[@]}"; then + set -u + EXIT_STATUS=1 + FAILURES+=$'Distributed tests on Spark (C++ and/or Java) failed\n' + log "Some tests that were run on Spark failed" + fi + set -u + unset extra_args + else + log "Neither C++ or Java tests are enabled, nothing to run on Spark." + fi + else + # A single-node way of running tests (without Spark). + + if [[ ${YB_BUILD_CPP} == "1" ]]; then + log "Run C++ tests in a non-distributed way" + export GTEST_OUTPUT="xml:${TEST_LOG_DIR}/" # Enable JUnit-compatible XML output. + + if ! spark_available; then + log "Did not find Spark on the system, falling back to a ctest-based way of running tests" + set +e + # We don't double-quote EXTRA_TEST_FLAGS on purpose, to allow specifying multiple flags. + # shellcheck disable=SC2086 + time ctest "-j${NUM_PARALLEL_TESTS}" ${EXTRA_TEST_FLAGS:-} \ + --output-log "${CTEST_FULL_OUTPUT_PATH}" \ + --output-on-failure 2>&1 | tee "${CTEST_OUTPUT_PATH}" + ctest_exit_code=$? + set -e + if [[ $ctest_exit_code -ne 0 ]]; then + EXIT_STATUS=1 + FAILURES+=$'C++ tests failed with exit code ${ctest_exit_code}\n' + fi + fi + log "Finished running C++ tests (see timing information above)" + fi + + if [[ ${YB_BUILD_JAVA} == "1" ]]; then + set_test_invocation_id + log "Running Java tests in a non-distributed way" + if ! time run_all_java_test_methods_separately; then + EXIT_STATUS=1 + FAILURES+=$'Java tests failed\n' + fi + log "Finished running Java tests (see timing information above)" + # shellcheck disable=SC2119 + kill_stuck_processes + fi + fi +fi + +# Finished running tests. +remove_latest_symlink + +log "Aggregating test reports" +"$YB_SRC_ROOT/python/yb/aggregate_test_reports.py" \ + --yb-src-root "${YB_SRC_ROOT}" \ + --output-dir "${YB_SRC_ROOT}" \ + --build-type "${build_type}" \ + --compiler-type "${YB_COMPILER_TYPE}" \ + --build-root "${BUILD_ROOT}" + +if [[ -n ${FAILURES} ]]; then + heading "Failure summary" + echo >&2 "${FAILURES}" +fi + +exit ${EXIT_STATUS} \ No newline at end of file diff --git a/build-support/jenkins/yb-jenkins-build.sh b/build-support/jenkins/yb-jenkins-build.sh index 25b211c0b334..03656e2d13d0 100755 --- a/build-support/jenkins/yb-jenkins-build.sh +++ b/build-support/jenkins/yb-jenkins-build.sh @@ -34,8 +34,6 @@ Environment variables: BUILD_TYPE Passed directly to build-and-test.sh. The default value is determined based on the job name if this environment variable is not specified or if the value is "auto". - YB_NUM_TESTS_TO_RUN - Maximum number of tests ctest should run before exiting. Used for testing Jenkins scripts. EOT } @@ -77,15 +75,6 @@ for branch_name in $( git for-each-ref --format="%(refname)" refs/heads/ ); do fi done -if [ -n "${YB_NUM_TESTS_TO_RUN:-}" ]; then - - if [[ ! "$YB_NUM_TESTS_TO_RUN" =~ ^[0-9]+$ ]]; then - echo "Invalid number of tests to run: $YB_NUM_TESTS_TO_RUN" >&2 - exit 1 - fi - export EXTRA_TEST_FLAGS="-I1,$YB_NUM_TESTS_TO_RUN" -fi - export YB_MINIMIZE_VERSION_DEFINES_CHANGES=1 export YB_MINIMIZE_RECOMPILATION=1 @@ -118,7 +107,7 @@ if is_mac; then fi set +e -"$YB_BUILD_SUPPORT_DIR"/jenkins/build-and-test.sh +"$YB_BUILD_SUPPORT_DIR"/jenkins/build.sh exit_code=$? set -e diff --git a/build-support/jenkins/yb-jenkins-test.sh b/build-support/jenkins/yb-jenkins-test.sh new file mode 100755 index 000000000000..50eeebb61e95 --- /dev/null +++ b/build-support/jenkins/yb-jenkins-test.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# +# Copyright (c) YugaByte, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. +# +set -euo pipefail + +# shellcheck source=build-support/common-test-env.sh +. "${0%/*}/../common-test-env.sh" + +print_help() { + cat <<-EOT +Usage: ${0##*} +Options: + -h, --help + Show help + --delete-arc-patch-branches + Delete branches starting with "arcpatch-D..." (except the current branch) so that the Jenkins + Phabricator plugin does not give up after three attempts. + +Environment variables: + JOB_NAME + Jenkins job name. + BUILD_TYPE + Passed directly to build-and-test.sh. The default value is determined based on the job name + if this environment variable is not specified or if the value is "auto". + YB_NUM_TESTS_TO_RUN + Maximum number of tests ctest should run before exiting. Used for testing Jenkins scripts. +EOT +} + +echo "Build script ${BASH_SOURCE[0]} is running" + +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + print_help + exit 0 + ;; + *) + echo "Invalid option: $1" >&2 + exit 1 + esac + shift +done + +JOB_NAME=${JOB_NAME:-} +build_type=${BUILD_TYPE:-} +set_build_type_based_on_jenkins_job_name +readonly BUILD_TYPE=$build_type +export BUILD_TYPE + +echo "Build type: ${BUILD_TYPE}"; + +set_compiler_type_based_on_jenkins_job_name + +if [ -n "${YB_NUM_TESTS_TO_RUN:-}" ]; then + + if [[ ! "$YB_NUM_TESTS_TO_RUN" =~ ^[0-9]+$ ]]; then + echo "Invalid number of tests to run: $YB_NUM_TESTS_TO_RUN" >&2 + exit 1 + fi + export EXTRA_TEST_FLAGS="-I1,$YB_NUM_TESTS_TO_RUN" +fi + +export YB_MINIMIZE_VERSION_DEFINES_CHANGES=1 +export YB_MINIMIZE_RECOMPILATION=1 + +export YB_BUILD_JAVA=${YB_BUILD_JAVA:-1} +export YB_BUILD_PYTHON=${YB_BUILD_PYTHON:-0} +export YB_BUILD_CPP=${YB_BUILD_CPP:-1} + +set +e +"$YB_BUILD_SUPPORT_DIR"/jenkins/test.sh +exit_code=$? + +set -e + +exit $exit_code From 53d576b56681a0750417cbb7a8bbce12297d7621 Mon Sep 17 00:00:00 2001 From: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> Date: Wed, 1 Mar 2023 13:09:39 -0500 Subject: [PATCH 20/81] [docs][explore] Destroy local cluster instructions (#16264) * Destroy local cluster instructions * xref to yugabyted page * minor edit to faq --- docs/content/preview/explore/_index.md | 23 ++----------- docs/content/preview/faq/operations-faq.md | 2 +- .../reference/configuration/yugabyted.md | 34 +++++++++++++------ 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/docs/content/preview/explore/_index.md b/docs/content/preview/explore/_index.md index c3b7038cf8a4..b2a8c1d5bc4f 100644 --- a/docs/content/preview/explore/_index.md +++ b/docs/content/preview/explore/_index.md @@ -73,7 +73,6 @@ You can run examples using a universe set up on your local machine or in a cloud
- To run the examples, you need to create a single- or multi-node universe. For testing and learning YugabyteDB on your computer, use the [yugabyted](../reference/configuration/yugabyted/) utility to create and manage universes. @@ -82,6 +81,8 @@ For testing and learning YugabyteDB on your computer, use the [yugabyted](../ref {{% tab header="Single-node universe" lang="Single-node universe" %}} +If a local universe is currently running, first [destroy it](../reference/configuration/yugabyted/#destroy-a-local-cluster). + You can create a single-node local universe with a replication factor (RF) of 1 by running the following command: ```sh @@ -97,23 +98,13 @@ Or, if you are running macOS Monterey, use the following command: For more information, refer to [Quick Start](../quick-start/linux/#create-a-local-cluster). -To stop a single-node universe, execute the following: - -```sh -./bin/yugabyted destroy -``` - {{% /tab %}} {{% tab header="Multi-node universe" lang="Multi-node universe" %}} The following instructions show how to simulate a multi-node universe on a single computer. To deploy a true multi-node universe, follow the instructions in [Deploy](../deploy/). -If a single-node universe is currently running, first destroy it, as follows: - -```sh -./bin/yugabyted destroy -``` +If a local universe is currently running, first [destroy it](../reference/configuration/yugabyted/#destroy-a-local-cluster). Start a local three-node universe with an RF of `3` by first creating a single node universe, as follows: @@ -157,14 +148,6 @@ After starting the yugabyted processes on all the nodes, configure the data plac This command can be executed on any node where you already started YugabyteDB. -To destroy the multi-node universe, execute the following: - -```sh -./bin/yugabyted destroy --base_dir=/tmp/ybd1 -./bin/yugabyted destroy --base_dir=/tmp/ybd2 -./bin/yugabyted destroy --base_dir=/tmp/ybd3 -``` - {{% /tab %}} {{< /tabpane >}} diff --git a/docs/content/preview/faq/operations-faq.md b/docs/content/preview/faq/operations-faq.md index 179361dda66d..8b08cf7554c7 100644 --- a/docs/content/preview/faq/operations-faq.md +++ b/docs/content/preview/faq/operations-faq.md @@ -14,7 +14,7 @@ showRightNav: false ### Do YugabyteDB clusters need an external load balancer? -For YSQL, an external load balancer is recommended. You can also use a YugabyteDB smart driver to balance connection load. To learn more about smart drivers, refer to [YugabyteDB smart drivers for YSQL](../../drivers-orms/smart-drivers/). +For YSQL, you should use a YugabyteDB smart driver. YugabyteDB smart drivers automatically balance connections to the database and eliminate the need for an external load balancer. If you are not using a smart driver, you will need an external load balancer. To learn more about smart drivers, refer to [YugabyteDB smart drivers for YSQL](../../drivers-orms/smart-drivers/). For YCQL, YugabyteDB provides automatic load balancing. diff --git a/docs/content/preview/reference/configuration/yugabyted.md b/docs/content/preview/reference/configuration/yugabyted.md index ffc1c2f582e7..09de5d4061e2 100644 --- a/docs/content/preview/reference/configuration/yugabyted.md +++ b/docs/content/preview/reference/configuration/yugabyted.md @@ -667,6 +667,30 @@ The following are combinations of environment variables and their uses: To deploy any type of secure cluster (that is, using the `--secure` flag) or use encryption at rest, OpenSSL must be installed on your machine. +### Destroy a local cluster + +If you are running YugabyteDB on your local computer, you can't run more than one cluster at a time. To set up a new local YugabyteDB cluster using yugabyted, first destroy the currently running cluster. + +To destroy a local single-node cluster, use the [destroy](#destroy) command as follows: + +```sh +./bin/yugabyted destroy +``` + +To destroy a local multi-node cluster, use the `destroy` command with the `--base_dir` flag set to the base directory path of each of the nodes. For example, for a three node cluster, you would execute commands similar to the following: + +```sh +./bin/yugabyted destroy --base_dir=/tmp/ybd1 +./bin/yugabyted destroy --base_dir=/tmp/ybd2 +./bin/yugabyted destroy --base_dir=/tmp/ybd3 +``` + +```sh +./bin/yugabyted destroy --base_dir=$HOME/yugabyte-{{< yb-version version="preview" >}}/node1 +./bin/yugabyted destroy --base_dir=$HOME/yugabyte-{{< yb-version version="preview" >}}/node2 +./bin/yugabyted destroy --base_dir=$HOME/yugabyte-{{< yb-version version="preview" >}}/node3 +``` + ### Create a single-node cluster Create a single-node cluster with a given base directory. Note the need to provide a fully-qualified directory path for the `base_dir` parameter. @@ -739,16 +763,6 @@ Add two more nodes to the cluster using the `join` option, as follows: --cloud_location=aws.us-east-1.us-east-1c ``` -### Destroy a local multi-node cluster - -To destroy the multi-node cluster, execute the following: - -```sh -./bin/yugabyted destroy --base_dir=$HOME/yugabyte-{{< yb-version version="preview" >}}/node1 -./bin/yugabyted destroy --base_dir=$HOME/yugabyte-{{< yb-version version="preview" >}}/node2 -./bin/yugabyted destroy --base_dir=$HOME/yugabyte-{{< yb-version version="preview" >}}/node3 -``` - ### Create a multi-zone cluster {{< tabpane text=true >}} From 31e30a0970c74f99063fbfd2660dfd11de189e93 Mon Sep 17 00:00:00 2001 From: Naorem Khogendro Singh Date: Wed, 22 Feb 2023 17:09:25 -0800 Subject: [PATCH 21/81] [PLAT-7396] Update on-prem manual provisioning pre-flight checks to ensure that the vm.max_map_count is 256k before adding the node to on-prem provider Summary: Add preflight check for virtual memory max map count. Fixed an issue where PrecheckNodeDetached was using SSH only when node-agent was configured. Preflight check (on-prem manually configured nodes) for https://phabricator.dev.yugabyte.com/D23016 Test Plan: Tested manually the below cases: 1. Create a Universe with an unconfigured on-prem node. The particular preflight check was not run as sudo access is given to modify it. 2. Create a Universe with a manually configured on-prem node. Preflight check failed when the value was set too low, otherwise it worked and the creation was successful. 3. Manually ran detached preflight-check (action drop down). Both passed. The node agent issue was found and fixed. Reviewers: nbhatia, sanketh, cwang, dshubin Reviewed By: cwang, dshubin Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23231 --- .../opscli/ybops/data/preflight_checks.sh | 6 ++++++ .../node-agent/resources/preflight_check.sh | 4 ++++ .../yugabyte/yw/common/NodeAgentClient.java | 2 +- .../yugabyte/yw/common/NodeAgentManager.java | 18 ++++++++++-------- .../com/yugabyte/yw/common/NodeManager.java | 19 +++++++++++++++---- .../yw/common/NodeUniverseManager.java | 2 +- .../yw/common/config/ProviderConfKeys.java | 9 +++++++++ .../yw/models/helpers/NodeConfig.java | 4 +++- .../models/helpers/NodeConfigValidator.java | 11 ++++++++++- managed/src/main/resources/reference.conf | 1 + .../src/main/resources/swagger-strict.json | 4 ++-- managed/src/main/resources/swagger.json | 4 ++-- .../yugabyte/yw/common/NodeManagerTest.java | 2 +- 13 files changed, 65 insertions(+), 21 deletions(-) diff --git a/managed/devops/opscli/ybops/data/preflight_checks.sh b/managed/devops/opscli/ybops/data/preflight_checks.sh index f22b327b67cf..f72615513eaf 100755 --- a/managed/devops/opscli/ybops/data/preflight_checks.sh +++ b/managed/devops/opscli/ybops/data/preflight_checks.sh @@ -21,6 +21,7 @@ YB_SUDO_PASS="" ports_to_check="" PROMETHEUS_FREE_SPACE_MB=100 HOME_FREE_SPACE_MB=2048 +VM_MAX_MAP_COUNT=262144 PYTHON_EXECUTABLES=('python3.6' 'python3' 'python3.7' 'python3.8' 'python') preflight_provision_check() { @@ -160,6 +161,11 @@ preflight_configure_check() { # Check home directory exists. check_filepath "Home Directory" "$yb_home_dir" false + + # Check virtual memory max map limit. + vm_max_map_count=$(cat /proc/sys/vm/max_map_count 2> /dev/null) + test ${vm_max_map_count:-0} -ge $VM_MAX_MAP_COUNT + update_result_json_with_rc "vm_max_map_count" "$?" } # Checks for an available python executable diff --git a/managed/node-agent/resources/preflight_check.sh b/managed/node-agent/resources/preflight_check.sh index 580f30d93978..97e72c86758c 100755 --- a/managed/node-agent/resources/preflight_check.sh +++ b/managed/node-agent/resources/preflight_check.sh @@ -234,6 +234,10 @@ preflight_configure_check() { else update_result_json "systemd_sudoer_entry" false fi + + # Check virtual memory max map limit. + vm_max_map_count=$(cat /proc/sys/vm/max_map_count 2> /dev/null) + update_result_json "vm_max_map_count" "${vm_max_map_count:-0}" } preflight_all_checks() { diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeAgentClient.java b/managed/src/main/java/com/yugabyte/yw/common/NodeAgentClient.java index c2cfddea9731..d4713b9ead79 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeAgentClient.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeAgentClient.java @@ -375,7 +375,7 @@ public static void addNodeAgentClientParams( } } - public Optional maybeGetNodeAgentClient(String ip, Provider provider) { + public Optional maybeGetNodeAgent(String ip, Provider provider) { if (isClientEnabled(provider)) { Optional optional = NodeAgent.maybeGetByIp(ip); if (optional.isPresent() && optional.get().state != State.REGISTERING) { diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeAgentManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeAgentManager.java index 55b749542c38..efa0e9c337dc 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeAgentManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeAgentManager.java @@ -370,14 +370,16 @@ public Path getNodeAgentPackagePath(NodeAgent.OSType osType, NodeAgent.ArchType // Search for a pattern like node_agent-2.15.3.0*-linux-amd64.tar.gz. FileFilter fileFilter = new WildcardFileFilter(pkgFileFilter); File[] files = releasesPath.toFile().listFiles(fileFilter); - for (File file : files) { - matcher = filePattern.matcher(file.getName()); - if (matcher.find()) { - // Extract the version with build number e.g. 2.15.3.0-b1372. - String version = matcher.group(1); - // Compare the full versions. The comparison ignores non-numeric build numbers. - if (Util.compareYbVersions(softwareVersion, version, true) == 0) { - return file.toPath(); + if (files != null) { + for (File file : files) { + matcher = filePattern.matcher(file.getName()); + if (matcher.find()) { + // Extract the version with build number e.g. 2.15.3.0-b1372. + String version = matcher.group(1); + // Compare the full versions. The comparison ignores non-numeric build numbers. + if (Util.compareYbVersions(softwareVersion, version, true) == 0) { + return file.toPath(); + } } } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java index afc05c188f26..7f163d34664c 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeManager.java @@ -52,6 +52,7 @@ import com.yugabyte.yw.common.utils.Pair; import com.yugabyte.yw.forms.CertificateParams; import com.yugabyte.yw.forms.CertsRotateParams.CertRotationType; +import com.yugabyte.yw.forms.NodeInstanceFormData.NodeInstanceData; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.forms.UniverseDefinitionTaskParams.UserIntent; import com.yugabyte.yw.forms.UniverseTaskParams; @@ -1270,11 +1271,21 @@ public ShellResponse detachedNodeCommand( InstanceType instanceType = InstanceType.get(provider.uuid, nodeTaskParam.getInstanceType()); commandArgs.add("--mount_points"); commandArgs.add(instanceType.instanceTypeDetails.volumeDetailsList.get(0).mountPath); - + NodeInstance nodeInstance = NodeInstance.getOrBadRequest(nodeTaskParam.getNodeUuid()); + NodeInstanceData instanceData = nodeInstance.getDetails(); + if (StringUtils.isNotBlank(instanceData.ip)) { + getNodeAgentClient() + .maybeGetNodeAgent(instanceData.ip, provider) + .ifPresent( + nodeAgent -> { + commandArgs.add("--connection_type"); + commandArgs.add("node_agent_rpc"); + NodeAgentClient.addNodeAgentClientParams(nodeAgent, commandArgs); + }); + } commandArgs.add(nodeTaskParam.getNodeName()); - NodeInstance nodeInstance = NodeInstance.getOrBadRequest(nodeTaskParam.getNodeUuid()); - JsonNode nodeDetails = Json.toJson(nodeInstance.getDetails()); + JsonNode nodeDetails = Json.toJson(instanceData); ((ObjectNode) nodeDetails).put("nodeName", DetachedNodeTaskParams.DEFAULT_NODE_NAME); List cloudArgs = Arrays.asList("--node_metadata", Json.stringify(nodeDetails)); @@ -1379,7 +1390,7 @@ private void addNodeAgentCommandArgs( if (StringUtils.isNotBlank(nodeIp) && StringUtils.isNotBlank(userIntent.provider)) { Provider provider = Provider.getOrBadRequest(UUID.fromString(userIntent.provider)); getNodeAgentClient() - .maybeGetNodeAgentClient(nodeIp, provider) + .maybeGetNodeAgent(nodeIp, provider) .ifPresent( nodeAgent -> { commandArgs.add("--connection_type"); diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java index 605e7803b6b2..aac6968bcf07 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java @@ -286,7 +286,7 @@ private void addConnectionParams( AccessKey accessKey = AccessKey.getOrBadRequest(providerUUID, cluster.userIntent.accessKeyCode); Optional optional = - getNodeAgentClient().maybeGetNodeAgentClient(node.cloudInfo.private_ip, provider); + getNodeAgentClient().maybeGetNodeAgent(node.cloudInfo.private_ip, provider); if (optional.isPresent()) { commandArgs.add("rpc"); NodeAgentClient.addNodeAgentClientParams(optional.get(), commandArgs); diff --git a/managed/src/main/java/com/yugabyte/yw/common/config/ProviderConfKeys.java b/managed/src/main/java/com/yugabyte/yw/common/config/ProviderConfKeys.java index 3f82091be603..6843e884261d 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/config/ProviderConfKeys.java +++ b/managed/src/main/java/com/yugabyte/yw/common/config/ProviderConfKeys.java @@ -251,4 +251,13 @@ public class ProviderConfKeys extends RuntimeConfigKeysModule { "Install node agent server on DB nodes.", ConfDataType.BooleanType, ImmutableList.of(ConfKeyTags.PUBLIC)); + + public static final ConfKeyInfo vmMaxMemCount = + new ConfKeyInfo<>( + "yb.node_agent.preflight_checks.vm_max_map_count", + ScopeType.PROVIDER, + "VM max map count", + "Max count of memory-mapped regions allowed in the system.", + ConfDataType.IntegerType, + ImmutableList.of(ConfKeyTags.BETA)); } diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfig.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfig.java index 450dbfd157d3..9b2060a71441 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfig.java +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfig.java @@ -142,7 +142,9 @@ public enum Type { YSQL_SERVER_HTTP_PORT("YSQL server http port is open"), - YSQL_SERVER_RPC_PORT("YSQL server rpc port is open"); + YSQL_SERVER_RPC_PORT("YSQL server rpc port is open"), + + VM_MAX_MAP_COUNT("VM max memory map count"); private final String description; diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfigValidator.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfigValidator.java index 8766d16c6d99..3603e472f6b2 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfigValidator.java +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/NodeConfigValidator.java @@ -241,6 +241,11 @@ private boolean isNodeConfigValid(ValidationData input) { int value = getFromConfig(CONFIG_INT_SUPPLIER, provider, "swappiness"); return Integer.parseInt(nodeConfig.getValue()) == value; } + case VM_MAX_MAP_COUNT: + { + int value = getFromConfig(CONFIG_INT_SUPPLIER, provider, "vm_max_map_count"); + return Integer.parseInt(nodeConfig.getValue()) >= value; + } case MOUNT_POINTS_WRITABLE: case MASTER_HTTP_PORT: case MASTER_RPC_PORT: @@ -308,6 +313,10 @@ private boolean isNodeConfigRequired(ValidationData input) { return input.getOperation() == Operation.CONFIGURE && nodeAgentClient.isClientEnabled(provider); } + case VM_MAX_MAP_COUNT: + { + return input.getOperation() == Operation.CONFIGURE; + } case CHRONYD_RUNNING: case RSYNC: case XXHASH: @@ -340,7 +349,7 @@ private T getFromConfig(Function function, Provider provider, ConfigKey configKey = ConfigKey.builder().provider(provider).path(String.format(CONFIG_KEY_FORMAT, key)).build(); T value = function.apply(configKey); - log.trace("Value for {}: {}", configKey, value); + log.debug("Value for {}: {}", configKey, value); return value; } diff --git a/managed/src/main/resources/reference.conf b/managed/src/main/resources/reference.conf index 6e1a56180475..2daf69f733dc 100644 --- a/managed/src/main/resources/reference.conf +++ b/managed/src/main/resources/reference.conf @@ -720,6 +720,7 @@ yb { ulimit_user_processes = 12000 swappiness = 0 ssh_timeout = 10 + vm_max_map_count = 262144 } releases { # Path to the node-agent releases. diff --git a/managed/src/main/resources/swagger-strict.json b/managed/src/main/resources/swagger-strict.json index 4be345fe456c..d0a15a1f50df 100644 --- a/managed/src/main/resources/swagger-strict.json +++ b/managed/src/main/resources/swagger-strict.json @@ -5693,7 +5693,7 @@ "description" : "A node configuration.", "properties" : { "type" : { - "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT" ], + "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT", "VM_MAX_MAP_COUNT" ], "type" : "string" }, "value" : { @@ -11316,7 +11316,7 @@ "type" : "boolean" }, "type" : { - "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT" ], + "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT", "VM_MAX_MAP_COUNT" ], "type" : "string" }, "value" : { diff --git a/managed/src/main/resources/swagger.json b/managed/src/main/resources/swagger.json index 507d1eb04422..d3755d685228 100644 --- a/managed/src/main/resources/swagger.json +++ b/managed/src/main/resources/swagger.json @@ -5693,7 +5693,7 @@ "description" : "A node configuration.", "properties" : { "type" : { - "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT" ], + "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT", "VM_MAX_MAP_COUNT" ], "type" : "string" }, "value" : { @@ -11393,7 +11393,7 @@ "type" : "boolean" }, "type" : { - "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT" ], + "enum" : [ "NTP_SERVICE_STATUS", "PROMETHEUS_SPACE", "MOUNT_POINTS_WRITABLE", "USER", "USER_GROUP", "HOME_DIR_SPACE", "HOME_DIR_EXISTS", "RAM_SIZE", "INTERNET_CONNECTION", "CPU_CORES", "PROMETHEUS_NO_NODE_EXPORTER", "TMP_DIR_SPACE", "PAM_LIMITS_WRITABLE", "PYTHON_VERSION", "MOUNT_POINTS_VOLUME", "CHRONYD_RUNNING", "SSH_PORT", "SUDO_ACCESS", "OPENSSL", "POLICYCOREUTILS", "RSYNC", "XXHASH", "LIBATOMIC1", "LIBNCURSES6", "LIBATOMIC", "AZCOPY", "GSUTIL", "S3CMD", "NODE_EXPORTER_RUNNING", "NODE_EXPORTER_PORT", "SWAPPINESS", "ULIMIT_CORE", "ULIMIT_OPEN_FILES", "ULIMIT_USER_PROCESSES", "SYSTEMD_SUDOER_ENTRY", "SSH_ACCESS", "NODE_AGENT_ACCESS", "MASTER_HTTP_PORT", "MASTER_RPC_PORT", "TSERVER_HTTP_PORT", "TSERVER_RPC_PORT", "YB_CONTROLLER_HTTP_PORT", "YB_CONTROLLER_RPC_PORT", "REDIS_SERVER_HTTP_PORT", "REDIS_SERVER_RPC_PORT", "YQL_SERVER_HTTP_PORT", "YQL_SERVER_RPC_PORT", "YSQL_SERVER_HTTP_PORT", "YSQL_SERVER_RPC_PORT", "VM_MAX_MAP_COUNT" ], "type" : "string" }, "value" : { diff --git a/managed/src/test/java/com/yugabyte/yw/common/NodeManagerTest.java b/managed/src/test/java/com/yugabyte/yw/common/NodeManagerTest.java index bac4bcbbc20d..76473a77ef45 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/NodeManagerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/NodeManagerTest.java @@ -517,7 +517,7 @@ public void setUp() { when(runtimeConfigFactory.forProvider(any())).thenReturn(mockConfig); when(runtimeConfigFactory.forUniverse(any())).thenReturn(app.config()); when(runtimeConfigFactory.globalRuntimeConf()).thenReturn(mockConfig); - when(nodeAgentClient.maybeGetNodeAgentClient(any(), any())).thenReturn(Optional.empty()); + when(nodeAgentClient.maybeGetNodeAgent(any(), any())).thenReturn(Optional.empty()); createTempFile("node_manager_test_ca.crt", "test-cert"); when(mockConfGetter.getConfForScope( any(Universe.class), eq(UniverseConfKeys.ybcEnableVervbose))) From c473157918cb272dc3e7f794a84d12c357e4cf16 Mon Sep 17 00:00:00 2001 From: Aishwarya Chakravarthy Date: Wed, 1 Mar 2023 13:53:40 -0500 Subject: [PATCH 22/81] [DOC-18] Doc for pg_stat_progress_create_index (#16007) * first push includes a new page and some content , intro left and examples * missed changes * more changes to the introduction * some changes from review * added example * Update docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * addressed PR comments * changes from review * backported to stable, 2.14 --------- Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> --- .../explore/query-1-performance/_index.md | 12 +++ .../pg-stat-progress-create-index.md | 98 +++++++++++++++++++ .../explore/query-1-performance/_index.md | 12 +++ .../pg-stat-progress-create-index.md | 98 +++++++++++++++++++ .../explore/query-1-performance/_index.md | 12 +++ .../pg-stat-progress-create-index.md | 98 +++++++++++++++++++ 6 files changed, 330 insertions(+) create mode 100644 docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md create mode 100644 docs/content/stable/explore/query-1-performance/pg-stat-progress-create-index.md create mode 100644 docs/content/v2.14/explore/query-1-performance/pg-stat-progress-create-index.md diff --git a/docs/content/preview/explore/query-1-performance/_index.md b/docs/content/preview/explore/query-1-performance/_index.md index 18ba17fc733c..082ab12ff9b3 100644 --- a/docs/content/preview/explore/query-1-performance/_index.md +++ b/docs/content/preview/explore/query-1-performance/_index.md @@ -75,6 +75,18 @@ type: indexpage
+
+ +
+ +
View CREATE INDEX status with pg_stat_progress_create_index
+
+
+ Get the CREATE INDEX command status, including the status of an ongoing concurrent index backfill, and the index build's progress reports. +
+ +
+
diff --git a/docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md b/docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md new file mode 100644 index 000000000000..0ba6cfd0d31f --- /dev/null +++ b/docs/content/preview/explore/query-1-performance/pg-stat-progress-create-index.md @@ -0,0 +1,98 @@ +--- +title: View CREATE INDEX status with pg_stat_progress_create_index +linkTitle: View CREATE INDEX status +description: Use pg_stat_progress_create_index to get the CREATE INDEX command status, including the status of an ongoing concurrent index backfill, and the index build's progress reports. +headerTitle: View CREATE INDEX status with pg_stat_progress_create_index +image: /images/section_icons/index/develop.png +menu: + preview: + identifier: pg-stat-progress-create-index + parent: query-tuning + weight: 450 +type: docs +--- + +You can add a new index to an existing table using the YSQL [CREATE INDEX](../../../api/ysql/the-sql-language/statements/ddl_create_index/#semantics) statement. YugabyteDB supports [online index backfill](https://github.com/yugabyte/yugabyte-db/blob/master/architecture/design/online-index-backfill.md), which is enabled by default. Using this feature, you can build indexes on non-empty tables online, without failing other concurrent writes. YugabyteDB also supports the [CREATE INDEX NONCONCURRENTLY](../../../api/ysql/the-sql-language/statements/ddl_create_index/#nonconcurrently) statement to disable online index backfill. + +### pg_stat_progress_create_index + +YugabyteDB supports the PostgreSQL `pg_stat_progress_create_index` view to report the progress of the CREATE INDEX command execution. The view contains one row for each backend connection that is currently running a CREATE INDEX command, and the row entry is cleared after the completion of command execution. + +The `pg_stat_progress_create_index` view can provide the following details: + +- Number of rows processed during an index backfill. +- The current phase of the command with `initializing` or `backfilling` as the possible phases. +- Index progress report for all the different configurations of an index or index build such as non-concurrent index builds, GIN indexes, partial indexes, and include indexes. + +The following table describes the view columns: + +| Column | Type | Description | +| :----- | :--- | :---------- | +| pid | integer | Process ID of backend that is running the `CREATE INDEX`. | +| datid | OID | Object ID of the database to which this backend is connected. | +| datname | name | Name of the database to which this backend is connected. | +| relid | OID | Object ID of the indexed relation.| +| index_relid | OID | Object ID of the index. | +| command | text | The command that is running `CREATE INDEX CONCURRENTLY`, or `CREATE INDEX NONCONCURRENTLY`. | +| phase | text | The current phase of the command. The possible phases are _initializing_, or _backfilling_. | +| tuples_total | bigint | Number of indexed table tuples already processed. | +| tuples_done | bigint | Estimate of total number of tuples (in the indexed table). This value is retrieved from `pg_class.reltuples`. | +| partitions_total | bigint | If the ongoing `CREATE INDEX` is for a partitioned table, this refers to the total number of partitions in the table. Set to 0 otherwise. | +| partitions_done | bigint | If the ongoing `CREATE INDEX` is for a partitioned table, this refers to the number of partitions the index has been created for. Set to 0 otherwise. | + +Columns such as `lockers_total`, `lockers_done`, `current_locker_pid`, `blocks_total`, and `blocks_done` are not applicable to YugabyteDB and always have null values. + +## YugabyteDB-specific changes + +The `pg_stat_progress_create_index` view includes the following YugabyteDB-specific changes: + +- In YugabyteDB, the `pg_stat_progress_create_index` view is a local view; it only has entries for CREATE INDEX commands issued by local YSQL clients. + +- In PostgreSQL, `tuples_done` and `tuples_total` refer to the tuples of the _index_. However, in YugabyteDB, these fields refer to the tuples of the _indexed table_. This discrepancy is only observed for partial indexes, where the reported progress is less than the actual progress. `tuples_total` is an estimate that is retrieved from `pg_class.reltuples`. + +- In YugabyteDB, `tuples_done` and `tuples_total` are not displayed (set to null) for temporary indexes. + +## Example + +The following example demonstrates the possible phases (initializing, backfilling) for the CREATE INDEX operation using the `pg_stat_progress_create_index` view. + +{{% explore-setup-single %}} + +1. From your local YugabyteDB installation directory, connect to the [YSQL](../../../admin/ysqlsh/) shell, and create an index on an existing table as follows: + + ```sql + CREATE TABLE customers(id int, customer_name text); + CREATE INDEX ON customers(customer_name); + ``` + +1. On a separate parallel YSQL connection on the same node, select from the view to see the progress of the command as follows: + + ```sql + SELECT * FROM pg_stat_progress_create_index; + ``` + + ```output + pid | datid | datname | relid | index_relid | command | phase | lockers_total | lockers_done | current_locker_pid | blocks_total | blocks_done | tuples_total | tuples_done | partitions_total | partitions_done + ------+-------+----------+-------+-------------+---------------------------+--------------+---------------+--------------+--------------------+--------------+-------------+--------------+-------------+------------------+----------------- + 78841 | 13291 | yugabyte | 16384 | 16390 | CREATE INDEX CONCURRENTLY | initializing | | | | | | 100000 | 0 | 0 | 0 + (1 row) + ``` + + ```sql + SELECT * FROM pg_stat_progress_create_index; + ``` + + ```output + pid | datid | datname | relid | index_relid | command | phase | lockers_total | lockers_done | current_locker_pid | blocks_total | blocks_done | tuples_total | tuples_done | partitions_total | partitions_done + ------+-------+----------+-------+-------------+---------------------------+-------------+---------------+--------------+--------------------+--------------+-------------+--------------+-------------+------------------+----------------- + 77444 | 13291 | yugabyte | 16404 | 16412 | CREATE INDEX CONCURRENTLY | backfilling | | | | | | 100000 | 49904 | 0 | 0 + (1 row) + ``` + +## Learn more + +- Refer to [View live queries with pg_stat_activity](../pg-stat-activity/) to analyze live queries. +- Refer to [View COPY progress with pg_stat_progress_copy](../pg-stat-progress-copy/) to track the COPY operation status. +- Refer to [Analyze queries with EXPLAIN](../explain-analyze/) to optimize YSQL's EXPLAIN and EXPLAIN ANALYZE queries. +- Refer to [Optimize YSQL queries using pg_hint_plan](../pg-hint-plan/) show the query execution plan generated by YSQL. +- Refer to [Get query statistics using pg_stat_statements](../pg-stat-statements/) to track planning and execution of all the SQL statements. diff --git a/docs/content/stable/explore/query-1-performance/_index.md b/docs/content/stable/explore/query-1-performance/_index.md index 4a28b8ba149e..0fb03168fd8e 100644 --- a/docs/content/stable/explore/query-1-performance/_index.md +++ b/docs/content/stable/explore/query-1-performance/_index.md @@ -74,6 +74,18 @@ type: indexpage
+
+ +
+ +
View CREATE INDEX status with pg_stat_progress_create_index
+
+
+ Get the CREATE INDEX command status, including the status of an ongoing concurrent index backfill, and the index build's progress reports. +
+ +
+
diff --git a/docs/content/stable/explore/query-1-performance/pg-stat-progress-create-index.md b/docs/content/stable/explore/query-1-performance/pg-stat-progress-create-index.md new file mode 100644 index 000000000000..11f2800c39b8 --- /dev/null +++ b/docs/content/stable/explore/query-1-performance/pg-stat-progress-create-index.md @@ -0,0 +1,98 @@ +--- +title: View CREATE INDEX status with pg_stat_progress_create_index +linkTitle: View CREATE INDEX status +description: Use pg_stat_progress_create_index to get the CREATE INDEX command status, including the status of an ongoing concurrent index backfill, and the index build's progress reports. +headerTitle: View CREATE INDEX status with pg_stat_progress_create_index +image: /images/section_icons/index/develop.png +menu: + stable: + identifier: pg-stat-progress-create-index + parent: query-tuning + weight: 450 +type: docs +--- + +You can add a new index to an existing table using the YSQL [CREATE INDEX](../../../api/ysql/the-sql-language/statements/ddl_create_index/#semantics) statement. YugabyteDB supports [online index backfill](https://github.com/yugabyte/yugabyte-db/blob/master/architecture/design/online-index-backfill.md), which is enabled by default. Using this feature, you can build indexes on non-empty tables online, without failing other concurrent writes. YugabyteDB also supports the [CREATE INDEX NONCONCURRENTLY](../../../api/ysql/the-sql-language/statements/ddl_create_index/#nonconcurrently) statement to disable online index backfill. + +### pg_stat_progress_create_index + +YugabyteDB supports the PostgreSQL `pg_stat_progress_create_index` view to report the progress of the CREATE INDEX command execution. The view contains one row for each backend connection that is currently running a CREATE INDEX command, and the row entry is cleared after the completion of command execution. + +The `pg_stat_progress_create_index` view can provide the following details: + +- Number of rows processed during an index backfill. +- The current phase of the command with `initializing` or `backfilling` as the possible phases. +- Index progress report for all the different configurations of an index or index build such as non-concurrent index builds, GIN indexes, partial indexes, and include indexes. + +The following table describes the view columns: + +| Column | Type | Description | +| :----- | :--- | :---------- | +| pid | integer | Process ID of backend that is running the `CREATE INDEX`. | +| datid | OID | Object ID of the database to which this backend is connected. | +| datname | name | Name of the database to which this backend is connected. | +| relid | OID | Object ID of the indexed relation.| +| index_relid | OID | Object ID of the index. | +| command | text | The command that is running `CREATE INDEX CONCURRENTLY`, or `CREATE INDEX NONCONCURRENTLY`. | +| phase | text | The current phase of the command. The possible phases are _initializing_, or _backfilling_. | +| tuples_total | bigint | Number of indexed table tuples already processed. | +| tuples_done | bigint | Estimate of total number of tuples (in the indexed table). This value is retrieved from `pg_class.reltuples`. | +| partitions_total | bigint | If the ongoing `CREATE INDEX` is for a partitioned table, this refers to the total number of partitions in the table. Set to 0 otherwise. | +| partitions_done | bigint | If the ongoing `CREATE INDEX` is for a partitioned table, this refers to the number of partitions the index has been created for. Set to 0 otherwise. | + +Columns such as `lockers_total`, `lockers_done`, `current_locker_pid`, `blocks_total`, and `blocks_done` are not applicable to YugabyteDB and always have null values. + +## YugabyteDB-specific changes + +The `pg_stat_progress_create_index` view includes the following YugabyteDB-specific changes: + +- In YugabyteDB, the `pg_stat_progress_create_index` view is a local view; it only has entries for CREATE INDEX commands issued by local YSQL clients. + +- In PostgreSQL, `tuples_done` and `tuples_total` refer to the tuples of the _index_. However, in YugabyteDB, these fields refer to the tuples of the _indexed table_. This discrepancy is only observed for partial indexes, where the reported progress is less than the actual progress. `tuples_total` is an estimate that is retrieved from `pg_class.reltuples`. + +- In YugabyteDB, `tuples_done` and `tuples_total` are not displayed (set to null) for temporary indexes. + +## Example + +The following example demonstrates the possible phases (initializing, backfilling) for the CREATE INDEX operation using the `pg_stat_progress_create_index` view. + +{{% explore-setup-single %}} + +1. From your local YugabyteDB installation directory, connect to the [YSQL](../../../admin/ysqlsh/) shell, and create an index on an existing table as follows: + + ```sql + CREATE TABLE customers(id int, customer_name text); + CREATE INDEX ON customers(customer_name); + ``` + +1. On a separate parallel YSQL connection on the same node, select from the view to see the progress of the command as follows: + + ```sql + SELECT * FROM pg_stat_progress_create_index; + ``` + + ```output + pid | datid | datname | relid | index_relid | command | phase | lockers_total | lockers_done | current_locker_pid | blocks_total | blocks_done | tuples_total | tuples_done | partitions_total | partitions_done + ------+-------+----------+-------+-------------+---------------------------+--------------+---------------+--------------+--------------------+--------------+-------------+--------------+-------------+------------------+----------------- + 78841 | 13291 | yugabyte | 16384 | 16390 | CREATE INDEX CONCURRENTLY | initializing | | | | | | 100000 | 0 | 0 | 0 + (1 row) + ``` + + ```sql + SELECT * FROM pg_stat_progress_create_index; + ``` + + ```output + pid | datid | datname | relid | index_relid | command | phase | lockers_total | lockers_done | current_locker_pid | blocks_total | blocks_done | tuples_total | tuples_done | partitions_total | partitions_done + ------+-------+----------+-------+-------------+---------------------------+-------------+---------------+--------------+--------------------+--------------+-------------+--------------+-------------+------------------+----------------- + 77444 | 13291 | yugabyte | 16404 | 16412 | CREATE INDEX CONCURRENTLY | backfilling | | | | | | 100000 | 49904 | 0 | 0 + (1 row) + ``` + +## Learn more + +- Refer to [View live queries with pg_stat_activity](../pg-stat-activity/) to analyze live queries. +- Refer to [View COPY progress with pg_stat_progress_copy](../pg-stat-progress-copy/) to track the COPY operation status. +- Refer to [Analyze queries with EXPLAIN](../explain-analyze/) to optimize YSQL's EXPLAIN and EXPLAIN ANALYZE queries. +- Refer to [Optimize YSQL queries using pg_hint_plan](../pg-hint-plan/) show the query execution plan generated by YSQL. +- Refer to [Get query statistics using pg_stat_statements](../pg-stat-statements/) to track planning and execution of all the SQL statements. diff --git a/docs/content/v2.14/explore/query-1-performance/_index.md b/docs/content/v2.14/explore/query-1-performance/_index.md index a4b6f06ca4e7..2d86a8d457db 100644 --- a/docs/content/v2.14/explore/query-1-performance/_index.md +++ b/docs/content/v2.14/explore/query-1-performance/_index.md @@ -74,6 +74,18 @@ type: indexpage
+
+ +
+ +
View CREATE INDEX status with pg_stat_progress_create_index
+
+
+ Get the CREATE INDEX command status, including the status of an ongoing concurrent index backfill, and the index build's progress reports. +
+ +
+
diff --git a/docs/content/v2.14/explore/query-1-performance/pg-stat-progress-create-index.md b/docs/content/v2.14/explore/query-1-performance/pg-stat-progress-create-index.md new file mode 100644 index 000000000000..9cacaa031634 --- /dev/null +++ b/docs/content/v2.14/explore/query-1-performance/pg-stat-progress-create-index.md @@ -0,0 +1,98 @@ +--- +title: View CREATE INDEX status with pg_stat_progress_create_index +linkTitle: View CREATE INDEX status +description: Use pg_stat_progress_create_index to get the CREATE INDEX command status, including the status of an ongoing concurrent index backfill, and the index build's progress reports. +headerTitle: View CREATE INDEX status with pg_stat_progress_create_index +image: /images/section_icons/index/develop.png +menu: + v2.14: + identifier: pg-stat-progress-create-index + parent: query-tuning + weight: 450 +type: docs +--- + +You can add a new index to an existing table using the YSQL [CREATE INDEX](../../../api/ysql/the-sql-language/statements/ddl_create_index/#semantics) statement. YugabyteDB supports [online index backfill](https://github.com/yugabyte/yugabyte-db/blob/master/architecture/design/online-index-backfill.md), which is enabled by default. Using this feature, you can build indexes on non-empty tables online, without failing other concurrent writes. YugabyteDB also supports the [CREATE INDEX NONCONCURRENTLY](../../../api/ysql/the-sql-language/statements/ddl_create_index/#nonconcurrently) statement to disable online index backfill. + +### pg_stat_progress_create_index + +YugabyteDB supports the PostgreSQL `pg_stat_progress_create_index` view to report the progress of the CREATE INDEX command execution. The view contains one row for each backend connection that is currently running a CREATE INDEX command, and the row entry is cleared after the completion of command execution. + +The `pg_stat_progress_create_index` view can provide the following details: + +- Number of rows processed during an index backfill. +- The current phase of the command with `initializing` or `backfilling` as the possible phases. +- Index progress report for all the different configurations of an index or index build such as non-concurrent index builds, GIN indexes, partial indexes, and include indexes. + +The following table describes the view columns: + +| Column | Type | Description | +| :----- | :--- | :---------- | +| pid | integer | Process ID of backend that is running the `CREATE INDEX`. | +| datid | OID | Object ID of the database to which this backend is connected. | +| datname | name | Name of the database to which this backend is connected. | +| relid | OID | Object ID of the indexed relation.| +| index_relid | OID | Object ID of the index. | +| command | text | The command that is running `CREATE INDEX CONCURRENTLY`, or `CREATE INDEX NONCONCURRENTLY`. | +| phase | text | The current phase of the command. The possible phases are _initializing_, or _backfilling_. | +| tuples_total | bigint | Number of indexed table tuples already processed. | +| tuples_done | bigint | Estimate of total number of tuples (in the indexed table). This value is retrieved from `pg_class.reltuples`. | +| partitions_total | bigint | If the ongoing `CREATE INDEX` is for a partitioned table, this refers to the total number of partitions in the table. Set to 0 otherwise. | +| partitions_done | bigint | If the ongoing `CREATE INDEX` is for a partitioned table, this refers to the number of partitions the index has been created for. Set to 0 otherwise. | + +Columns such as `lockers_total`, `lockers_done`, `current_locker_pid`, `blocks_total`, and `blocks_done` are not applicable to YugabyteDB and always have null values. + +## YugabyteDB-specific changes + +The `pg_stat_progress_create_index` view includes the following YugabyteDB-specific changes: + +- In YugabyteDB, the `pg_stat_progress_create_index` view is a local view; it only has entries for CREATE INDEX commands issued by local YSQL clients. + +- In PostgreSQL, `tuples_done` and `tuples_total` refer to the tuples of the _index_. However, in YugabyteDB, these fields refer to the tuples of the _indexed table_. This discrepancy is only observed for partial indexes, where the reported progress is less than the actual progress. `tuples_total` is an estimate that is retrieved from `pg_class.reltuples`. + +- In YugabyteDB, `tuples_done` and `tuples_total` are not displayed (set to null) for temporary indexes. + +## Example + +The following example demonstrates the possible phases (initializing, backfilling) for the CREATE INDEX operation using the `pg_stat_progress_create_index` view. + +{{% explore-setup-single %}} + +1. From your local YugabyteDB installation directory, connect to the [YSQL](../../../admin/ysqlsh/) shell, and create an index on an existing table as follows: + + ```sql + CREATE TABLE customers(id int, customer_name text); + CREATE INDEX ON customers(customer_name); + ``` + +1. On a separate parallel YSQL connection on the same node, select from the view to see the progress of the command as follows: + + ```sql + SELECT * FROM pg_stat_progress_create_index; + ``` + + ```output + pid | datid | datname | relid | index_relid | command | phase | lockers_total | lockers_done | current_locker_pid | blocks_total | blocks_done | tuples_total | tuples_done | partitions_total | partitions_done + ------+-------+----------+-------+-------------+---------------------------+--------------+---------------+--------------+--------------------+--------------+-------------+--------------+-------------+------------------+----------------- + 78841 | 13291 | yugabyte | 16384 | 16390 | CREATE INDEX CONCURRENTLY | initializing | | | | | | 100000 | 0 | 0 | 0 + (1 row) + ``` + + ```sql + SELECT * FROM pg_stat_progress_create_index; + ``` + + ```output + pid | datid | datname | relid | index_relid | command | phase | lockers_total | lockers_done | current_locker_pid | blocks_total | blocks_done | tuples_total | tuples_done | partitions_total | partitions_done + ------+-------+----------+-------+-------------+---------------------------+-------------+---------------+--------------+--------------------+--------------+-------------+--------------+-------------+------------------+----------------- + 77444 | 13291 | yugabyte | 16404 | 16412 | CREATE INDEX CONCURRENTLY | backfilling | | | | | | 100000 | 49904 | 0 | 0 + (1 row) + ``` + +## Learn more + +- Refer to [View live queries with pg_stat_activity](../pg-stat-activity/) to analyze live queries. +- Refer to [View COPY progress with pg_stat_progress_copy](../pg-stat-progress-copy/) to track the COPY operation status. +- Refer to [Analyze queries with EXPLAIN](../explain-analyze/) to optimize YSQL's EXPLAIN and EXPLAIN ANALYZE queries. +- Refer to [Optimize YSQL queries using pg_hint_plan](../pg-hint-plan/) show the query execution plan generated by YSQL. +- Refer to [Get query statistics using pg_stat_statements](../pg-stat-statements/) to track planning and execution of all the SQL statements. From f69dc2a3a854ece887d2a98c97ee66fe400562e5 Mon Sep 17 00:00:00 2001 From: rsami Date: Tue, 28 Feb 2023 15:47:49 -0500 Subject: [PATCH 23/81] [#13580] DocDB: Improve fairness in wait queues Summary: The main contribution of this revision is to drastically improve p99 performance of workloads using wait-on-conflict concurrency control under high contention, without harming p50 or average performance under normal amounts of contention. We achieve this by making the following improvements: 1. Force incoming requests to check the wait queue once for active blockers, to ensure incoming requests cannot starve waiting transactions which are racing to exit the wait queue 2. Assign serial numbers to incoming requests, and whenever a batch of waiters can be resumed at the same time, ensure they are resumed roughly in the order in which they arrived at the tserver Additional enhancements include: 1. Reduce copying by consolidating on using TransactionData everywhere, which is pulled into a conflict_data.h file with associated data structures 2. Populate granular intent information on a sub-transaction basis for use by the wait queue 3. Piggy-back off transaction status request in conflict resolution to obtain status tablet info Test Plan: Performance was tested on a 16 core 32gb ram alma8 server with a full-LTO release build. In both cases we used the following set-up: ``` create table test (k int primary key, v int); insert into test select generate_series(0, 11), 0; ``` In both cases, we also ran ysql_bench as follows: ``` build/latest/postgres/bin/ysql_bench --transactions=2000 --jobs=16 --client=16 --file=workload.sql --progress=1 --debug=fails ``` = First test: Max contention = `workload.sql` ``` begin; select * from test where k=1 for update; commit; ``` Baseline: ``` latency average = 19.779 ms latency stddev = 26.684 ms tps = 792.780284 (including connections establishing) tps = 793.793930 (excluding connections establishing) ``` With revision: ``` latency average = 22.632 ms latency stddev = 3.266 ms tps = 705.108285 (including connections establishing) tps = 705.914647 (excluding connections establishing) ``` = Second test: Normal contention = `workload.sql` ``` begin; with rand as (select floor(random() * 10 + 1)::int as k) select * from test join rand on rand.k=test.k for update; commit; ``` Baseline: ``` latency average = 7.317 ms latency stddev = 6.516 ms tps = 2117.437801 (including connections establishing) tps = 2126.594897 (excluding connections establishing) ``` With revision: ``` latency average = 7.055 ms latency stddev = 5.124 ms tps = 2236.062486 (including connections establishing) tps = 2244.260708 (excluding connections establishing) ``` ==Takeaways== 1. stddev of latency is substantially improved with this revision, at the expense of a 25% drop in throughput and 35% increase in latency 2. Throughput is not significantly changed in normal contention case Reviewers: pjain, bkolagani, sergei Reviewed By: sergei Subscribers: mbautin, rthallam, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D22968 --- src/yb/common/transaction-test-util.h | 2 - src/yb/common/transaction.h | 5 +- src/yb/docdb/CMakeLists.txt | 1 + src/yb/docdb/conflict_data.cc | 119 +++++ src/yb/docdb/conflict_data.h | 124 +++++ src/yb/docdb/conflict_resolution.cc | 301 +++++------ src/yb/docdb/docdb.cc | 16 +- src/yb/docdb/docdb_test_util.cc | 4 - src/yb/docdb/intent.cc | 19 + src/yb/docdb/intent.h | 7 + src/yb/docdb/local_waiting_txn_registry.cc | 20 +- src/yb/docdb/local_waiting_txn_registry.h | 8 +- src/yb/docdb/lock_batch.h | 2 + src/yb/docdb/wait_queue.cc | 580 +++++++++++++++++---- src/yb/docdb/wait_queue.h | 31 +- src/yb/tablet/running_transaction.cc | 4 + src/yb/tablet/tablet_options.h | 2 +- src/yb/tablet/transaction_participant.cc | 13 - src/yb/tablet/transaction_participant.h | 2 - src/yb/tablet/write_query.cc | 3 - src/yb/tablet/write_query.h | 5 - src/yb/tserver/read_query.cc | 8 +- src/yb/tserver/ts_tablet_manager.cc | 2 +- src/yb/tserver/ts_tablet_manager.h | 2 +- src/yb/tserver/tserver-path-handlers.cc | 23 +- src/yb/util/lazy_invoke.h | 39 ++ 26 files changed, 1019 insertions(+), 323 deletions(-) create mode 100644 src/yb/docdb/conflict_data.cc create mode 100644 src/yb/docdb/conflict_data.h create mode 100644 src/yb/util/lazy_invoke.h diff --git a/src/yb/common/transaction-test-util.h b/src/yb/common/transaction-test-util.h index a30435237791..8b8ac9f30ae9 100644 --- a/src/yb/common/transaction-test-util.h +++ b/src/yb/common/transaction-test-util.h @@ -69,8 +69,6 @@ class TransactionStatusManagerMock : public TransactionStatusManager { void FillPriorities( boost::container::small_vector_base>* inout) override {} - void FillStatusTablets(std::vector* inout) override { } - boost::optional FindStatusTablet(const TransactionId& id) override { return boost::none; } diff --git a/src/yb/common/transaction.h b/src/yb/common/transaction.h index 3dc0554efd04..640d001e5671 100644 --- a/src/yb/common/transaction.h +++ b/src/yb/common/transaction.h @@ -127,6 +127,9 @@ struct StatusRequest { const std::string* reason; TransactionLoadFlags flags; TransactionStatusCallback callback; + // If non-null, populate status_tablet_id for known transactions in the same thread the request is + // initiated. + std::string* status_tablet_id = nullptr; std::string ToString() const { return Format("{ id: $0 read_ht: $1 global_limit_ht: $2 serial_no: $3 reason: $4 flags: $5}", @@ -177,8 +180,6 @@ class TransactionStatusManager { virtual void FillPriorities( boost::container::small_vector_base>* inout) = 0; - virtual void FillStatusTablets(std::vector* inout) = 0; - virtual boost::optional FindStatusTablet(const TransactionId& id) = 0; // Returns minimal running hybrid time of all running transactions. diff --git a/src/yb/docdb/CMakeLists.txt b/src/yb/docdb/CMakeLists.txt index b42811cc577a..cdb6ab39d90a 100644 --- a/src/yb/docdb/CMakeLists.txt +++ b/src/yb/docdb/CMakeLists.txt @@ -57,6 +57,7 @@ include_directories("${YB_BUILD_ROOT}/postgres_build/src/include/catalog") set(DOCDB_SRCS bounded_rocksdb_iterator.cc + conflict_data.cc conflict_resolution.cc consensus_frontier.cc cql_operation.cc diff --git a/src/yb/docdb/conflict_data.cc b/src/yb/docdb/conflict_data.cc new file mode 100644 index 000000000000..acc07a47a5f3 --- /dev/null +++ b/src/yb/docdb/conflict_data.cc @@ -0,0 +1,119 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#include "yb/docdb/conflict_data.h" +#include "yb/common/transaction.h" + +namespace yb::docdb { + +void TransactionConflictInfo::RemoveAbortedSubtransactions(const SubtxnSet& aborted_subtxn_set) { + auto it = subtransactions.begin(); + while (it != subtransactions.end()) { + if (aborted_subtxn_set.Test(it->first)) { + it = subtransactions.erase(it); + } else { + it++; + } + } +} + +std::string TransactionConflictInfo::ToString() const { + return Format("{ subtransactions: $0 }", subtransactions.size()); +} + +TransactionConflictData::TransactionConflictData( + TransactionId id_, const TransactionConflictInfoPtr& conflict_info_, + const TabletId& status_tablet_): + id(id_), conflict_info(DCHECK_NOTNULL(conflict_info_)), status_tablet(status_tablet_) {} + +void TransactionConflictData::ProcessStatus(const TransactionStatusResult& result) { + status = result.status; + if (status == TransactionStatus::COMMITTED) { + LOG_IF(DFATAL, !result.status_time.is_valid()) + << "Status time not specified for committed transaction: " << id; + commit_time = result.status_time; + } + if (status != TransactionStatus::ABORTED) { + DCHECK(status == TransactionStatus::PENDING || status == TransactionStatus::COMMITTED); + conflict_info->RemoveAbortedSubtransactions(result.aborted_subtxn_set); + } +} + +std::string TransactionConflictData::ToString() const { + return Format("{ id: $0 status: $1 commit_time: $2 priority: $3 failure: $4 }", + id, TransactionStatus_Name(status), commit_time, priority, failure); +} + + +ConflictDataManager::ConflictDataManager(size_t capacity) { + transactions_.reserve(capacity); +} + +void ConflictDataManager::AddTransaction( + const TransactionId& id, const TransactionConflictInfoPtr& conflict_info, + const TabletId& status_tablet) { +#ifndef NDEBUG + DCHECK(!has_filtered_); +#endif // NDEBUG + transactions_.emplace_back(id, conflict_info, status_tablet); + remaining_transactions_ = transactions_.size(); +} + +boost::iterator_range ConflictDataManager::RemainingTransactions() { + auto begin = transactions_.data(); + return boost::make_iterator_range(begin, begin + remaining_transactions_); +} + +size_t ConflictDataManager::NumActiveTransactions() const { + return remaining_transactions_; +} + +// Apply specified functor to all active (i.e. not resolved) transactions. +// If functor returns true, it means that transaction was resolved. +// So such transaction is moved out of active transactions range. +// Returns true if there are no active transaction left. +Result ConflictDataManager::FilterInactiveTransactions(const InactiveTxnFilter& filter) { +#ifndef NDEBUG + has_filtered_.exchange(true); +#endif // NDEBUG + + auto end = transactions_.begin() + remaining_transactions_; + for (auto transaction_it = transactions_.begin(); transaction_it != end;) { + if (!VERIFY_RESULT(filter(&*transaction_it))) { + ++transaction_it; + continue; + } + if (--end == transaction_it) { + break; + } + std::swap(*transaction_it, *end); + } + remaining_transactions_ = end - transactions_.begin(); + + return remaining_transactions_ == 0; +} + +Slice ConflictDataManager::DumpConflicts() const { + return Slice(pointer_cast(transactions_.data()), + transactions_.size() * sizeof(transactions_[0])); +} + +std::string ConflictDataManager::ToString() const { + return AsString(transactions_); +} + +std::ostream& operator<<(std::ostream& out, const ConflictDataManager& data) { + return out << data.ToString(); +} + +} // namespace yb::docdb diff --git a/src/yb/docdb/conflict_data.h b/src/yb/docdb/conflict_data.h new file mode 100644 index 000000000000..3cb548143445 --- /dev/null +++ b/src/yb/docdb/conflict_data.h @@ -0,0 +1,124 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#pragma once + +#include "yb/common/entity_ids_types.h" +#include "yb/common/hybrid_time.h" +#include "yb/common/transaction.h" +#include "yb/common/transaction.pb.h" +#include "yb/docdb/intent.h" +#include "yb/util/ref_cnt_buffer.h" +#include "yb/util/status.h" + +namespace yb::docdb { + +// Key and intent type info for a lock or conflicting write. +struct LockInfo { + const RefCntPrefix doc_path; + const IntentTypeSet intent_types; +}; + +// Represents the conflicts found from a specific subtransaction. +struct SubTransactionConflictInfo { + bool has_non_lock_conflict; + std::vector locks; +}; + +// Represents the conflicts found from a specific transaction. May be updated based on the aborted +// subtransactions of that transaction. +struct TransactionConflictInfo { + std::unordered_map subtransactions; + + // Remove any entries from subtransactions which appear to be aborted according to + // aborted_subtxn_set. + void RemoveAbortedSubtransactions(const SubtxnSet& aborted_subtxn_set); + + std::string ToString() const; +}; +using TransactionConflictInfoPtr = std::shared_ptr; + +// Represents the state of a transaction found to be in conflict with a request during conflict +// resolution. Used throughout the lifecycle of conflict resolution, including in the wait queue and +// in registering wait-for relationships to send to coordinators for deadlock detection. +struct TransactionConflictData { + TransactionId id; + TransactionConflictInfoPtr conflict_info; + TransactionStatus status; + HybridTime commit_time; + uint64_t priority; + Status failure; + TabletId status_tablet = ""; + + TransactionConflictData(TransactionId id_, const TransactionConflictInfoPtr& conflict_info_, + const TabletId& status_tablet_); + + // Update the state of this transaction based on result, including status, commit_time, failure, + // and aborted subtransactions. + void ProcessStatus(const TransactionStatusResult& result); + + std::string ToString() const; +}; + +// This class is responsible for managing conflicting transaction status during conflict resolution +// and providing convenient access to TransactionData for all active conflicting transactions +// as a request moves from conflict resolution, to the wait queue, and reports wait-for information +// to deadlock detection via LocalWaitingTxnRegistry. +// +// When using this class, all transactions should be added before any calls to +// FilterInactiveTransactions. Once FilterInactiveTransactions is called, it is invalid to call +// AddTransaction. The typical lifecycle is as follows: +// 1. Construct a ConflictDataManager instance +// 2. Add transactions +// 3. Filter out transactions based on local participant status or as transaction status RPCs +// return their status +// 4. Pass around ConflictDataManager instance to enable easy access to remaining transaction data +// +// Deviating from this pattern by e.g. adding transactions after some have been filtered will cause +// crashes in debug builds and incorrect behavior in release builds. +class ConflictDataManager { + public: + explicit ConflictDataManager(size_t capacity); + + void AddTransaction( + const TransactionId& id, const TransactionConflictInfoPtr& conflict_info, + const TabletId& status_tablet = ""); + + boost::iterator_range RemainingTransactions(); + + size_t NumActiveTransactions() const; + + // Apply specified functor to all active (i.e. not resolved) transactions. + // If functor returns true, it means that transaction was resolved. + // So such transaction is moved out of active transactions range. + // Returns true if there are no active transaction left. + using InactiveTxnFilter = std::function(TransactionConflictData*)>; + Result FilterInactiveTransactions(const InactiveTxnFilter& filter); + + // Output a Slice of data representing the state of this instance, as expected by the + // YB_TRANSACTION_DUMP macro. + Slice DumpConflicts() const; + + std::string ToString() const; + + private: + std::vector transactions_; + size_t remaining_transactions_; +#ifndef NDEBUG + std::atomic_bool has_filtered_ = false; +#endif // NDEBUG +}; + +std::ostream& operator<<(std::ostream& out, const ConflictDataManager& data); + +} // namespace yb::docdb diff --git a/src/yb/docdb/conflict_resolution.cc b/src/yb/docdb/conflict_resolution.cc index d72331a8eea4..fe7772ed04d4 100644 --- a/src/yb/docdb/conflict_resolution.cc +++ b/src/yb/docdb/conflict_resolution.cc @@ -29,9 +29,11 @@ #include "yb/docdb/shared_lock_manager.h" #include "yb/docdb/transaction_dump.h" #include "yb/gutil/stl_util.h" +#include "yb/util/lazy_invoke.h" #include "yb/util/logging.h" #include "yb/util/memory/memory.h" #include "yb/util/metrics.h" +#include "yb/util/ref_cnt_buffer.h" #include "yb/util/scope_exit.h" #include "yb/util/status_format.h" #include "yb/util/trace.h" @@ -55,52 +57,9 @@ namespace { // non-modification intents, we don't have to consider them for conflicts. using TransactionConflictInfoMap = std::unordered_map, + TransactionConflictInfoPtr, TransactionIdHash>; -struct TransactionData { - TransactionData( - TransactionId id_, - std::shared_ptr subtransactions_) - : id(id_), subtransactions(subtransactions_) {} - - TransactionId id; - std::shared_ptr subtransactions; - TransactionStatus status; - HybridTime commit_time; - uint64_t priority; - Status failure; - - void RemoveAbortedSubtransactions(const SubtxnSet& aborted_subtxn_set) { - auto it = subtransactions->begin(); - while (it != subtransactions->end()) { - if (aborted_subtxn_set.Test(it->first)) { - it = subtransactions->erase(it); - } else { - it++; - } - } - } - - void ProcessStatus(const TransactionStatusResult& result) { - status = result.status; - if (status == TransactionStatus::COMMITTED) { - LOG_IF(DFATAL, !result.status_time.is_valid()) - << "Status time not specified for committed transaction: " << id; - commit_time = result.status_time; - } - if (status != TransactionStatus::ABORTED) { - DCHECK(status == TransactionStatus::PENDING || status == TransactionStatus::COMMITTED); - RemoveAbortedSubtransactions(result.aborted_subtxn_set); - } - } - - std::string ToString() const { - return Format("{ id: $0 status: $1 commit_time: $2 priority: $3 failure: $4 }", - id, TransactionStatus_Name(status), commit_time, priority, failure); - } -}; - Status MakeConflictStatus(const TransactionId& our_id, const TransactionId& other_id, const char* reason, Counter* conflicts_metric) { conflicts_metric->Increment(); @@ -118,15 +77,15 @@ class ConflictResolverContext { // Check priority of this one against existing transactions. virtual Status CheckPriority( ConflictResolver* resolver, - boost::iterator_range transactions) = 0; + boost::iterator_range transactions) = 0; // Check subtransaction data of pending transaction to determine if conflict can be avoided. - virtual bool CheckConflictWithPending(const TransactionData& transaction_data) = 0; + virtual bool CheckConflictWithPending(const TransactionConflictData& transaction_data) = 0; // Check for conflict against committed transaction. // Returns true if transaction could be removed from list of conflicts. virtual Result CheckConflictWithCommitted( - const TransactionData& transaction_data, HybridTime commit_time) = 0; + const TransactionConflictData& transaction_data, HybridTime commit_time) = 0; virtual HybridTime GetResolutionHt() = 0; @@ -140,6 +99,8 @@ class ConflictResolverContext { virtual ConflictManagementPolicy GetConflictManagementPolicy() const = 0; + virtual bool IsSingleShardTransaction() const = 0; + std::string LogPrefix() const { return ToString() + ": "; } @@ -151,11 +112,10 @@ class ConflictResolver : public std::enable_shared_from_this { public: ConflictResolver(const DocDB& doc_db, TransactionStatusManager* status_manager, - RequestScope request_scope, PartialRangeKeyIntents partial_range_key_intents, std::unique_ptr context, ResolutionCallback callback) - : doc_db_(doc_db), status_manager_(*status_manager), request_scope_(std::move(request_scope)), + : doc_db_(doc_db), status_manager_(*status_manager), partial_range_key_intents_(partial_range_key_intents), context_(std::move(context)), callback_(std::move(callback)) {} @@ -183,7 +143,10 @@ class ConflictResolver : public std::enable_shared_from_this { } void Resolve() { - auto status = context_->ReadConflicts(this); + auto status = SetRequestScope(); + if (status.ok()) { + status = context_->ReadConflicts(this); + } if (!status.ok()) { InvokeCallback(status); return; @@ -192,14 +155,17 @@ class ConflictResolver : public std::enable_shared_from_this { ResolveConflicts(); } - // Reset all state to prepare for running conflict resolution again. - void Reset() { + std::shared_ptr ConsumeTransactionDataAndReset() { + auto conflict_data = std::move(conflict_data_); + DCHECK(!conflict_data_); + + request_scope_ = RequestScope(); intent_iter_.Reset(); DCHECK(intent_key_upperbound_.empty()); conflicts_.clear(); - transactions_.clear(); - remaining_transactions_ = 0; DCHECK_EQ(pending_requests_.load(std::memory_order_acquire), 0); + + return conflict_data; } // Reads conflicts for specified intent from DB. @@ -264,8 +230,20 @@ class ConflictResolver : public std::enable_shared_from_this { << ", body: " << decoded_value.body.ToDebugHexString(); if (!context_->IgnoreConflictsWith(transaction_id)) { - auto p = conflicts_.emplace(transaction_id, std::make_shared()); - (*p.first->second)[decoded_value.subtransaction_id] |= !lock_only; + auto it = conflicts_.try_emplace( + transaction_id, + MakeLazyFactory([]() { return std::make_shared(); })); + auto& txn_conflict_info = DCHECK_NOTNULL(it.first->second); + auto& subtxn_conflict_info = txn_conflict_info->subtransactions[ + decoded_value.subtransaction_id]; + subtxn_conflict_info.has_non_lock_conflict |= !lock_only; + + if (RecordLockInfo()) { + auto doc_path = RefCntPrefix(existing_intent.doc_path); + RETURN_NOT_OK(RemoveGroupEndSuffix(&doc_path)); + auto lock_it = subtxn_conflict_info.locks.emplace_back( + LockInfo {std::move(doc_path), existing_intent.types}); + } } } @@ -289,12 +267,15 @@ class ConflictResolver : public std::enable_shared_from_this { } protected: + Status SetRequestScope() { + request_scope_ = VERIFY_RESULT(RequestScope::Create(&status_manager())); + return Status::OK(); + } + void InvokeCallback(const Result& result) { YB_TRANSACTION_DUMP( Conflicts, context_->transaction_id(), - result.ok() ? *result : HybridTime::kInvalid, - Slice(pointer_cast(transactions_.data()), - transactions_.size() * sizeof(transactions_[0]))); + result.ok() ? *result : HybridTime::kInvalid, conflict_data_->DumpConflicts()); intent_iter_.Reset(); callback_(result); } @@ -326,13 +307,10 @@ class ConflictResolver : public std::enable_shared_from_this { return; } - transactions_.reserve(conflicts_.size()); - for (const auto& kv : conflicts_) { - transactions_.emplace_back(kv.first /* id */, - // TODO - avoid potential copy here. - kv.second); + conflict_data_ = std::make_shared(conflicts_.size()); + for (const auto& [id, conflict_info] : conflicts_) { + conflict_data_->AddTransaction(id, conflict_info); } - remaining_transactions_ = transactions_.size(); DoResolveConflicts(); } @@ -361,21 +339,23 @@ class ConflictResolver : public std::enable_shared_from_this { virtual Status OnConflictingTransactionsFound() = 0; + virtual bool RecordLockInfo() const = 0; + // Returns true when there are no conflicts left. Result CheckLocalCommits() { - return DoCleanup([this](auto* transaction) -> Result { + return conflict_data_->FilterInactiveTransactions([this](auto* transaction) -> Result { return this->CheckLocalRunningTransaction(transaction); }); } // Check whether specified transaction was locally committed, and store this state if so. // Returns true if conflict with specified transaction is resolved. - Result CheckLocalRunningTransaction(TransactionData* transaction) { + Result CheckLocalRunningTransaction(TransactionConflictData* transaction) { auto local_txn_data = status_manager().LocalTxnData(transaction->id); if (!local_txn_data) { return false; } - transaction->RemoveAbortedSubtransactions(local_txn_data->aborted_subtxn_set); + transaction->conflict_info->RemoveAbortedSubtransactions(local_txn_data->aborted_subtxn_set); auto commit_time = local_txn_data->commit_ht; if (commit_time.is_valid()) { transaction->commit_time = commit_time; @@ -389,37 +369,15 @@ class ConflictResolver : public std::enable_shared_from_this { return context_->CheckConflictWithPending(*transaction); } - // Apply specified functor to all active (i.e. not resolved) transactions. - // If functor returns true, it means that transaction was resolved. - // So such transaction is moved out of active transactions range. - // Returns true if there are no active transaction left. - template - Result DoCleanup(const F& f) { - auto end = transactions_.begin() + remaining_transactions_; - for (auto transaction = transactions_.begin(); transaction != end;) { - if (!VERIFY_RESULT(f(&*transaction))) { - ++transaction; - continue; - } - if (--end == transaction) { - break; - } - std::swap(*transaction, *end); - } - remaining_transactions_ = end - transactions_.begin(); - - return remaining_transactions_ == 0; - } - // Removes all transactions that would not conflict with us anymore. // Returns failure if we conflict with transaction that cannot be aborted. Result Cleanup() { - return DoCleanup([this](auto* transaction) -> Result { + return conflict_data_->FilterInactiveTransactions([this](auto* transaction) -> Result { return this->CheckCleanup(transaction); }); } - Result CheckCleanup(TransactionData* transaction) { + Result CheckCleanup(TransactionConflictData* transaction) { RETURN_NOT_OK(transaction->failure); auto status = transaction->status; if (status == TransactionStatus::COMMITTED) { @@ -455,16 +413,11 @@ class ConflictResolver : public std::enable_shared_from_this { return false; } - boost::iterator_range RemainingTransactions() { - auto begin = transactions_.data(); - return boost::make_iterator_range(begin, begin + remaining_transactions_); - } - void FetchTransactionStatuses() { static const std::string kRequestReason = "conflict resolution"s; auto self = shared_from_this(); - pending_requests_.store(remaining_transactions_); - for (auto& i : RemainingTransactions()) { + pending_requests_.store(conflict_data_->NumActiveTransactions()); + for (auto& i : conflict_data_->RemainingTransactions()) { auto& transaction = i; TRACE("FetchingTransactionStatus for $0", yb::ToString(transaction.id)); StatusRequest request = { @@ -486,10 +439,13 @@ class ConflictResolver : public std::enable_shared_from_this { } else { transaction.failure = result.status(); } + DCHECK(!transaction.status_tablet.empty() || + transaction.status != TransactionStatus::PENDING); if (self->pending_requests_.fetch_sub(1, std::memory_order_acq_rel) == 1) { self->FetchTransactionStatusesDone(); } - } + }, + &transaction.status_tablet, }; status_manager().RequestStatusAt(request); } @@ -510,10 +466,7 @@ class ConflictResolver : public std::enable_shared_from_this { Slice intent_key_upperbound_; TransactionConflictInfoMap conflicts_; - // Resolution state for all transactions. Resolved transactions are moved to the end of it. - std::vector transactions_; - // Number of transactions that are not yet resolved. After successful resolution should be 0. - size_t remaining_transactions_; + std::shared_ptr conflict_data_; std::atomic pending_requests_{0}; }; @@ -532,33 +485,36 @@ class FailOnConflictResolver : public ConflictResolver { FailOnConflictResolver( const DocDB& doc_db, TransactionStatusManager* status_manager, - RequestScope request_scope, PartialRangeKeyIntents partial_range_key_intents, std::unique_ptr context, ResolutionCallback callback) - : ConflictResolver( - doc_db, status_manager, std::move(request_scope), partial_range_key_intents, - std::move(context), std::move(callback)) + : ConflictResolver( + doc_db, status_manager, partial_range_key_intents, std::move(context), + std::move(callback)) {} Status OnConflictingTransactionsFound() override { - DCHECK_GT(remaining_transactions_, 0); + DCHECK_GT(conflict_data_->NumActiveTransactions(), 0); if (context_->GetConflictManagementPolicy() == SKIP_ON_CONFLICT) { return STATUS(InternalError, "Skip locking since entity is already locked", TransactionError(TransactionErrorCode::kSkipLocking)); } - RETURN_NOT_OK(context_->CheckPriority(this, RemainingTransactions())); + RETURN_NOT_OK(context_->CheckPriority(this, conflict_data_->RemainingTransactions())); AbortTransactions(); return Status::OK(); } + bool RecordLockInfo() const override { + return false; + } + private: void AbortTransactions() { auto self = shared_from(this); - pending_requests_.store(remaining_transactions_); - for (auto& i : RemainingTransactions()) { + pending_requests_.store(conflict_data_->NumActiveTransactions()); + for (auto& i : conflict_data_->RemainingTransactions()) { auto& transaction = i; TRACE("Aborting $0", yb::ToString(transaction.id)); status_manager().Abort( @@ -594,45 +550,60 @@ class WaitOnConflictResolver : public ConflictResolver { WaitOnConflictResolver( const DocDB& doc_db, TransactionStatusManager* status_manager, - RequestScope request_scope, PartialRangeKeyIntents partial_range_key_intents, std::unique_ptr context, ResolutionCallback callback, WaitQueue* wait_queue, LockBatch* lock_batch) : ConflictResolver( - doc_db, status_manager, std::move(request_scope), partial_range_key_intents, - std::move(context), std::move(callback)), wait_queue_(wait_queue), - lock_batch_(lock_batch) {} + doc_db, status_manager, partial_range_key_intents, std::move(context), std::move(callback)), + wait_queue_(wait_queue), lock_batch_(lock_batch), serial_no_(wait_queue_->GetSerialNo()) {} + + ~WaitOnConflictResolver() { + VLOG(3) << "Wait-on-Conflict resolution complete after " << wait_for_iters_ << " iters."; + } + + void Run() { + if (context_->IsSingleShardTransaction()) { + return ConflictResolver::Resolve(); + } + + auto status_tablet_res = context_->GetStatusTablet(this); + if (status_tablet_res.ok()) { + status_tablet_id_ = std::move(*status_tablet_res); + TryPreWait(); + } else { + InvokeCallback(status_tablet_res.status()); + } + } + + bool RecordLockInfo() const override { + return true; + } + + void TryPreWait() { + DCHECK(!status_tablet_id_.empty()); + auto did_wait_or_status = wait_queue_->MaybeWaitOnLocks( + context_->transaction_id(), lock_batch_, status_tablet_id_, serial_no_, + std::bind(&WaitOnConflictResolver::WaitingDone, shared_from(this), _1)); + if (!did_wait_or_status.ok()) { + InvokeCallback(did_wait_or_status.status()); + } else if (!*did_wait_or_status) { + ConflictResolver::Resolve(); + } else { + VLOG(3) << "Wait-on-Conflict resolution entered wait queue in PreWaitOn stage"; + } + } Status OnConflictingTransactionsFound() override { - DCHECK_GT(remaining_transactions_, 0); + DCHECK_GT(conflict_data_->NumActiveTransactions(), 0); wait_for_iters_++; VTRACE(3, "Waiting on $0 transactions after $1 tries.", - remaining_transactions_, wait_for_iters_); - - std::vector blockers; - blockers.reserve(remaining_transactions_); - for (auto& txn : RemainingTransactions()) { - blockers.emplace_back(BlockingTransactionData { - .id = txn.id, - .status_tablet = "", - .subtransactions = txn.subtransactions, - }); - } - status_manager().FillStatusTablets(&blockers); - EraseIf([](const auto& blocker) { - if (blocker.status_tablet.empty()) { - LOG(WARNING) << "Cannot find status tablet for blocking transaction " << blocker.id << ". " - << "Ignoring this transaction, as it should be either committed or aborted."; - return true; - } - return false; - }, &blockers); + conflict_data_->NumActiveTransactions(), wait_for_iters_); return wait_queue_->WaitOn( - context_->transaction_id(), lock_batch_, std::move(blockers), - context_->transaction_id().IsNil() ? "" : VERIFY_RESULT(context_->GetStatusTablet(this)), + context_->transaction_id(), lock_batch_, ConsumeTransactionDataAndReset(), + status_tablet_id_, serial_no_, std::bind(&WaitOnConflictResolver::WaitingDone, shared_from(this), _1)); } @@ -651,14 +622,15 @@ class WaitOnConflictResolver : public ConflictResolver { // TODO(wait-queues): In case wait queue finds that a blocker was committed, and if that blocker // has still-live modification conflicts with this operation (i.e. not from rolled back subtxn), // we can avoid re-running conflict resolution here and just abort. - Reset(); Resolve(); } private: WaitQueue* const wait_queue_; LockBatch* const lock_batch_; + uint64_t serial_no_; uint32_t wait_for_iters_ = 0; + TabletId status_tablet_id_; }; using IntentTypesContainer = std::map; @@ -861,7 +833,7 @@ class ConflictResolverContextBase : public ConflictResolverContext { protected: Status CheckPriorityInternal( ConflictResolver* resolver, - boost::iterator_range transactions, + boost::iterator_range transactions, const TransactionId& our_transaction_id, uint64_t our_priority) { @@ -926,6 +898,7 @@ class TransactionConflictResolverContext : public ConflictResolverContextBase { virtual ~TransactionConflictResolverContext() {} Result GetStatusTablet(ConflictResolver* resolver) const override { + RETURN_NOT_OK(transaction_id_); // If this is the first operation for this transaction at this tablet, then GetStatusTablet // will return boost::none since the transaction has not been registered with the tablet's // transaction participant. However, the write_batch_ transaction metadata only includes the @@ -1032,29 +1005,29 @@ class TransactionConflictResolverContext : public ConflictResolverContextBase { } Status CheckPriority(ConflictResolver* resolver, - boost::iterator_range transactions) override { + boost::iterator_range transactions) override { return CheckPriorityInternal(resolver, transactions, metadata_.transaction_id, metadata_.priority); } - bool CheckConflictWithPending(const TransactionData& transaction_data) override { + bool CheckConflictWithPending(const TransactionConflictData& transaction_data) override { // We remove aborted subtransactions when processing the SubtxnSet stored // locally or returned by the status tablet. If this is now empty, then all potentially // conflicting intents have been aborted and there is no longer a conflict with this // transaction. - return transaction_data.subtransactions->empty(); + return transaction_data.conflict_info->subtransactions.empty(); } Result CheckConflictWithCommitted( - const TransactionData& transaction_data, HybridTime commit_time) override { + const TransactionConflictData& transaction_data, HybridTime commit_time) override { RSTATUS_DCHECK(commit_time.is_valid(), Corruption, "Invalid transaction commit time"); VLOG_WITH_PREFIX(4) << "Committed: " << transaction_data.id << ", commit_time: " << commit_time << ", read_time: " << read_time_ << ", transaction_data: " << transaction_data.ToString(); - for (const auto& subtxn_and_data : *transaction_data.subtransactions) { - auto has_non_lock_conflict = subtxn_and_data.second; + for (const auto& subtxn_and_data : transaction_data.conflict_info->subtransactions) { + auto has_non_lock_conflict = subtxn_and_data.second.has_non_lock_conflict; // If the intents to be written conflict with only "explicit row lock" intents of a committed // transaction, we can proceed now because a committed transaction implies that the locks are // released. In other words, only a committed transaction with some conflicting intent that @@ -1093,6 +1066,10 @@ class TransactionConflictResolverContext : public ConflictResolverContextBase { return *transaction_id_; } + bool IsSingleShardTransaction() const override { + return false; + } + std::string ToString() const override { return yb::ToString(transaction_id_); } @@ -1167,7 +1144,7 @@ class OperationConflictResolverContext : public ConflictResolverContextBase { } Status CheckPriority(ConflictResolver* resolver, - boost::iterator_range transactions) override { + boost::iterator_range transactions) override { return CheckPriorityInternal(resolver, transactions, TransactionId::Nil(), @@ -1182,16 +1159,20 @@ class OperationConflictResolverContext : public ConflictResolverContextBase { return TransactionId::Nil(); } + bool IsSingleShardTransaction() const override { + return true; + } + std::string ToString() const override { return "Operation Context"; } - bool CheckConflictWithPending(const TransactionData& transaction_data) override { + bool CheckConflictWithPending(const TransactionConflictData& transaction_data) override { return false; } Result CheckConflictWithCommitted( - const TransactionData& transaction_data, HybridTime commit_time) override { + const TransactionConflictData& transaction_data, HybridTime commit_time) override { if (commit_time != HybridTime::kMax) { MakeResolutionAtLeast(commit_time); return true; @@ -1221,22 +1202,20 @@ Status ResolveTransactionConflicts(const DocOperations& doc_ops, auto context = std::make_unique( doc_ops, write_batch, hybrid_time, read_time, conflicts_metric, conflict_management_policy); - auto request_scope = VERIFY_RESULT(RequestScope::Create(status_manager)); if (conflict_management_policy == WAIT_ON_CONFLICT) { if (!wait_queue) { return STATUS_FORMAT(NotSupported, "Wait queues are not enabled"); } DCHECK(lock_batch); auto resolver = std::make_shared( - doc_db, status_manager, std::move(request_scope), partial_range_key_intents, - std::move(context), std::move(callback), wait_queue, lock_batch); - resolver->Resolve(); + doc_db, status_manager, partial_range_key_intents, std::move(context), std::move(callback), + wait_queue, lock_batch); + resolver->Run(); } else { // SKIP_ON_CONFLICT is piggybacked on FailOnConflictResolver since it is almost the same // with just a few lines of extra handling. auto resolver = std::make_shared( - doc_db, status_manager, std::move(request_scope), partial_range_key_intents, - std::move(context), std::move(callback)); + doc_db, status_manager, partial_range_key_intents, std::move(context), std::move(callback)); resolver->Resolve(); } TRACE("resolver->Resolve done"); @@ -1258,22 +1237,20 @@ Status ResolveOperationConflicts(const DocOperations& doc_ops, auto context = std::make_unique( &doc_ops, resolution_ht, conflicts_metric, conflict_management_policy); - auto request_scope = VERIFY_RESULT(RequestScope::Create(status_manager)); if (conflict_management_policy == WAIT_ON_CONFLICT) { if (!wait_queue) { return STATUS_FORMAT(NotSupported, "Wait queues are not enabled"); } auto resolver = std::make_shared( - doc_db, status_manager, std::move(request_scope), partial_range_key_intents, - std::move(context), std::move(callback), wait_queue, lock_batch); - resolver->Resolve(); + doc_db, status_manager, partial_range_key_intents, std::move(context), std::move(callback), + wait_queue, lock_batch); + resolver->Run(); } else { // SKIP_ON_CONFLICT is piggybacked on FailOnConflictResolver since it is almost the same // with just a few lines of extra handling. auto resolver = std::make_shared( - doc_db, status_manager, std::move(request_scope), partial_range_key_intents, - std::move(context), std::move(callback)); + doc_db, status_manager, partial_range_key_intents, std::move(context), std::move(callback)); resolver->Resolve(); } TRACE("resolver->Resolve done"); diff --git a/src/yb/docdb/docdb.cc b/src/yb/docdb/docdb.cc index 446916e14d73..7c01e9fb5020 100644 --- a/src/yb/docdb/docdb.cc +++ b/src/yb/docdb/docdb.cc @@ -92,21 +92,7 @@ Status ApplyIntent(RefCntPrefix key, LockBatchEntries *keys_locked) { // Have to strip kGroupEnd from end of key, because when only hash key is specified, we will // get two kGroupEnd at end of strong intent. - size_t size = key.size(); - if (size > 0) { - if (key.data()[0] == KeyEntryTypeAsChar::kGroupEnd) { - if (size != 1) { - return STATUS_FORMAT(Corruption, "Key starting with group end: $0", - key.as_slice().ToDebugHexString()); - } - size = 0; - } else { - while (key.data()[size - 1] == KeyEntryTypeAsChar::kGroupEnd) { - --size; - } - } - } - key.Resize(size); + RETURN_NOT_OK(RemoveGroupEndSuffix(&key)); keys_locked->push_back({key, intent_types}); return Status::OK(); } diff --git a/src/yb/docdb/docdb_test_util.cc b/src/yb/docdb/docdb_test_util.cc index 926404de1f2d..91955e031a84 100644 --- a/src/yb/docdb/docdb_test_util.cc +++ b/src/yb/docdb/docdb_test_util.cc @@ -106,10 +106,6 @@ class NonTransactionalStatusProvider: public TransactionStatusManager { Fail(); } - void FillStatusTablets(std::vector* inout) override { - Fail(); - } - boost::optional FindStatusTablet(const TransactionId& id) override { return boost::none; } diff --git a/src/yb/docdb/intent.cc b/src/yb/docdb/intent.cc index b748a2bd94c3..721aa59df2a0 100644 --- a/src/yb/docdb/intent.cc +++ b/src/yb/docdb/intent.cc @@ -30,6 +30,25 @@ namespace yb { namespace docdb { +Status RemoveGroupEndSuffix(RefCntPrefix* key) { + size_t size = DCHECK_NOTNULL(key)->size(); + if (size > 0) { + if (key->data()[0] == KeyEntryTypeAsChar::kGroupEnd) { + if (size != 1) { + return STATUS_FORMAT(Corruption, "Key starting with group end: $0", + key->as_slice().ToDebugHexString()); + } + size = 0; + } else { + while (key->data()[size - 1] == KeyEntryTypeAsChar::kGroupEnd) { + --size; + } + } + } + key->Resize(size); + return Status::OK(); +} + Result DecodeIntentKey(const Slice &encoded_intent_key) { DecodedIntentKey result; auto& intent_prefix = result.intent_prefix; diff --git a/src/yb/docdb/intent.h b/src/yb/docdb/intent.h index 7bdc37711702..09583de7278f 100644 --- a/src/yb/docdb/intent.h +++ b/src/yb/docdb/intent.h @@ -17,10 +17,17 @@ #include "yb/common/transaction.h" #include "yb/docdb/docdb_encoding_fwd.h" +#include "yb/util/ref_cnt_buffer.h" namespace yb { namespace docdb { +// We may write intents with empty groups to intents_db, but when interacting with SharedLockManager +// or WaitQueue, we expect no kGroupEnd markers in keys. This method normalizes the passed in key to +// the format expected by conflict resolution. Returns an error if the provided key begins with a +// kGroupEnd marker. +Status RemoveGroupEndSuffix(RefCntPrefix* key); + // "Intent types" are used for single-tablet operations and cross-shard transactions. For example, // multiple write-only operations don't need to conflict. However, if one operation is a // read-modify-write snapshot isolation operation, then a write-only operation cannot proceed in diff --git a/src/yb/docdb/local_waiting_txn_registry.cc b/src/yb/docdb/local_waiting_txn_registry.cc index 5c6d9ffca266..32f95c1b7fd2 100644 --- a/src/yb/docdb/local_waiting_txn_registry.cc +++ b/src/yb/docdb/local_waiting_txn_registry.cc @@ -44,7 +44,7 @@ using namespace std::literals; DECLARE_bool(enable_deadlock_detection); namespace yb { -namespace tablet { +namespace docdb { class StatusTabletData; using StatusTabletDataPtr = std::shared_ptr; @@ -54,7 +54,7 @@ using StatusTabletDataPtr = std::shared_ptr; // StatusTabletData instance being tracked by the LocalWaitingTxnRegistry. struct WaitingTransactionData { TransactionId id; - std::vector blockers; + std::shared_ptr blockers; StatusTabletDataPtr status_tablet_data; HybridTime wait_start_time; rpc::Rpcs::Handle rpc_handle; @@ -65,7 +65,7 @@ void AttachWaitingTransaction( auto* txn = req->add_waiting_transactions(); txn->set_transaction_id(data.id.data(), data.id.size()); txn->set_wait_start_time(data.wait_start_time.ToUint64()); - for (const auto& blocker : data.blockers) { + for (const auto& blocker : data.blockers->RemainingTransactions()) { auto* blocking_txn = txn->add_blocking_transaction(); blocking_txn->set_transaction_id(blocker.id.data(), blocker.id.size()); blocking_txn->set_status_tablet_id(blocker.status_tablet); @@ -151,16 +151,16 @@ class LocalWaitingTxnRegistry::Impl { // A wrapper for WaitingTransactionData which determines the lifetime of the // LocalWaitingTxnRegistry's metadata associated with this waiter. When this wrapper is // destructed, the LocalWaitingTxnRegistry will cease reporting on this waiter. - class WaitingTransactionDataWrapper : public docdb::ScopedWaitingTxnRegistration { + class WaitingTransactionDataWrapper : public ScopedWaitingTxnRegistration { public: explicit WaitingTransactionDataWrapper(LocalWaitingTxnRegistry::Impl* registry) : registry_(registry) {} Status Register( const TransactionId& waiting, - std::vector&& blocking, + std::shared_ptr blockers, const TabletId& status_tablet) override { - return registry_->RegisterWaitingFor(waiting, std::move(blocking), status_tablet, this); + return registry_->RegisterWaitingFor(waiting, std::move(blockers), status_tablet, this); } void SetData(std::shared_ptr&& blocked_data) { @@ -258,14 +258,14 @@ class LocalWaitingTxnRegistry::Impl { } Status RegisterWaitingFor( - const TransactionId& waiting, std::vector&& blocking, + const TransactionId& waiting, std::shared_ptr blockers, const TabletId& status_tablet_id, WaitingTransactionDataWrapper* wrapper) { DCHECK(!status_tablet_id.empty()); auto shared_tablet_data = VERIFY_RESULT(GetOrAdd(status_tablet_id)); auto blocked_data = std::make_shared(WaitingTransactionData { .id = waiting, - .blockers = std::move(blocking), + .blockers = std::move(blockers), .status_tablet_data = shared_tablet_data, .wait_start_time = clock_->Now(), .rpc_handle = rpcs_.InvalidHandle(), @@ -331,7 +331,7 @@ LocalWaitingTxnRegistry::LocalWaitingTxnRegistry( LocalWaitingTxnRegistry::~LocalWaitingTxnRegistry() {} -std::unique_ptr LocalWaitingTxnRegistry::Create() { +std::unique_ptr LocalWaitingTxnRegistry::Create() { return impl_->Create(); } @@ -347,5 +347,5 @@ void LocalWaitingTxnRegistry::CompleteShutdown() { return impl_->CompleteShutdown(); } -} // namespace tablet +} // namespace docdb } // namespace yb diff --git a/src/yb/docdb/local_waiting_txn_registry.h b/src/yb/docdb/local_waiting_txn_registry.h index a0e003d8659b..3b03d8715e68 100644 --- a/src/yb/docdb/local_waiting_txn_registry.h +++ b/src/yb/docdb/local_waiting_txn_registry.h @@ -18,12 +18,12 @@ #include "yb/server/server_fwd.h" namespace yb { -namespace tablet { +namespace docdb { // This class is responsible for aggregating all wait-for relationships across WaitQueue instances // on a tserver and reporting these relationships to each waiting transaction's respective // transaction coordinator. -class LocalWaitingTxnRegistry : public docdb::WaitingTxnRegistry { +class LocalWaitingTxnRegistry : public WaitingTxnRegistry { public: explicit LocalWaitingTxnRegistry( const std::shared_future& client_future, const server::ClockPtr& clock); @@ -37,7 +37,7 @@ class LocalWaitingTxnRegistry : public docdb::WaitingTxnRegistry { std::unique_ptr Create() override; Status RegisterWaitingFor( - const TransactionId& waiting, std::vector&& blocking, + const TransactionId& waiting, std::shared_ptr blockers, const TabletId& status_tablet_id, docdb::ScopedWaitingTxnRegistration* wrapper); // Triggers a report of all wait-for relationships tracked by this instance to each waiting @@ -53,5 +53,5 @@ class LocalWaitingTxnRegistry : public docdb::WaitingTxnRegistry { std::unique_ptr impl_; }; -} // namespace tablet +} // namespace docdb } // namespace yb diff --git a/src/yb/docdb/lock_batch.h b/src/yb/docdb/lock_batch.h index cd857e6d705f..8d8c97307139 100644 --- a/src/yb/docdb/lock_batch.h +++ b/src/yb/docdb/lock_batch.h @@ -82,6 +82,8 @@ class LockBatch { // re-lock the keys. std::optional Unlock(); + const LockBatchEntries& Get() const { return data_.key_to_type; } + private: void MoveFrom(LockBatch* other); diff --git a/src/yb/docdb/wait_queue.cc b/src/yb/docdb/wait_queue.cc index f2af78ac97da..1d871a731481 100644 --- a/src/yb/docdb/wait_queue.cc +++ b/src/yb/docdb/wait_queue.cc @@ -13,9 +13,12 @@ #include "yb/docdb/wait_queue.h" +#include #include #include #include +#include +#include #include @@ -24,6 +27,8 @@ #include "yb/common/transaction.h" #include "yb/common/transaction.pb.h" #include "yb/common/wire_protocol.h" +#include "yb/docdb/intent.h" +#include "yb/docdb/shared_lock_manager.h" #include "yb/gutil/stl_util.h" #include "yb/gutil/thread_annotations.h" #include "yb/rpc/rpc.h" @@ -34,6 +39,7 @@ #include "yb/tablet/transaction_participant_context.h" #include "yb/tserver/tserver_service.pb.h" #include "yb/util/backoff_waiter.h" +#include "yb/util/debug-util.h" #include "yb/util/flags.h" #include "yb/util/locks.h" #include "yb/util/logging.h" @@ -108,8 +114,12 @@ auto GetMaxSingleShardWaitDuration() { YB_DEFINE_ENUM(ResolutionStatus, (kPending)(kCommitted)(kAborted)); class BlockerData; -using BlockerDataAndSubtxnInfo = std::pair, - std::shared_ptr>; +using BlockerDataAndConflictInfo = std::pair, + TransactionConflictInfoPtr>; + +using BlockerDataConflictMap = std::unordered_map; Result UnwrapResult(const Result& res) { if (!res.ok()) { @@ -152,33 +162,41 @@ inline auto GetMicros(CoarseMonoClock::Duration duration) { // in blocker_status_ are presumed to no lonber be of concern to any pending waiting transactions // and are discarded. struct WaiterData : public std::enable_shared_from_this { - WaiterData(const TransactionId id_, LockBatch* const locks_, const TabletId& status_tablet_, - const std::vector blockers_, + WaiterData(const TransactionId id_, LockBatch* const locks_, uint64_t serial_no_, + const TabletId& status_tablet_, + const std::vector blockers_, const WaitDoneCallback callback_, - std::unique_ptr waiter_registration_, - rpc::Rpcs* rpcs, + std::unique_ptr waiter_registration_, rpc::Rpcs* rpcs, scoped_refptr* finished_waiting_latency) : id(id_), locks(locks_), + serial_no(serial_no_), status_tablet(status_tablet_), blockers(std::move(blockers_)), callback(std::move(callback_)), waiter_registration(std::move(waiter_registration_)), finished_waiting_latency_(*finished_waiting_latency), unlocked_(locks->Unlock()), - rpcs_(*rpcs) {} + rpcs_(*rpcs) { + VLOG_WITH_PREFIX(4) << "Constructed waiter"; + } + + ~WaiterData() { + VLOG_WITH_PREFIX(4) << "Destructed waiter"; + } const TransactionId id; LockBatch* const locks; + const uint64_t serial_no; const TabletId status_tablet; - const std::vector blockers; + const std::vector blockers; const WaitDoneCallback callback; std::unique_ptr waiter_registration; const CoarseTimePoint created_at = CoarseMonoClock::Now(); void InvokeCallback(const Status& status) { VLOG_WITH_PREFIX(4) << "Invoking waiter callback " << status; - UniqueLock l(mutex_); + UniqueLock l(mutex_); if (!unlocked_) { LOG_WITH_PREFIX(INFO) << "Skipping InvokeCallback for waiter whose callback was already invoked. This should " @@ -204,7 +222,7 @@ struct WaiterData : public std::enable_shared_from_this { void(const Status& status, const tserver::GetTransactionStatusResponsePB& resp)>; void TriggerStatusRequest(HybridTime now, client::YBClient* client, StatusCb cb) { - UniqueLock l(mutex_); + UniqueLock l(mutex_); if (!unlocked_) { VLOG_WITH_PREFIX(1) << "Skipping GetTransactionStatus RPC for waiter whose callback was already invoked."; @@ -230,7 +248,7 @@ struct WaiterData : public std::enable_shared_from_this { &req, [instance = shared_from(this), cb](const auto& status, const auto& resp) { { - UniqueLock l(instance->mutex_); + UniqueLock l(instance->mutex_); if (instance->handle_ != instance->rpcs_.InvalidHandle()) { instance->rpcs_.Unregister(&instance->handle_); } @@ -241,7 +259,7 @@ struct WaiterData : public std::enable_shared_from_this { } void ShutdownStatusRequest() { - UniqueLock l(mutex_); + UniqueLock l(mutex_); if (handle_ != rpcs_.InvalidHandle()) { (**handle_).Abort(); rpcs_.Unregister(&handle_); @@ -268,6 +286,13 @@ using WaiterDataPtr = std::shared_ptr; // active and has exited the wait queue. class BlockerData { public: + BlockerData(const TransactionId& id, const TabletId& status_tablet) + : id_(id), status_tablet_(status_tablet) {} + + const TransactionId& id() { return id_; } + + const TabletId& status_tablet() { return status_tablet_; } + std::vector Signal(Result&& txn_status_response) { VLOG(4) << "Signaling waiters " << (txn_status_response.ok() ? @@ -277,7 +302,7 @@ class BlockerData { bool is_txn_pending = txn_status.ok() && *txn_status == ResolutionStatus::kPending; bool should_signal = !is_txn_pending; - UniqueLock l(mutex_); + UniqueLock l(mutex_); std::vector waiters_to_signal; txn_status_ = txn_status; @@ -291,7 +316,7 @@ class BlockerData { } if (should_signal) { - waiters_to_signal.reserve(waiters_.size()); + waiters_to_signal.reserve(waiters_.size() + post_resolve_waiters_.size()); EraseIf([&waiters_to_signal](const auto& weak_waiter) { if (auto waiter = weak_waiter.lock()) { waiters_to_signal.push_back(waiter); @@ -299,32 +324,69 @@ class BlockerData { } return true; }, &waiters_); + EraseIf([&waiters_to_signal](const auto& weak_waiter) { + if (auto waiter = weak_waiter.lock()) { + waiters_to_signal.push_back(waiter); + return false; + } + return true; + }, &post_resolve_waiters_); } return waiters_to_signal; } void AddWaiter(const WaiterDataPtr& waiter_data) { - UniqueLock blocker_lock(mutex_); - waiters_.push_back(waiter_data); + UniqueLock blocker_lock(mutex_); + if (IsPendingUnlocked()) { + waiters_.push_back(waiter_data); + } else { + post_resolve_waiters_.push_back(waiter_data); + } } Result IsResolved() { - SharedLock blocker_lock(mutex_); + SharedLock blocker_lock(mutex_); return VERIFY_RESULT(Copy(txn_status_)) != ResolutionStatus::kPending; } + bool IsPending() { + SharedLock blocker_lock(mutex_); + return IsPendingUnlocked(); + } + + bool IsPendingUnlocked() REQUIRES_SHARED(mutex_) { + return txn_status_.ok() && *txn_status_ == ResolutionStatus::kPending; + } + auto CleanAndGetSize() { - UniqueLock l(mutex_); + UniqueLock l(mutex_); EraseIf([](const auto& weak_waiter) { return weak_waiter.expired(); }, &waiters_); - return waiters_.size(); + EraseIf([](const auto& weak_waiter) { + return weak_waiter.expired(); + }, &post_resolve_waiters_); + if (waiters_.size() == 0 && !IsPendingUnlocked()) { + // If we hit this branch, then we've processed all waiters which arrived before this blocker + // was first resolved. Therefore, new incoming requests need not worry about starving the + // original waiters any more, and can ignore this blocker. This may cause marginal amounts of + // starvation for any waiters stuck in post_resolve_waiters_, but such starvation should be + // rare and unlikely to repeat. + is_active_ = false; + } + return waiters_.size() + post_resolve_waiters_.size(); + } + + // Returns true if an incoming waiter should consider this blocker for purposes of fairness. + auto IsActive() { + SharedLock l(mutex_); + return is_active_; } auto DEBUG_GetWaiterIds() const { std::vector waiters; - SharedLock l(mutex_); + SharedLock l(mutex_); for (auto it = waiters_.begin(); it != waiters_.end(); ++it) { if (auto waiter = it->lock()) { waiters.push_back(waiter->id.ToString()); @@ -333,8 +395,9 @@ class BlockerData { return boost::algorithm::join(waiters, ","); } - bool HasLiveSubtransaction(const SubtxnHasNonLockConflict& subtransaction_info) { - SharedLock blocker_lock(mutex_); + bool HasLiveSubtransaction( + const decltype(TransactionConflictInfo::subtransactions)& subtransaction_info) { + SharedLock blocker_lock(mutex_); for (const auto& [id, _] : subtransaction_info) { if (!aborted_subtransactions_.Test(id)) { return true; @@ -343,11 +406,125 @@ class BlockerData { return false; } + void AddIntents(const TransactionConflictInfo& conflict_info) { + UniqueLock l(mutex_); + for (const auto& [subtxn_id, data] : conflict_info.subtransactions) { + auto& intents_by_key = subtxn_intents_by_key_[subtxn_id]; + for (const auto& lock : data.locks) { + auto [iter, did_insert] = intents_by_key.emplace(lock.doc_path, lock.intent_types); + if (!did_insert) { + // Combine new intent types with existing intent types. + iter->second |= lock.intent_types; + } + } + } + } + + // Find any of this blockers known conflicts with provided doc_path and intent_type_set. If + // conflict_info is not null, populate conflict information. Otherwise, early return true to + // indicate a conflict was found. Return false if no conflict is found. + bool PopulateConflictInfo( + const RefCntPrefix& doc_path, const IntentTypeSet& intent_type_set, + TransactionConflictInfo* conflict_info) { + bool did_find_conflicts = false; + SharedLock l(mutex_); + for (const auto& [subtxn_id, intents_by_key] : subtxn_intents_by_key_) { + auto it = intents_by_key.find(doc_path); + if (it != intents_by_key.end() && IntentTypeSetsConflict(intent_type_set, it->second)) { + if (!conflict_info) { + return true; + } + did_find_conflicts = true; + conflict_info->subtransactions[subtxn_id].locks.emplace_back( + LockInfo {doc_path, intent_type_set}); + } + } + return did_find_conflicts; + } + + void DumpHtmlTableRow(std::ostream& out) { + SharedLock l(mutex_); + out << "" + << "|" << id_ << "" + << "|" << txn_status_ << "" + << "" << std::endl; + } + private: + const TransactionId id_; + const TabletId status_tablet_; mutable rw_spinlock mutex_; Result txn_status_ GUARDED_BY(mutex_) = ResolutionStatus::kPending; SubtxnSet aborted_subtransactions_ GUARDED_BY(mutex_); std::vector> waiters_ GUARDED_BY(mutex_); + std::vector> post_resolve_waiters_ GUARDED_BY(mutex_); + bool is_active_ GUARDED_BY(mutex_) = true; + + using IntentsByKey = std::unordered_map< + RefCntPrefix, + IntentTypeSet, + RefCntPrefixHash>; + std::unordered_map subtxn_intents_by_key_ GUARDED_BY(mutex_); +}; + +struct WaiterSerialComparator { + bool operator()(const WaiterDataPtr& w1, const WaiterDataPtr& w2) const { + return w1->serial_no > w2->serial_no; + } +}; + +// Resumes waiters async, in serial, and in the order of the waiter's serial_no, running the lowest +// serial number first in a best effort manner. +class ResumedWaiterRunner { + public: + explicit ResumedWaiterRunner(std::unique_ptr thread_pool_token) + : thread_pool_token_(std::move(thread_pool_token)) {} + + void Submit(const WaiterDataPtr& waiter, const Status& status) { + { + UniqueLock l(mutex_); + AddWaiter(waiter, status); + } + TriggerPoll(); + } + + void Shutdown() { + thread_pool_token_->Shutdown(); + } + + private: + void TriggerPoll() { + WARN_NOT_OK(thread_pool_token_->SubmitFunc([this]() { + for (;;) { + WaiterDataPtr to_invoke; + { + UniqueLock l(this->mutex_); + if (pq_.empty()) { + return; + } + to_invoke = pq_.top(); + pq_.pop(); + } + to_invoke->InvokeCallback(Status::OK()); + } + }), "Failed to trigger poll of ResumedWaiterRunner in wait queue"); + } + + void AddWaiter(const WaiterDataPtr& waiter, const Status& status) REQUIRES(mutex_) { + if (status.ok()) { + pq_.push(waiter); + } else { + // If error status, resume waiter right away, no need to respect serial_no + WARN_NOT_OK(thread_pool_token_->SubmitFunc([waiter, status]() { + waiter->InvokeCallback(status); + }), "Failed to submit waiter resumption"); + } + } + + mutable rw_spinlock mutex_; + std::priority_queue< + WaiterDataPtr, std::vector, WaiterSerialComparator> pq_ GUARDED_BY(mutex_); + std::unique_ptr thread_pool_token_; }; const Status kShuttingDownError = STATUS( @@ -368,7 +545,7 @@ class WaitQueue::Impl { std::unique_ptr thread_pool_token) : txn_status_manager_(txn_status_manager), permanent_uuid_(permanent_uuid), waiting_txn_registry_(waiting_txn_registry), client_future_(client_future), clock_(clock), - thread_pool_token_(std::move(thread_pool_token)), + waiter_runner_(std::move(thread_pool_token)), pending_time_waiting_(METRIC_wait_queue_pending_time_waiting.Instantiate(metrics)), finished_waiting_latency_(METRIC_wait_queue_finished_waiting_latency.Instantiate(metrics)), blockers_per_waiter_(METRIC_wait_queue_blockers_per_waiter.Instantiate(metrics)), @@ -385,20 +562,132 @@ class WaitQueue::Impl { } } + // Find blockers in this wait queue which conflict with the provided locks. If the blockers + // argument is not null, populate conflict information any time a conflict is found. Otherwise, + // return true as soon as any conflict is found. Return false if there are no conflicts. + bool PopulateBlockersUnlocked( + const TransactionId& waiter_txn_id, LockBatch* locks, + BlockerDataConflictMap* blockers = nullptr) REQUIRES_SHARED(mutex_) { + for (const auto& entry : locks->Get()) { + auto it = blockers_by_key_.find(entry.key); + if (it == blockers_by_key_.end()) { + VLOG_WITH_PREFIX(5) << "No blockers found for key " << entry.key.ShortDebugString(); + continue; + } + VLOG_WITH_PREFIX(4) << "Found blockers for key " << entry.key.ShortDebugString(); + + for (const auto& blocker_id : it->second) { + if (blocker_id == waiter_txn_id) { + continue; + } + auto blocker_it = blocker_status_.find(blocker_id); + if (blocker_it == blocker_status_.end()) { + LOG(DFATAL) << "Unexpected blocker found in blockers_by_key_ but not blocker_status_"; + continue; + } + + auto blocker = blocker_it->second.lock(); + if (!blocker || !blocker->IsActive()) { + continue; + } + DCHECK_NE(blocker->id(), waiter_txn_id); + + TransactionConflictInfoPtr conflict_info; + bool should_emplace_conflict_info = false; + if (blockers) { + auto it = blockers->find(blocker->id()); + if (it == blockers->end()) { + // Need to use a new conflict_info instance and emplace it on the blockers map in case + // we do find a conflict. We should avoid creating entries in blockers if there is not + // a genuine conflict. + conflict_info = std::make_shared(); + should_emplace_conflict_info = true; + } else { + // If there is already an entry for this blocker, populate the same conflict info. + conflict_info = it->second.second; + } + } + + if (blocker->PopulateConflictInfo(entry.key, entry.intent_types, conflict_info.get())) { + VLOG_WITH_PREFIX(3) << "Found conflict for " << waiter_txn_id + << " on blocker " << blocker->id(); + if (!blockers) { + return true; + } + if (should_emplace_conflict_info) { + VLOG_WITH_PREFIX(5) << "Emplacing conflict info for " << waiter_txn_id + << " on blocker " << blocker->id(); + auto emplace_it = DCHECK_NOTNULL(blockers)->emplace( + blocker->id(), std::make_pair(blocker, conflict_info)); + DCHECK(emplace_it.first->second.first == blocker); + } + } + + } + } + return blockers && !blockers->empty(); + } + + Result MaybeWaitOnLocks( + const TransactionId& waiter_txn_id, LockBatch* locks, const TabletId& status_tablet_id, + uint64_t serial_no, WaitDoneCallback callback) { + VLOG_WITH_PREFIX_AND_FUNC(4) << "waiter_txn_id=" << waiter_txn_id + << " status_tablet_id=" << status_tablet_id; + bool found_blockers = false; + { + // First check if there are /any/ conflicts using a shared lock. + SharedLock wq_lock(mutex_); + found_blockers = PopulateBlockersUnlocked(waiter_txn_id, locks); + } + + if (found_blockers) { + // If there were conflicts, acquire a unique lock, get all conflicts, and insert a new waiter + // into the wait queue. + BlockerDataConflictMap blockers_map; + UniqueLock wq_lock(mutex_); + + if (PopulateBlockersUnlocked(waiter_txn_id, locks, &blockers_map)) { + VLOG_WITH_PREFIX_AND_FUNC(1) << "Found " << blockers_map.size() + << " blockers for " << waiter_txn_id; + + std::vector blocker_datas; + auto blockers = std::make_shared(blockers_map.size()); + blocker_datas.reserve(blockers_map.size()); + for (const auto& [id, blocker_data] : blockers_map) { + blocker_datas.emplace_back(blocker_data); + blockers->AddTransaction(id, blocker_data.second, blocker_data.first->status_tablet()); + } + + RETURN_NOT_OK(SetupWaiterUnlocked( + waiter_txn_id, locks, status_tablet_id, serial_no, std::move(callback), + std::move(blocker_datas), std::move(blockers))); + return true; + } else { + // It's possible that between checking above with a shared lock and checking again with a + // unique lock that conflicting blockers have become inactive. + VLOG_WITH_PREFIX_AND_FUNC(1) + << "Pre-wait second check found no blockers for " << waiter_txn_id; + } + } else { + VLOG_WITH_PREFIX_AND_FUNC(4) << "Pre-wait found no blockers for " << waiter_txn_id; + } + + return false; + } + Status WaitOn( const TransactionId& waiter_txn_id, LockBatch* locks, - std::vector&& blockers, const TabletId& status_tablet_id, - WaitDoneCallback callback) { + std::shared_ptr blockers, const TabletId& status_tablet_id, + uint64_t serial_no, WaitDoneCallback callback) { VLOG_WITH_PREFIX_AND_FUNC(4) << "waiter_txn_id=" << waiter_txn_id - << " blockers=" << ToString(blockers) + << " blockers=" << *blockers << " status_tablet_id=" << status_tablet_id; // TODO(wait-queues): We can detect tablet-local deadlocks here. // See https://github.com/yugabyte/yugabyte-db/issues/13586 - std::vector blocker_datas; - WaiterDataPtr waiter_data = nullptr; + std::vector blocker_datas; { - UniqueLock wq_lock(mutex_); + UniqueLock wq_lock(mutex_); if (shutting_down_) { return kShuttingDownError; } @@ -411,8 +700,8 @@ class WaitQueue::Impl { STATUS(IllegalState, "Unexpected duplicate waiter in wait queue - try again.")); } - for (const auto& blocker : blockers) { - auto blocker_data = std::make_shared(); + for (const auto& blocker : blockers->RemainingTransactions()) { + auto blocker_data = std::make_shared(blocker.id, blocker.status_tablet); auto [iter, did_insert] = blocker_status_.emplace(blocker.id, blocker_data); if (!did_insert) { @@ -431,43 +720,60 @@ class WaitQueue::Impl { } else { VLOG_WITH_PREFIX_AND_FUNC(4) << "Created blocker " << blocker.id; } - blocker_datas.emplace_back(blocker_data, blocker.subtransactions); - } - // TODO(wait-queues): similar to pg, we can wait 1s or so before beginning deadlock detection. - // See https://github.com/yugabyte/yugabyte-db/issues/13576 - auto scoped_reporter = waiting_txn_registry_->Create(); - if (!waiter_txn_id.IsNil()) { - // If waiter_txn_id is Nil, then we're processing a single-shard transaction. We do not have - // to report single shard transactions to transaction coordinators because they can't - // possibly be involved in a deadlock. This is true because no transactions can wait on - // single shard transactions, so they only have out edges in the wait-for graph and cannot - // be a part of a cycle. - DCHECK(!status_tablet_id.empty()); - RETURN_NOT_OK(scoped_reporter->Register( - waiter_txn_id, std::move(blockers), status_tablet_id)); - DCHECK_GE(scoped_reporter->GetDataUseCount(), 1); - } + blocker_data->AddIntents(*DCHECK_NOTNULL(blocker.conflict_info)); + blocker_datas.emplace_back(blocker_data, blocker.conflict_info); - waiter_data = std::make_shared( - waiter_txn_id, locks, status_tablet_id, std::move(blocker_datas), std::move(callback), - std::move(scoped_reporter), &rpcs_, &finished_waiting_latency_); - if (waiter_data->IsSingleShard()) { - DCHECK(single_shard_waiters_.size() == 0 || - waiter_data->created_at >= single_shard_waiters_.front()->created_at); - single_shard_waiters_.push_front(waiter_data); - } else { - waiter_status_[waiter_txn_id] = waiter_data; + for (const auto& [_, subtxn_data] : blocker.conflict_info->subtransactions) { + for (const auto& lock : subtxn_data.locks) { + blockers_by_key_[lock.doc_path].insert(blocker.id); + } + } } - // We must add waiters to blockers while holding the wait queue mutex. Otherwise, we may - // end up removing blockers from blocker_status_ during Poll() after we've already added them - // to waiter_data. - for (auto [blocker, _] : waiter_data->blockers) { - blocker->AddWaiter(waiter_data); - } + return SetupWaiterUnlocked( + waiter_txn_id, locks, status_tablet_id, serial_no, std::move(callback), + std::move(blocker_datas), std::move(blockers)); } + } + Status SetupWaiterUnlocked( + const TransactionId& waiter_txn_id, LockBatch* locks, const TabletId& status_tablet_id, + uint64_t serial_no, WaitDoneCallback callback, + std::vector&& blocker_datas, + std::shared_ptr blockers) REQUIRES(mutex_) { + // TODO(wait-queues): similar to pg, we can wait 1s or so before beginning deadlock detection. + // See https://github.com/yugabyte/yugabyte-db/issues/13576 + auto scoped_reporter = waiting_txn_registry_->Create(); + if (!waiter_txn_id.IsNil()) { + // If waiter_txn_id is Nil, then we're processing a single-shard transaction. We do not have + // to report single shard transactions to transaction coordinators because they can't + // possibly be involved in a deadlock. This is true because no transactions can wait on + // single shard transactions, so they only have out edges in the wait-for graph and cannot + // be a part of a cycle. + DCHECK(!status_tablet_id.empty()); + RETURN_NOT_OK(scoped_reporter->Register( + waiter_txn_id, std::move(blockers), status_tablet_id)); + DCHECK_GE(scoped_reporter->GetDataUseCount(), 1); + } + + auto waiter_data = std::make_shared( + waiter_txn_id, locks, serial_no, status_tablet_id, std::move(blocker_datas), + std::move(callback), std::move(scoped_reporter), &rpcs_, &finished_waiting_latency_); + if (waiter_data->IsSingleShard()) { + DCHECK(single_shard_waiters_.size() == 0 || + waiter_data->created_at >= single_shard_waiters_.front()->created_at); + single_shard_waiters_.push_front(waiter_data); + } else { + waiter_status_[waiter_txn_id] = waiter_data; + } + + // We must add waiters to blockers while holding the wait queue mutex. Otherwise, we may + // end up removing blockers from blocker_status_ during Poll() after we've already added them + // to waiter_data. + for (auto [blocker, _] : waiter_data->blockers) { + blocker->AddWaiter(waiter_data); + } return Status::OK(); } @@ -483,7 +789,7 @@ class WaitQueue::Impl { std::vector stale_single_shard_waiters; { - UniqueLock l(mutex_); + UniqueLock l(mutex_); if (shutting_down_) { return; } @@ -511,6 +817,24 @@ class WaitQueue::Impl { } it = blocker_status_.erase(it); } + for (auto it = blockers_by_key_.begin(); it != blockers_by_key_.end();) { + auto& key_blockers = it->second; + auto has_live = false; + for (auto txn_id_it = key_blockers.begin(); txn_id_it != key_blockers.end();) { + auto blocker_it = blocker_status_.find(*txn_id_it); + if (blocker_it == blocker_status_.end()) { + txn_id_it = key_blockers.erase(txn_id_it); + } else { + has_live = true; + txn_id_it++; + } + } + if (has_live) { + it++; + } else { + it = blockers_by_key_.erase(it); + } + } EraseIf([&stale_single_shard_waiters](const auto& waiter) { if (waiter->created_at + GetMaxSingleShardWaitDuration() < CoarseMonoClock::Now()) { stale_single_shard_waiters.push_back(waiter); @@ -573,7 +897,7 @@ class WaitQueue::Impl { decltype(single_shard_waiters_) single_shard_waiters_copy; { - UniqueLock l(mutex_); + UniqueLock l(mutex_); if (shutting_down_) { return false; } @@ -581,9 +905,10 @@ class WaitQueue::Impl { waiter_status_copy.swap(waiter_status_); single_shard_waiters_copy.swap(single_shard_waiters_); blocker_status_.clear(); + blockers_by_key_.clear(); } - thread_pool_token_->Shutdown(); + waiter_runner_.Shutdown(); for (const auto& [_, waiter_data] : waiter_status_copy) { waiter_data->ShutdownStatusRequest(); @@ -601,7 +926,7 @@ class WaitQueue::Impl { VLOG_WITH_PREFIX(1) << "Attempted to shutdown wait queue that is already shutdown"; return; } - SharedLock l(mutex_); + SharedLock l(mutex_); rpcs_.Shutdown(); LOG_IF(DFATAL, !shutting_down_) << "Called CompleteShutdown() while not in shutting_down_ state."; @@ -611,12 +936,74 @@ class WaitQueue::Impl { << "Called CompleteShutdown without empty waiter_status_"; } + void DumpStatusHtml(std::ostream& out) { + SharedLock l(mutex_); + if (shutting_down_) { + out << "Shutting down..."; + return; + } + + out << "

Txn Waiters:

" << std::endl; + + out << "" << std::endl; + out << "" << std::endl; + for (const auto& [txn_id, data] : waiter_status_) { + for (const auto& blocker : data->blockers) { + out << "" + << "" + << "" + << "" << std::endl; + } + } + out << "
WaiterIdBlockerId
|" << txn_id << "|" << blocker.first->id() << "
" << std::endl; + + out << "

Single Shard Waiters:

" << std::endl; + + out << "" << std::endl; + out << "" << std::endl; + auto idx = 0; + for (const auto& data : single_shard_waiters_) { + for (const auto& blocker : data->blockers) { + out << "" + << "" + << "" + << "" << std::endl; + } + ++idx; + } + out << "
IndexBlockerId
|" << idx << "|" << blocker.first->id() << "
" << std::endl; + + out << "

Blockers:

" << std::endl; + out << "" << std::endl; + out << "" << std::endl; + for (const auto& [txn_id, weak_data] : blocker_status_) { + if (auto data = weak_data.lock()) { + data->DumpHtmlTableRow(out); + } else { + out << "" + << "" + << "" + << "" << std::endl; + } + } + out << "
BlockerIdStatus
|" << txn_id << "|released
" << std::endl; + + out << "

Extra data:

" << std::endl; + out << "

Num blocking keys: " << blockers_by_key_.size() << "

" << std::endl; + out << "

Num single shard waiters: " + << single_shard_waiters_.size() << "

" << std::endl; + } + + uint64_t GetSerialNo() { + return serial_no_.fetch_add(1); + } + private: void HandleWaiterStatusFromParticipant( WaiterDataPtr waiter, Result res) { if (!res.ok() && res.status().IsNotFound()) { { - SharedLock l(mutex_); + SharedLock l(mutex_); if (shutting_down_) { VLOG_WITH_PREFIX_AND_FUNC(1) << "Skipping status RPC for waiter in shutdown wait queue."; return; @@ -645,7 +1032,7 @@ class WaitQueue::Impl { } WaiterDataPtr waiter; { - UniqueLock l(mutex_); + UniqueLock l(mutex_); auto it = waiter_status_.find(waiter_id); if (it == waiter_status_.end()) { VLOG_WITH_PREFIX(4) << "Got RPC status for removed waiter " << waiter_id; @@ -698,7 +1085,7 @@ class WaitQueue::Impl { << (res.ok() ? res->aborted_subtxn_set.ToString() : "error"); std::shared_ptr resolved_blocker = nullptr; { - SharedLock l(mutex_); + SharedLock l(mutex_); if (shutting_down_) { return; } @@ -724,27 +1111,14 @@ class WaitQueue::Impl { } for (const auto& waiter : resolved_blocker->Signal(std::move(res))) { - WARN_NOT_OK(thread_pool_token_->SubmitFunc([weak_waiter = std::weak_ptr(waiter), this]() { - { - SharedLock l(mutex_); - if (shutting_down_) { - VLOG(4) << "Skipping waiter signal - shutting down"; - return; - } - } - if (auto waiter = weak_waiter.lock()) { - this->SignalWaiter(waiter); - } else { - LOG(INFO) << "Failed to lock weak_ptr to waiter to signal. Skipping."; - } - }), "Failed to submit waiter resumption"); + SignalWaiter(waiter); } } void InvokeWaiterCallback( const Status& status, const WaiterDataPtr& waiter_data) EXCLUDES(mutex_) { if (waiter_data->IsSingleShard()) { - waiter_data->InvokeCallback(status); + waiter_runner_.Submit(waiter_data, status); return; } @@ -754,7 +1128,7 @@ class WaitQueue::Impl { // was now waiting. In this situation, we would want to signal the new waiter. WaiterDataPtr found_waiter = nullptr; { - UniqueLock l(mutex_); + UniqueLock l(mutex_); auto it = waiter_status_.find(waiter_data->id); if (it != waiter_status_.end()) { found_waiter = it->second; @@ -770,7 +1144,7 @@ class WaitQueue::Impl { // callback. Otherwise, the callback will re-run conflict resolution, end up back in the wait // queue, and attempt to reuse the WaiterData still present in waiter_status_. if (found_waiter) { - found_waiter->InvokeCallback(status); + waiter_runner_.Submit(found_waiter, status); } } @@ -779,14 +1153,14 @@ class WaitQueue::Impl { Status status = Status::OK(); size_t num_resolved_blockers = 0; - for (const auto& [blocker_data, subtransaction_info] : waiter_data->blockers) { + for (const auto& [blocker_data, conflict_info] : waiter_data->blockers) { auto is_resolved = blocker_data->IsResolved(); if (!is_resolved.ok()) { status = is_resolved.status(); break; } if (*is_resolved || - !blocker_data->HasLiveSubtransaction(*DCHECK_NOTNULL(subtransaction_info))) { + (conflict_info && !blocker_data->HasLiveSubtransaction(conflict_info->subtransactions))) { num_resolved_blockers++; } } @@ -817,6 +1191,12 @@ class WaitQueue::Impl { TransactionIdHash> blocker_status_ GUARDED_BY(mutex_); + std::unordered_map< + RefCntPrefix, + std::unordered_set, + RefCntPrefixHash> + blockers_by_key_ GUARDED_BY(mutex_); + std::unordered_map< TransactionId, WaiterDataPtr, @@ -827,12 +1207,14 @@ class WaitQueue::Impl { rpc::Rpcs rpcs_; + std::atomic_uint64_t serial_no_ = 0; + TransactionStatusManager* const txn_status_manager_; const std::string& permanent_uuid_; WaitingTxnRegistry* const waiting_txn_registry_; const std::shared_future& client_future_; const server::ClockPtr& clock_; - std::unique_ptr thread_pool_token_; + ResumedWaiterRunner waiter_runner_; scoped_refptr pending_time_waiting_; scoped_refptr finished_waiting_latency_; scoped_refptr blockers_per_waiter_; @@ -856,10 +1238,17 @@ WaitQueue::~WaitQueue() = default; Status WaitQueue::WaitOn( const TransactionId& waiter, LockBatch* locks, - std::vector&& blockers, const TabletId& status_tablet_id, - WaitDoneCallback callback) { + std::shared_ptr blockers, const TabletId& status_tablet_id, + uint64_t serial_no, WaitDoneCallback callback) { return impl_->WaitOn( - waiter, locks, std::move(blockers), status_tablet_id, callback); + waiter, locks, std::move(blockers), status_tablet_id, serial_no, callback); +} + + +Result WaitQueue::MaybeWaitOnLocks( + const TransactionId& waiter, LockBatch* locks, const TabletId& status_tablet_id, + uint64_t serial_no, WaitDoneCallback callback) { + return impl_->MaybeWaitOnLocks(waiter, locks, status_tablet_id, serial_no, callback); } void WaitQueue::Poll(HybridTime now) { @@ -873,6 +1262,13 @@ void WaitQueue::StartShutdown() { void WaitQueue::CompleteShutdown() { return impl_->CompleteShutdown(); } +void WaitQueue::DumpStatusHtml(std::ostream& out) { + return impl_->DumpStatusHtml(out); +} + +uint64_t WaitQueue::GetSerialNo() { + return impl_->GetSerialNo(); +} } // namespace docdb } // namespace yb diff --git a/src/yb/docdb/wait_queue.h b/src/yb/docdb/wait_queue.h index c7d686888b52..7bc012f6f1d0 100644 --- a/src/yb/docdb/wait_queue.h +++ b/src/yb/docdb/wait_queue.h @@ -13,13 +13,14 @@ #pragma once -#include +#include #include "yb/client/client.h" #include "yb/common/common_fwd.h" #include "yb/common/transaction.h" +#include "yb/docdb/conflict_data.h" #include "yb/docdb/lock_batch.h" #include "yb/server/server_fwd.h" @@ -33,7 +34,7 @@ class ScopedWaitingTxnRegistration { public: virtual Status Register( const TransactionId& waiting, - std::vector&& blocking, + std::shared_ptr blockers, const TabletId& status_tablet) = 0; virtual int64 GetDataUseCount() const = 0; virtual ~ScopedWaitingTxnRegistration() = default; @@ -74,8 +75,19 @@ class WaitQueue { // callback. Status WaitOn( const TransactionId& waiter, LockBatch* locks, - std::vector&& blockers, const TabletId& status_tablet_id, - WaitDoneCallback callback); + std::shared_ptr blockers, const TabletId& status_tablet_id, + uint64_t serial_no, WaitDoneCallback callback); + + // Check the wait queue for any active blockers which would conflict with locks. This method + // should be called as the first step in conflict resolution when processing a new request to + // ensure incoming requests do not starve existing blocked requests which are about to resume. + // Returns true if this call results in the request entering the wait queue, in which case the + // provided callback is used as described in the comment of WaitOn() above. Returns false in case + // the request is not entered into the wait queue and the callback is never invoked. Returns + // status in case of some unresolvable error. + Result MaybeWaitOnLocks( + const TransactionId& waiter, LockBatch* locks, const TabletId& status_tablet_id, + uint64_t serial_no, WaitDoneCallback callback); void Poll(HybridTime now); @@ -83,6 +95,17 @@ class WaitQueue { void CompleteShutdown(); + // Provides access to a monotonically increasing serial number to be used by waiting requests to + // enforce fairness in a best effort manner. Incoming requests should retain a serial number as + // soon as they begin conflict resolution, and the same serial number should be used any time the + // request enters the wait queue, to ensure it is resolved before any requests which arrived later + // than it did to this tserver. + uint64_t GetSerialNo(); + + // Output html to display information to an admin page about the internal state of this wait + // queue. Useful for debugging. + void DumpStatusHtml(std::ostream& out); + private: class Impl; std::unique_ptr impl_; diff --git a/src/yb/tablet/running_transaction.cc b/src/yb/tablet/running_transaction.cc index 5aa445766587..40460144bc9c 100644 --- a/src/yb/tablet/running_transaction.cc +++ b/src/yb/tablet/running_transaction.cc @@ -98,6 +98,10 @@ void RunningTransaction::RequestStatusAt(const StatusRequest& request, DCHECK_LE(request.global_limit_ht, HybridTime::kMax); DCHECK_LE(request.read_ht, request.global_limit_ht); + if (request.status_tablet_id) { + *request.status_tablet_id = status_tablet(); + } + if (last_known_status_hybrid_time_ > HybridTime::kMin) { auto transaction_status = GetStatusAt(request.global_limit_ht, last_known_status_hybrid_time_, last_known_status_, diff --git a/src/yb/tablet/tablet_options.h b/src/yb/tablet/tablet_options.h index f745bc779ec1..0dba15571019 100644 --- a/src/yb/tablet/tablet_options.h +++ b/src/yb/tablet/tablet_options.h @@ -81,7 +81,7 @@ struct TabletInitData { TabletSplitter* tablet_splitter = nullptr; std::function allowed_history_cutoff_provider; TransactionManagerProvider transaction_manager_provider; - LocalWaitingTxnRegistry* waiting_txn_registry = nullptr; + docdb::LocalWaitingTxnRegistry* waiting_txn_registry = nullptr; ThreadPool* wait_queue_pool = nullptr; AutoFlagsManager* auto_flags_manager = nullptr; ThreadPool* full_compaction_pool; diff --git a/src/yb/tablet/transaction_participant.cc b/src/yb/tablet/transaction_participant.cc index f92f555559eb..81710f0503cc 100644 --- a/src/yb/tablet/transaction_participant.cc +++ b/src/yb/tablet/transaction_participant.cc @@ -523,14 +523,6 @@ class TransactionParticipant::Impl } } - void FillStatusTablets(std::vector* inout) { - // TODO(wait-queues) optimize locking - std::vector> status_tablet_opts; - for (auto& blocker : *inout) { - blocker.status_tablet = GetStatusTablet(blocker.id).get_value_or(""); - } - } - boost::optional GetStatusTablet(const TransactionId& id) { auto lock_and_iterator = LockAndFind(id, "get status tablet"s, TransactionLoadFlags{}); if (!lock_and_iterator.found() || lock_and_iterator.transaction().WasAborted()) { @@ -1880,11 +1872,6 @@ void TransactionParticipant::FillPriorities( return impl_->FillPriorities(inout); } -void TransactionParticipant::FillStatusTablets( - std::vector* inout) { - return impl_->FillStatusTablets(inout); -} - boost::optional TransactionParticipant::FindStatusTablet(const TransactionId& id) { return impl_->GetStatusTablet(id); } diff --git a/src/yb/tablet/transaction_participant.h b/src/yb/tablet/transaction_participant.h index 0f4b623b2829..aa080aaf06c0 100644 --- a/src/yb/tablet/transaction_participant.h +++ b/src/yb/tablet/transaction_participant.h @@ -178,8 +178,6 @@ class TransactionParticipant : public TransactionStatusManager { void FillPriorities( boost::container::small_vector_base>* inout) override; - void FillStatusTablets(std::vector* inout) override; - boost::optional FindStatusTablet(const TransactionId& id) override; void GetStatus(const TransactionId& transaction_id, diff --git a/src/yb/tablet/write_query.cc b/src/yb/tablet/write_query.cc index 40538c1fa168..8408748b0009 100644 --- a/src/yb/tablet/write_query.cc +++ b/src/yb/tablet/write_query.cc @@ -501,9 +501,6 @@ Status WriteQuery::DoExecute() { TEST_SYNC_POINT("WriteQuery::DoExecute::PreparedDocWriteOps"); auto* transaction_participant = tablet->transaction_participant(); - if (transaction_participant) { - request_scope_ = VERIFY_RESULT(RequestScope::Create(transaction_participant)); - } if (!tablet->txns_enabled() || !transactional_table) { CompleteExecute(); diff --git a/src/yb/tablet/write_query.h b/src/yb/tablet/write_query.h index 276dbd2833dc..672c0ddf7d25 100644 --- a/src/yb/tablet/write_query.h +++ b/src/yb/tablet/write_query.h @@ -112,10 +112,6 @@ class WriteQuery { // Cancel query even before sending underlying operation to the Raft. void Cancel(const Status& status); - const ReadHybridTime& read_time() const { - return read_time_; - } - const tserver::WriteRequestPB* client_request() { return client_request_; } @@ -226,7 +222,6 @@ class WriteQuery { ExecuteMode execute_mode_; IsolationLevel isolation_level_; docdb::PrepareDocWriteOperationResult prepare_result_; - RequestScope request_scope_; std::unique_ptr self_; // Keep self while Execute is performed. }; diff --git a/src/yb/tserver/read_query.cc b/src/yb/tserver/read_query.cc index 252942b7437d..de6a208e24af 100644 --- a/src/yb/tserver/read_query.cc +++ b/src/yb/tserver/read_query.cc @@ -341,9 +341,15 @@ Status ReadQuery::DoPerform() { } if (transactional) { + // TODO(wait-queues) -- having this RequestScope live during conflict resolution may prevent + // intent cleanup for any transactions resolved after this is created and before it's destroyed. + // This may be especially problematic for operations which need to wait, as their waiting may + // now cause intents_db scans to become less performant. Moving this initialization to only + // cover cases where we avoid writes may cause inconsistency issues, as exposed by + // PgOnConflictTest.OnConflict which fails if we move this code below. + request_scope_ = VERIFY_RESULT(RequestScope::Create(tablet()->transaction_participant())); // Serial number is used to check whether this operation was initiated before // transaction status request. So we should initialize it as soon as possible. - request_scope_ = VERIFY_RESULT(RequestScope::Create(tablet()->transaction_participant())); read_time_.serial_no = request_scope_.request_id(); } diff --git a/src/yb/tserver/ts_tablet_manager.cc b/src/yb/tserver/ts_tablet_manager.cc index c029ffcd135c..1993cae605cd 100644 --- a/src/yb/tserver/ts_tablet_manager.cc +++ b/src/yb/tserver/ts_tablet_manager.cc @@ -519,7 +519,7 @@ Status TSTabletManager::Init() { local_peer_pb_.cloud_info()); if (FLAGS_enable_wait_queues) { - waiting_txn_registry_ = std::make_unique( + waiting_txn_registry_ = std::make_unique( client_future(), scoped_refptr(server_->clock())); } diff --git a/src/yb/tserver/ts_tablet_manager.h b/src/yb/tserver/ts_tablet_manager.h index 1daf7eb27bf0..158650a17d96 100644 --- a/src/yb/tserver/ts_tablet_manager.h +++ b/src/yb/tserver/ts_tablet_manager.h @@ -630,7 +630,7 @@ class TSTabletManager : public tserver::TabletPeerLookupIf, public tablet::Table // Used for cleaning up old metrics. std::unique_ptr metrics_cleaner_; - std::unique_ptr waiting_txn_registry_; + std::unique_ptr waiting_txn_registry_; std::unique_ptr waiting_txn_registry_poller_; diff --git a/src/yb/tserver/tserver-path-handlers.cc b/src/yb/tserver/tserver-path-handlers.cc index 3d6f22e34447..8cc6b0cea960 100644 --- a/src/yb/tserver/tserver-path-handlers.cc +++ b/src/yb/tserver/tserver-path-handlers.cc @@ -241,7 +241,8 @@ void HandleTabletPage( {"tablet-consensus-status", "Consensus Status"}, {"log-anchors", "Tablet Log Anchors"}, {"transactions", "Transactions"}, - {"rocksdb", "RocksDB" }}; + {"rocksdb", "RocksDB" }, + {"waitqueue", "Wait Queue"}}; auto encoded_tablet_id = UrlEncodeToString(tablet_id); for (const auto& entry : entries) { @@ -387,6 +388,25 @@ void HandleRocksDBPage( DumpRocksDB("Intents", doc_db.intents, output); } +void HandleWaitQueuePage( + const std::string& tablet_id, const tablet::TabletPeerPtr& peer, + const Webserver::WebRequest& req, Webserver::WebResponse* resp) { + std::stringstream *out = &resp->output; + *out << "

Waiters for Tablet " << EscapeForHtmlToString(tablet_id) << "

" << std::endl; + + auto tablet_result = peer->shared_tablet_safe(); + if (!tablet_result.ok()) { + *out << tablet_result.status(); + return; + } + auto* wq = (*tablet_result)->wait_queue(); + if (wq) { + wq->DumpStatusHtml(*out); + } else { + *out << "

" << "No wait queue found" << "

" << std::endl; + } +} + template void RegisterTabletPathHandler( Webserver* web_server, TabletServer* tserver, const std::string& path, const F& f) { @@ -423,6 +443,7 @@ Status TabletServerPathHandlers::Register(Webserver* server) { RegisterTabletPathHandler(server, tserver_, "/log-anchors", &HandleLogAnchorsPage); RegisterTabletPathHandler(server, tserver_, "/transactions", &HandleTransactionsPage); RegisterTabletPathHandler(server, tserver_, "/rocksdb", &HandleRocksDBPage); + RegisterTabletPathHandler(server, tserver_, "/waitqueue", &HandleWaitQueuePage); server->RegisterPathHandler( "/", "Dashboards", std::bind(&TabletServerPathHandlers::HandleDashboardsPage, this, _1, _2), true /* styled */, diff --git a/src/yb/util/lazy_invoke.h b/src/yb/util/lazy_invoke.h new file mode 100644 index 000000000000..59f368b87702 --- /dev/null +++ b/src/yb/util/lazy_invoke.h @@ -0,0 +1,39 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace yb { + +// Wrapper for a factory of some type T, which is only invoked once the wrappers operator() is +// invoked, i.e. when the factory is cast to type T. +template +class LazyFactory { + public: + explicit LazyFactory(const Factory& factory) : factory_(factory) {} + + constexpr operator std::invoke_result_t() const { + return factory_(); + } + + private: + Factory factory_; +}; + +template +auto MakeLazyFactory(const Factory& factory) { + return LazyFactory(factory); +} + +} // namespace yb From e5caedd0773c35c922eb632b5a9e7990bf9f5aa5 Mon Sep 17 00:00:00 2001 From: Aishwarya Chakravarthy Date: Wed, 1 Mar 2023 17:09:04 -0500 Subject: [PATCH 24/81] [docs] updated supported stable release version for YBM clusters (#16200) * updated release version * date updated --- docs/content/preview/yugabyte-cloud/release-notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/content/preview/yugabyte-cloud/release-notes.md b/docs/content/preview/yugabyte-cloud/release-notes.md index 201d3ef4a256..21af09d586e0 100644 --- a/docs/content/preview/yugabyte-cloud/release-notes.md +++ b/docs/content/preview/yugabyte-cloud/release-notes.md @@ -16,6 +16,10 @@ On this page: ## Releases +### March 1, 2023 + +- [Stable release](../../faq/yugabytedb-managed-faq/#what-version-of-yugabytedb-does-my-cluster-run-on) reset to [version 2.12.9](../../releases/release-notes/v2.12/#v2.12.9.0) for dedicated clusters. New clusters use this version by default. + ### February 13, 2023 **Database** From 088079050fc26b144c516f1975cd1ad869354b4f Mon Sep 17 00:00:00 2001 From: Hari Krishna Sunder Date: Wed, 1 Mar 2023 12:12:02 -0800 Subject: [PATCH 25/81] [#16265] Rename cdc and twodc to xcluster Summary: Rename cdc and twodc to xcluster Files renamed: twodc_output_client.h -> xcluster_output_client.h twodc_output_client.cc -> xcluster_output_client.cc twodc_write_implementations.h -> xcluster_write_implementations.h twodc_write_implementations.cc -> xcluster_write_implementations.cc twodc_write_interface.h -> xcluster_write_interface.h cdc_output_client_interface.h -> xcluster_output_client_interface.h cdc_consumer.h -> xcluster_consumer.h cdc_consumer.cc -> xcluster_consumer.cc cdc_poller.h -> xcluster_poller.h cdc_poller.cc -> xcluster_poller.cc Classes renamed: TwoDCWriteInterface - > XClusterWriteInterface cdc::OutputClientResponse -> XClusterOutputClientResponse cdc::CDCOutputClient -> XClusterOutputClientIf TwoDCOutputClient -> XClusterOutputClient CDCClient -> XClusterClient CDCConsumer -> XClusterConsumer CDCPoller -> XClusterPoller Test gFlag TEST_twodc_write_hybrid_time is renamed to TEST_xcluster_write_hybrid_time Test Plan: All twodc and xcluster tests Reviewers: rahuldesirazu, jhe Reviewed By: jhe Subscribers: ybase, slingam, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D23233 --- src/yb/cdc/cdc_producer.cc | 2 +- src/yb/cdc/cdc_service.cc | 2 +- src/yb/cdc/cdcsdk_producer.cc | 2 +- src/yb/integration-tests/cdc_test_util.cc | 9 +- .../integration-tests/cdcsdk_stream-test.cc | 2 - .../integration-tests/cdcsdk_ysql_test_base.h | 1 - src/yb/integration-tests/twodc-test.cc | 18 +- src/yb/integration-tests/twodc_test_base.cc | 10 +- src/yb/integration-tests/twodc_ysql-test.cc | 4 +- .../xcluster-tablet-split-itest.cc | 2 +- src/yb/tablet/tablet.cc | 12 +- src/yb/tablet/tablet_metadata.cc | 13 +- src/yb/tablet/tablet_metadata.h | 6 +- src/yb/tserver/CMakeLists.txt | 8 +- src/yb/tserver/tablet_server.cc | 30 +-- src/yb/tserver/tablet_server.h | 6 +- ...tserver_metrics_heartbeat_data_provider.cc | 10 +- .../{cdc_consumer.cc => xcluster_consumer.cc} | 190 +++++++++--------- .../{cdc_consumer.h => xcluster_consumer.h} | 28 +-- ...ut_client.cc => xcluster_output_client.cc} | 167 +++++++-------- ...tput_client.h => xcluster_output_client.h} | 15 +- .../xcluster_output_client_interface.h} | 15 +- .../{cdc_poller.cc => xcluster_poller.cc} | 107 +++++----- .../{cdc_poller.h => xcluster_poller.h} | 29 ++- ...s.cc => xcluster_write_implementations.cc} | 13 +- ...interface.h => xcluster_write_interface.h} | 13 +- 26 files changed, 358 insertions(+), 356 deletions(-) rename src/yb/tserver/{cdc_consumer.cc => xcluster_consumer.cc} (79%) rename src/yb/tserver/{cdc_consumer.h => xcluster_consumer.h} (91%) rename src/yb/tserver/{twodc_output_client.cc => xcluster_output_client.cc} (83%) rename src/yb/tserver/{twodc_output_client.h => xcluster_output_client.h} (74%) rename src/yb/{cdc/cdc_output_client_interface.h => tserver/xcluster_output_client_interface.h} (82%) rename src/yb/tserver/{cdc_poller.cc => xcluster_poller.cc} (79%) rename src/yb/tserver/{cdc_poller.h => xcluster_poller.h} (83%) rename src/yb/tserver/{twodc_write_implementations.cc => xcluster_write_implementations.cc} (96%) rename src/yb/tserver/{twodc_write_interface.h => xcluster_write_interface.h} (85%) diff --git a/src/yb/cdc/cdc_producer.cc b/src/yb/cdc/cdc_producer.cc index 3a0d7e856d72..396735e721c1 100644 --- a/src/yb/cdc/cdc_producer.cc +++ b/src/yb/cdc/cdc_producer.cc @@ -345,7 +345,7 @@ Status PopulateWriteRecord(const ReplicateMsgPtr& msg, RETURN_NOT_OK(decoded_key.DecodeFrom(&sub_doc_key, docdb::HybridTimeRequired::kFalse)); if (metadata.record_format == CDCRecordFormat::WAL) { - // For 2DC, populate serialized data from WAL, to avoid unnecessary deserializing on + // For xCluster, populate serialized data from WAL, to avoid unnecessary deserializing on // producer and re-serializing on consumer. auto kv_pair = record->add_key(); if (decoded_key.doc_key().has_hash()) { diff --git a/src/yb/cdc/cdc_service.cc b/src/yb/cdc/cdc_service.cc index b220affcfd03..e2a662b8549a 100644 --- a/src/yb/cdc/cdc_service.cc +++ b/src/yb/cdc/cdc_service.cc @@ -2804,7 +2804,7 @@ void CDCServiceImpl::TabletLeaderGetCheckpoint( auto cdc_proxy = GetCDCServiceProxy(*ts_leader); rpc::RpcController rpc; rpc.set_deadline(GetDeadline(*context, client())); - // TODO(NIC): Change to GetCheckpointAsync like CDCPoller::DoPoll. + // TODO(NIC): Change to GetCheckpointAsync like XClusterPoller::DoPoll. auto status = cdc_proxy->GetCheckpoint(*req, resp, &rpc); RPC_STATUS_RETURN_ERROR(status, resp->mutable_error(), CDCErrorPB::INTERNAL_ERROR, *context); context->RespondSuccess(); diff --git a/src/yb/cdc/cdcsdk_producer.cc b/src/yb/cdc/cdcsdk_producer.cc index b6528e6cd5bf..2070d18b29bc 100644 --- a/src/yb/cdc/cdcsdk_producer.cc +++ b/src/yb/cdc/cdcsdk_producer.cc @@ -1248,7 +1248,7 @@ bool VerifyTabletSplitOnParentTablet( return (children_tablet_count == 2); } -// CDC get changes is different from 2DC as it doesn't need +// CDC get changes is different from xCluster as it doesn't need // to read intents from WAL. Status GetChangesForCDCSDK( diff --git a/src/yb/integration-tests/cdc_test_util.cc b/src/yb/integration-tests/cdc_test_util.cc index 48150335bf01..a75855d0efb1 100644 --- a/src/yb/integration-tests/cdc_test_util.cc +++ b/src/yb/integration-tests/cdc_test_util.cc @@ -22,7 +22,7 @@ #include "yb/tablet/tablet_metadata.h" #include "yb/tablet/tablet_peer.h" -#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/xcluster_consumer.h" #include "yb/tserver/mini_tablet_server.h" #include "yb/tserver/tablet_server.h" #include "yb/tserver/ts_tablet_manager.h" @@ -104,9 +104,10 @@ size_t NumProducerTabletsPolled(MiniCluster* cluster) { for (const auto& mini_tserver : cluster->mini_tablet_servers()) { size_t new_size = 0; auto* tserver = mini_tserver->server(); - tserver::CDCConsumer* cdc_consumer; - if (tserver && (cdc_consumer = tserver->GetCDCConsumer()) && mini_tserver->is_started()) { - auto tablets_running = cdc_consumer->TEST_producer_tablets_running(); + tserver::XClusterConsumer* xcluster_consumer; + if (tserver && (xcluster_consumer = tserver->GetXClusterConsumer()) && + mini_tserver->is_started()) { + auto tablets_running = xcluster_consumer->TEST_producer_tablets_running(); new_size = tablets_running.size(); } size += new_size; diff --git a/src/yb/integration-tests/cdcsdk_stream-test.cc b/src/yb/integration-tests/cdcsdk_stream-test.cc index cd46c0294602..d6fdfc62849e 100644 --- a/src/yb/integration-tests/cdcsdk_stream-test.cc +++ b/src/yb/integration-tests/cdcsdk_stream-test.cc @@ -54,7 +54,6 @@ #include "yb/tablet/tablet.h" #include "yb/tablet/tablet_peer.h" -#include "yb/tserver/cdc_consumer.h" #include "yb/tserver/mini_tablet_server.h" #include "yb/tserver/tablet_server.h" #include "yb/tserver/ts_tablet_manager.h" @@ -86,7 +85,6 @@ using client::YBTableType; using master::GetNamespaceInfoResponsePB; using master::MiniMaster; using tserver::MiniTabletServer; -using tserver::CDCConsumer; using pgwrapper::GetInt32; using pgwrapper::PGConn; diff --git a/src/yb/integration-tests/cdcsdk_ysql_test_base.h b/src/yb/integration-tests/cdcsdk_ysql_test_base.h index fad885f49518..ea7c7d5d5d7b 100644 --- a/src/yb/integration-tests/cdcsdk_ysql_test_base.h +++ b/src/yb/integration-tests/cdcsdk_ysql_test_base.h @@ -65,7 +65,6 @@ #include "yb/tablet/tablet.h" #include "yb/tablet/tablet_peer.h" -#include "yb/tserver/cdc_consumer.h" #include "yb/tserver/mini_tablet_server.h" #include "yb/tserver/tablet_server.h" #include "yb/tserver/ts_tablet_manager.h" diff --git a/src/yb/integration-tests/twodc-test.cc b/src/yb/integration-tests/twodc-test.cc index da2d198263cc..407cefd1c738 100644 --- a/src/yb/integration-tests/twodc-test.cc +++ b/src/yb/integration-tests/twodc-test.cc @@ -60,8 +60,8 @@ #include "yb/server/hybrid_clock.h" #include "yb/tablet/tablet.h" #include "yb/tablet/tablet_peer.h" -#include "yb/tserver/cdc_consumer.h" -#include "yb/tserver/cdc_poller.h" +#include "yb/tserver/xcluster_consumer.h" +#include "yb/tserver/xcluster_poller.h" #include "yb/tserver/mini_tablet_server.h" #include "yb/tserver/tablet_server.h" #include "yb/tserver/ts_tablet_manager.h" @@ -81,7 +81,7 @@ using std::string; using namespace std::literals; DECLARE_bool(enable_ysql); -DECLARE_bool(TEST_twodc_write_hybrid_time); +DECLARE_bool(TEST_xcluster_write_hybrid_time); DECLARE_int32(cdc_wal_retention_time_secs); DECLARE_int32(replication_failure_delay_exponent); DECLARE_double(TEST_respond_write_failed_probability); @@ -136,11 +136,11 @@ using client::YBSession; using client::YBTable; using client::YBTableAlterer; using client::YBTableCreator; -using client::YBTableType; using client::YBTableName; +using client::YBTableType; using master::MiniMaster; using tserver::MiniTabletServer; -using tserver::CDCConsumer; +using tserver::XClusterConsumer; using SessionTransactionPair = std::pair; @@ -1936,7 +1936,7 @@ TEST_P(TwoDCTest, TestExternalWriteHybridTime) { ASSERT_OK(VerifyWrittenRecords(tables[0]->name(), tables[1]->name())); // Delete 2nd record but replicate at a low timestamp (timestamp lower than insertion timestamp). - FLAGS_TEST_twodc_write_hybrid_time = true; + FLAGS_TEST_xcluster_write_hybrid_time = true; DeleteWorkload(1, 2, producer_client(), tables[0]->name()); // Verify that record exists on consumer universe, but is deleted from producer universe. @@ -2575,10 +2575,10 @@ TEST_P(TwoDCTest, TestAlterDDLWithRestarts) { // Verify that the new Consumer poller had read the ALTER DDL and stopped polling. auto* tserver = new_ts->server(); - CDCConsumer* cdc_consumer; - ASSERT_TRUE(tserver && (cdc_consumer = tserver->GetCDCConsumer())); + XClusterConsumer* xcluster_consumer; + ASSERT_TRUE(tserver && (xcluster_consumer = tserver->GetXClusterConsumer())); ASSERT_OK(LoggedWaitFor([&]() -> Result { - auto pollers = tserver->GetCDCConsumer()->TEST_ListPollers(); + auto pollers = xcluster_consumer->TEST_ListPollers(); return pollers.size() == 1 && !pollers[0]->IsPolling(); }, MonoDelta::FromSeconds(10), "ConsumerNotPolling")); } diff --git a/src/yb/integration-tests/twodc_test_base.cc b/src/yb/integration-tests/twodc_test_base.cc index abe1f05dbbb1..3b6074dab6fc 100644 --- a/src/yb/integration-tests/twodc_test_base.cc +++ b/src/yb/integration-tests/twodc_test_base.cc @@ -32,7 +32,7 @@ #include "yb/master/master_replication.proxy.h" #include "yb/master/mini_master.h" #include "yb/rpc/rpc_controller.h" -#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/xcluster_consumer.h" #include "yb/tserver/mini_tablet_server.h" #include "yb/tserver/tablet_server.h" #include "yb/util/backoff_waiter.h" @@ -53,7 +53,7 @@ namespace yb { using client::YBClient; using client::YBTableName; -using tserver::CDCConsumer; +using tserver::XClusterConsumer; Status TwoDCTestBase::InitClusters(const MiniClusterOptions& opts) { FLAGS_replication_factor = static_cast(opts.num_tablet_servers); @@ -442,9 +442,9 @@ uint32_t TwoDCTestBase::GetSuccessfulWriteOps(MiniCluster* cluster) { uint32_t size = 0; for (const auto& mini_tserver : cluster->mini_tablet_servers()) { auto* tserver = mini_tserver->server(); - CDCConsumer* cdc_consumer; - if (tserver && (cdc_consumer = tserver->GetCDCConsumer())) { - size += cdc_consumer->GetNumSuccessfulWriteRpcs(); + XClusterConsumer* xcluster_consumer; + if (tserver && (xcluster_consumer = tserver->GetXClusterConsumer())) { + size += xcluster_consumer->GetNumSuccessfulWriteRpcs(); } } return size; diff --git a/src/yb/integration-tests/twodc_ysql-test.cc b/src/yb/integration-tests/twodc_ysql-test.cc index 91f09486fd55..90bcd2fb0323 100644 --- a/src/yb/integration-tests/twodc_ysql-test.cc +++ b/src/yb/integration-tests/twodc_ysql-test.cc @@ -71,7 +71,6 @@ #include "yb/tserver/tablet_server.h" #include "yb/tserver/ts_tablet_manager.h" -#include "yb/tserver/cdc_consumer.h" #include "yb/util/atomic.h" #include "yb/util/backoff_waiter.h" #include "yb/util/faststring.h" @@ -134,12 +133,11 @@ using client::YBSession; using client::YBTable; using client::YBTableAlterer; using client::YBTableCreator; -using client::YBTableType; using client::YBTableName; +using client::YBTableType; using master::GetNamespaceInfoResponsePB; using master::MiniMaster; using tserver::MiniTabletServer; -using tserver::CDCConsumer; using pgwrapper::ToString; using pgwrapper::GetInt32; diff --git a/src/yb/integration-tests/xcluster-tablet-split-itest.cc b/src/yb/integration-tests/xcluster-tablet-split-itest.cc index 90611aed1f70..918101896a60 100644 --- a/src/yb/integration-tests/xcluster-tablet-split-itest.cc +++ b/src/yb/integration-tests/xcluster-tablet-split-itest.cc @@ -34,7 +34,7 @@ #include "yb/tablet/tablet_metadata.h" #include "yb/tablet/tablet_peer.h" #include "yb/tools/admin-test-base.h" -#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/xcluster_consumer.h" #include "yb/tserver/mini_tablet_server.h" #include "yb/tserver/tablet_server.h" #include "yb/util/backoff_waiter.h" diff --git a/src/yb/tablet/tablet.cc b/src/yb/tablet/tablet.cc index e09a77b13311..8275e9786592 100644 --- a/src/yb/tablet/tablet.cc +++ b/src/yb/tablet/tablet.cc @@ -880,7 +880,7 @@ void Tablet::CleanupIntentFiles() { } void Tablet::DoCleanupIntentFiles() { - if (metadata_->is_under_twodc_replication()) { + if (metadata_->IsUnderXClusterReplication()) { VLOG_WITH_PREFIX_AND_FUNC(4) << "Exit because of TwoDC replication"; return; } @@ -1387,10 +1387,10 @@ Status Tablet::ApplyKeyValueRowOperations( // See comments for PrepareExternalWriteBatch. if (put_batch.enable_replicate_transaction_status_table()) { - if (!metadata_->is_under_twodc_replication()) { + if (!metadata_->IsUnderXClusterReplication()) { // The first time the consumer tablet sees an external write batch, set - // is_under_twodc_replication to true. - RETURN_NOT_OK(metadata_->SetIsUnderTwodcReplicationAndFlush(true)); + // is_under_xcluster_replication to true. + RETURN_NOT_OK(metadata_->SetIsUnderXClusterReplicationAndFlush(true)); } ThreadSafeArena arena; auto batches_by_transaction = SplitExternalBatchIntoTransactionBatches(put_batch, &arena); @@ -1412,8 +1412,8 @@ Status Tablet::ApplyKeyValueRowOperations( external_txn_intents_state_.get()); if (intents_write_batch.Count() != 0) { - if (!metadata_->is_under_twodc_replication()) { - RETURN_NOT_OK(metadata_->SetIsUnderTwodcReplicationAndFlush(true)); + if (!metadata_->IsUnderXClusterReplication()) { + RETURN_NOT_OK(metadata_->SetIsUnderXClusterReplicationAndFlush(true)); } WriteToRocksDB(frontiers, &intents_write_batch, StorageDbType::kIntents); } diff --git a/src/yb/tablet/tablet_metadata.cc b/src/yb/tablet/tablet_metadata.cc index cae47879279f..0ea3f0a565de 100644 --- a/src/yb/tablet/tablet_metadata.cc +++ b/src/yb/tablet/tablet_metadata.cc @@ -791,7 +791,7 @@ Status RaftGroupMetadata::LoadFromSuperBlock(const RaftGroupReplicaSuperBlockPB& } cdc_sdk_safe_time_ = HybridTime::FromPB(superblock.cdc_sdk_safe_time()); - is_under_twodc_replication_ = superblock.is_under_twodc_replication(); + is_under_xcluster_replication_ = superblock.is_under_twodc_replication(); hidden_ = superblock.hidden(); auto restoration_hybrid_time = HybridTime::FromPB(superblock.restoration_hybrid_time()); if (restoration_hybrid_time) { @@ -934,7 +934,7 @@ void RaftGroupMetadata::ToSuperBlockUnlocked(RaftGroupReplicaSuperBlockPB* super pb.set_cdc_min_replicated_index(cdc_min_replicated_index_); cdc_sdk_min_checkpoint_op_id_.ToPB(pb.mutable_cdc_sdk_min_checkpoint_op_id()); pb.set_cdc_sdk_safe_time(cdc_sdk_safe_time_.ToUint64()); - pb.set_is_under_twodc_replication(is_under_twodc_replication_); + pb.set_is_under_twodc_replication(is_under_xcluster_replication_); pb.set_hidden(hidden_); if (restoration_hybrid_time_) { pb.set_restoration_hybrid_time(restoration_hybrid_time_.ToUint64()); @@ -1235,17 +1235,18 @@ Status RaftGroupMetadata::set_cdc_sdk_safe_time(const HybridTime& cdc_sdk_safe_t return Flush(); } -Status RaftGroupMetadata::SetIsUnderTwodcReplicationAndFlush(bool is_under_twodc_replication) { +Status RaftGroupMetadata::SetIsUnderXClusterReplicationAndFlush( + bool is_under_xcluster_replication) { { std::lock_guard lock(data_mutex_); - is_under_twodc_replication_ = is_under_twodc_replication; + is_under_xcluster_replication_ = is_under_xcluster_replication; } return Flush(); } -bool RaftGroupMetadata::is_under_twodc_replication() const { +bool RaftGroupMetadata::IsUnderXClusterReplication() const { std::lock_guard lock(data_mutex_); - return is_under_twodc_replication_; + return is_under_xcluster_replication_; } void RaftGroupMetadata::SetHidden(bool value) { diff --git a/src/yb/tablet/tablet_metadata.h b/src/yb/tablet/tablet_metadata.h index d9b3eff1a4d0..203aeb3b2d58 100644 --- a/src/yb/tablet/tablet_metadata.h +++ b/src/yb/tablet/tablet_metadata.h @@ -358,9 +358,9 @@ class RaftGroupMetadata : public RefCountedThreadSafe, HybridTime cdc_sdk_safe_time() const; - Status SetIsUnderTwodcReplicationAndFlush(bool is_under_twodc_replication); + Status SetIsUnderXClusterReplicationAndFlush(bool is_under_xcluster_replication); - bool is_under_twodc_replication() const; + bool IsUnderXClusterReplication() const; bool has_been_fully_compacted() const { std::lock_guard lock(data_mutex_); @@ -668,7 +668,7 @@ class RaftGroupMetadata : public RefCountedThreadSafe, // The minimum hybrid time based on which data is retained for before image HybridTime cdc_sdk_safe_time_ GUARDED_BY(data_mutex_); - bool is_under_twodc_replication_ GUARDED_BY(data_mutex_) = false; + bool is_under_xcluster_replication_ GUARDED_BY(data_mutex_) = false; bool hidden_ GUARDED_BY(data_mutex_) = false; diff --git a/src/yb/tserver/CMakeLists.txt b/src/yb/tserver/CMakeLists.txt index fa3aa5e2a214..e144d8d7affb 100644 --- a/src/yb/tserver/CMakeLists.txt +++ b/src/yb/tserver/CMakeLists.txt @@ -219,10 +219,10 @@ set(TSERVER_SRCS stateful_services/stateful_service_base.cc stateful_services/test_echo_service.cc xcluster_safe_time_map.cc - cdc_consumer.cc - cdc_poller.cc - twodc_output_client.cc - twodc_write_implementations.cc) + xcluster_consumer.cc + xcluster_poller.cc + xcluster_output_client.cc + xcluster_write_implementations.cc) set(TSERVER_DEPS protobuf diff --git a/src/yb/tserver/tablet_server.cc b/src/yb/tserver/tablet_server.cc index b26a47e6bee8..ece7846d78c7 100644 --- a/src/yb/tserver/tablet_server.cc +++ b/src/yb/tserver/tablet_server.cc @@ -81,7 +81,7 @@ #include "yb/tserver/ts_tablet_manager.h" #include "yb/tserver/tserver-path-handlers.h" #include "yb/tserver/tserver_service.proxy.h" -#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/xcluster_consumer.h" #include "yb/tserver/backup_service.h" #include "yb/cdc/cdc_service.h" @@ -558,9 +558,9 @@ void TabletServer::Shutdown() { bool expected = true; if (initted_.compare_exchange_strong(expected, false, std::memory_order_acq_rel)) { - auto cdc_consumer = GetCDCConsumer(); - if (cdc_consumer) { - cdc_consumer->Shutdown(); + auto xcluster_consumer = GetXClusterConsumer(); + if (xcluster_consumer) { + xcluster_consumer->Shutdown(); } maintenance_manager_->Shutdown(); @@ -971,9 +971,9 @@ Status TabletServer::SetupMessengerBuilder(rpc::MessengerBuilder* builder) { return Status::OK(); } -CDCConsumer* TabletServer::GetCDCConsumer() { +XClusterConsumer* TabletServer::GetXClusterConsumer() { std::lock_guard l(cdc_consumer_mutex_); - return cdc_consumer_.get(); + return xcluster_consumer_.get(); } encryption::UniverseKeyManager* TabletServer::GetUniverseKeyManager() { @@ -995,8 +995,8 @@ Status TabletServer::CreateCDCConsumer() { return tablet_peer->LeaderStatus() == consensus::LeaderStatus::LEADER_AND_READY; }; - cdc_consumer_ = - VERIFY_RESULT(CDCConsumer::Create(std::move(is_leader_clbk), proxy_cache_.get(), this)); + xcluster_consumer_ = + VERIFY_RESULT(XClusterConsumer::Create(std::move(is_leader_clbk), proxy_cache_.get(), this)); return Status::OK(); } @@ -1005,11 +1005,11 @@ Status TabletServer::SetConfigVersionAndConsumerRegistry( std::lock_guard l(cdc_consumer_mutex_); // Only create a cdc consumer if consumer_registry is not null. - if (!cdc_consumer_ && consumer_registry) { + if (!xcluster_consumer_ && consumer_registry) { RETURN_NOT_OK(CreateCDCConsumer()); } - if (cdc_consumer_) { - cdc_consumer_->RefreshWithNewRegistryFromMaster(consumer_registry, cluster_config_version); + if (xcluster_consumer_) { + xcluster_consumer_->RefreshWithNewRegistryFromMaster(consumer_registry, cluster_config_version); } return Status::OK(); } @@ -1019,10 +1019,10 @@ int32_t TabletServer::cluster_config_version() const { // If no CDC consumer, we will return -1, which will force the master to send the consumer // registry if one exists. If we receive one, we will create a new CDC consumer in // SetConsumerRegistry. - if (!cdc_consumer_) { + if (!xcluster_consumer_) { return -1; } - return cdc_consumer_->cluster_config_version(); + return xcluster_consumer_->cluster_config_version(); } Status TabletServer::ReloadKeysAndCertificates() { @@ -1037,8 +1037,8 @@ Status TabletServer::ReloadKeysAndCertificates() { options_.HostsString())); std::lock_guard l(cdc_consumer_mutex_); - if (cdc_consumer_) { - RETURN_NOT_OK(cdc_consumer_->ReloadCertificates()); + if (xcluster_consumer_) { + RETURN_NOT_OK(xcluster_consumer_->ReloadCertificates()); } for (const auto& reloader : certificate_reloaders_) { diff --git a/src/yb/tserver/tablet_server.h b/src/yb/tserver/tablet_server.h index ee4cdc6ec818..4a46cb783edb 100644 --- a/src/yb/tserver/tablet_server.h +++ b/src/yb/tserver/tablet_server.h @@ -72,7 +72,7 @@ class AutoFlagsManager; namespace tserver { -class CDCConsumer; +class XClusterConsumer; class PgClientServiceImpl; class TabletServer : public DbServerBase, public TabletServerIf { @@ -269,7 +269,7 @@ class TabletServer : public DbServerBase, public TabletServerIf { Status SetConfigVersionAndConsumerRegistry( int32_t cluster_config_version, const cdc::ConsumerRegistryPB* consumer_registry); - CDCConsumer* GetCDCConsumer(); + XClusterConsumer* GetXClusterConsumer(); // Mark the CDC service as enabled via heartbeat. Status SetCDCServiceEnabled(); @@ -382,7 +382,7 @@ class TabletServer : public DbServerBase, public TabletServerIf { // CDC consumer. mutable std::mutex cdc_consumer_mutex_; - std::unique_ptr cdc_consumer_ GUARDED_BY(cdc_consumer_mutex_); + std::unique_ptr xcluster_consumer_ GUARDED_BY(cdc_consumer_mutex_); // CDC service. std::shared_ptr cdc_service_; diff --git a/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc b/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc index 669d939a09b1..46ed41e1a0d2 100644 --- a/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc +++ b/src/yb/tserver/tserver_metrics_heartbeat_data_provider.cc @@ -23,7 +23,7 @@ #include "yb/tablet/tablet_metadata.h" #include "yb/tablet/tablet_peer.h" -#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/xcluster_consumer.h" #include "yb/tserver/tablet_server.h" #include "yb/tserver/ts_tablet_manager.h" #include "yb/tserver/tserver_service.service.h" @@ -95,10 +95,10 @@ void TServerMetricsHeartbeatDataProvider::DoAddData( } } - // Report replication errors from the CDC consumer. - auto consumer = server().GetCDCConsumer(); - if (consumer != nullptr && should_add_replication_status) { - const auto tablet_replication_error_map = consumer->GetReplicationErrors(); + // Report replication errors from the xCluster consumer. + auto xcluster_consumer = server().GetXClusterConsumer(); + if (xcluster_consumer != nullptr && should_add_replication_status) { + const auto tablet_replication_error_map = xcluster_consumer->GetReplicationErrors(); for (const auto& tablet_kv : tablet_replication_error_map) { const TabletId& tablet_id = tablet_kv.first; diff --git a/src/yb/tserver/cdc_consumer.cc b/src/yb/tserver/xcluster_consumer.cc similarity index 79% rename from src/yb/tserver/cdc_consumer.cc rename to src/yb/tserver/xcluster_consumer.cc index c02c3e03f7c6..c52416344551 100644 --- a/src/yb/tserver/cdc_consumer.cc +++ b/src/yb/tserver/xcluster_consumer.cc @@ -28,10 +28,10 @@ #include "yb/rpc/proxy.h" #include "yb/rpc/rpc.h" #include "yb/rpc/secure_stream.h" -#include "yb/tserver/cdc_consumer.h" -#include "yb/tserver/twodc_output_client.h" +#include "yb/tserver/xcluster_consumer.h" +#include "yb/tserver/xcluster_output_client.h" #include "yb/tserver/tablet_server.h" -#include "yb/tserver/cdc_poller.h" +#include "yb/tserver/xcluster_poller.h" #include "yb/cdc/cdc_consumer.pb.h" #include "yb/cdc/cdc_util.h" @@ -51,9 +51,9 @@ using std::string; DEFINE_UNKNOWN_int32(cdc_consumer_handler_thread_pool_size, 0, - "Override the max thread pool size for CDCConsumerHandler, which is used by " - "CDCPollers. If set to 0, then the thread pool will use the default size (number of " - "cpus on the system)."); + "Override the max thread pool size for CDCConsumerHandler, which is used by " + "CDCPollers. If set to 0, then the thread pool will use the default size (number of " + "cpus on the system)."); TAG_FLAG(cdc_consumer_handler_thread_pool_size, advanced); DEFINE_RUNTIME_int32(xcluster_safe_time_update_interval_secs, 1, @@ -83,13 +83,13 @@ namespace yb { namespace tserver { using cdc::ProducerTabletInfo; -CDCClient::~CDCClient() { +XClusterClient::~XClusterClient() { if (messenger) { messenger->Shutdown(); } } -void CDCClient::Shutdown() { +void XClusterClient::Shutdown() { if (client) { client->Shutdown(); } @@ -98,7 +98,7 @@ void CDCClient::Shutdown() { } } -Result> CDCConsumer::Create( +Result> XClusterConsumer::Create( std::function is_leader_for_tablet, rpc::ProxyCache* proxy_cache, TabletServer* tserver) { @@ -109,7 +109,7 @@ Result> CDCConsumer::Create( hostport_strs.push_back(HostPort::ToCommaSeparatedString(hp)); } - auto local_client = std::make_unique(); + auto local_client = std::make_unique(); if (FLAGS_use_node_to_node_encryption) { rpc::MessengerBuilder messenger_builder("cdc-consumer"); @@ -119,34 +119,36 @@ Result> CDCConsumer::Create( local_client->messenger = VERIFY_RESULT(messenger_builder.Build()); } - local_client->client = VERIFY_RESULT(client::YBClientBuilder() - .master_server_addrs(hostport_strs) - .set_client_name("CDCConsumerLocal") - .default_rpc_timeout(MonoDelta::FromMilliseconds(FLAGS_cdc_write_rpc_timeout_ms)) - .Build(local_client->messenger.get())); + local_client->client = VERIFY_RESULT( + client::YBClientBuilder() + .master_server_addrs(hostport_strs) + .set_client_name("CDCConsumerLocal") + .default_rpc_timeout(MonoDelta::FromMilliseconds(FLAGS_cdc_write_rpc_timeout_ms)) + .Build(local_client->messenger.get())); local_client->client->SetLocalTabletServer(tserver->permanent_uuid(), tserver->proxy(), tserver); - auto cdc_consumer = std::make_unique(std::move(is_leader_for_tablet), proxy_cache, - tserver->permanent_uuid(), std::move(local_client), &tserver->TransactionManager()); + auto xcluster_consumer = std::make_unique( + std::move(is_leader_for_tablet), proxy_cache, tserver->permanent_uuid(), + std::move(local_client), &tserver->TransactionManager()); - // TODO(NIC): Unify cdc_consumer thread_pool & remote_client_ threadpools + // TODO(NIC): Unify xcluster_consumer thread_pool & remote_client_ threadpools RETURN_NOT_OK(yb::Thread::Create( - "CDCConsumer", "Poll", &CDCConsumer::RunThread, cdc_consumer.get(), - &cdc_consumer->run_trigger_poll_thread_)); + "XClusterConsumer", "Poll", &XClusterConsumer::RunThread, xcluster_consumer.get(), + &xcluster_consumer->run_trigger_poll_thread_)); ThreadPoolBuilder cdc_consumer_thread_pool_builder("CDCConsumerHandler"); if (FLAGS_cdc_consumer_handler_thread_pool_size > 0) { cdc_consumer_thread_pool_builder.set_max_threads(FLAGS_cdc_consumer_handler_thread_pool_size); } - RETURN_NOT_OK(cdc_consumer_thread_pool_builder.Build(&cdc_consumer->thread_pool_)); + RETURN_NOT_OK(cdc_consumer_thread_pool_builder.Build(&xcluster_consumer->thread_pool_)); - return cdc_consumer; + return xcluster_consumer; } -CDCConsumer::CDCConsumer( +XClusterConsumer::XClusterConsumer( std::function is_leader_for_tablet, rpc::ProxyCache* proxy_cache, const string& ts_uuid, - std::unique_ptr + std::unique_ptr local_client, client::TransactionManager* transaction_manager) : is_leader_for_tablet_(std::move(is_leader_for_tablet)), @@ -156,13 +158,13 @@ CDCConsumer::CDCConsumer( last_safe_time_published_at_(MonoTime::Now()), transaction_manager_(transaction_manager) {} -CDCConsumer::~CDCConsumer() { +XClusterConsumer::~XClusterConsumer() { Shutdown(); SharedLock read_lock(producer_pollers_map_mutex_); DCHECK(producer_pollers_map_.empty()); } -void CDCConsumer::Shutdown() { +void XClusterConsumer::Shutdown() { LOG_WITH_PREFIX(INFO) << "Shutting down CDC Consumer"; { std::lock_guard l(should_run_mutex_); @@ -175,7 +177,7 @@ void CDCConsumer::Shutdown() { } // Shutdown the pollers outside of the master_data_mutex lock to keep lock ordering the same. - std::vector> pollers_to_shutdown; + std::vector> pollers_to_shutdown; { std::lock_guard write_lock(master_data_mutex_); producer_consumer_tablet_map_from_master_.clear(); @@ -183,7 +185,7 @@ void CDCConsumer::Shutdown() { { std::lock_guard producer_pollers_map_write_lock(producer_pollers_map_mutex_); // Shutdown the remote and local clients, and abort any of their ongoing rpcs. - for (auto &uuid_and_client : remote_clients_) { + for (auto& uuid_and_client : remote_clients_) { uuid_and_client.second->Shutdown(); } @@ -207,7 +209,7 @@ void CDCConsumer::Shutdown() { } } -void CDCConsumer::RunThread() { +void XClusterConsumer::RunThread() { while (true) { { std::unique_lock l(should_run_mutex_); @@ -228,13 +230,13 @@ void CDCConsumer::RunThread() { } } -void CDCConsumer::RefreshWithNewRegistryFromMaster(const cdc::ConsumerRegistryPB* consumer_registry, - int32_t cluster_config_version) { +void XClusterConsumer::RefreshWithNewRegistryFromMaster( + const cdc::ConsumerRegistryPB* consumer_registry, int32_t cluster_config_version) { UpdateInMemoryState(consumer_registry, cluster_config_version); cond_.notify_all(); } -std::vector CDCConsumer::TEST_producer_tablets_running() { +std::vector XClusterConsumer::TEST_producer_tablets_running() { SharedLock read_lock(producer_pollers_map_mutex_); std::vector tablets; @@ -244,8 +246,8 @@ std::vector CDCConsumer::TEST_producer_tablets_running() { return tablets; } -std::vector> CDCConsumer::TEST_ListPollers() { - std::vector> ret; +std::vector> XClusterConsumer::TEST_ListPollers() { + std::vector> ret; { SharedLock read_lock(producer_pollers_map_mutex_); for (const auto& producer : producer_pollers_map_) { @@ -256,11 +258,11 @@ std::vector> CDCConsumer::TEST_ListPollers() { } // NOTE: This happens on TS.heartbeat, so it needs to finish quickly -void CDCConsumer::UpdateInMemoryState(const cdc::ConsumerRegistryPB* consumer_registry, - int32_t cluster_config_version) { +void XClusterConsumer::UpdateInMemoryState( + const cdc::ConsumerRegistryPB* consumer_registry, int32_t cluster_config_version) { { std::lock_guard l(should_run_mutex_); - if(!should_run_) { + if (!should_run_) { return; } } @@ -274,7 +276,7 @@ void CDCConsumer::UpdateInMemoryState(const cdc::ConsumerRegistryPB* consumer_re if (consumer_registry->enable_replicate_transaction_status_table() && !global_transaction_status_table_) { auto global_transaction_status_table_name = client::YBTableName( - YQL_DATABASE_CQL, master::kSystemNamespaceName, kGlobalTransactionsTableName); + YQL_DATABASE_CQL, master::kSystemNamespaceName, kGlobalTransactionsTableName); auto global_transaction_status_table_res = local_client_->client->OpenTable(global_transaction_status_table_name); if (!global_transaction_status_table_res.ok()) { @@ -323,8 +325,8 @@ void CDCConsumer::UpdateInMemoryState(const cdc::ConsumerRegistryPB* consumer_re for (const auto& stream_entry : producer_entry_pb.stream_map()) { const auto& stream_entry_pb = stream_entry.second; if (stream_entry_pb.local_tserver_optimized()) { - LOG_WITH_PREFIX(INFO) << Format("Stream $0 will use local tserver optimization", - stream_entry.first); + LOG_WITH_PREFIX(INFO) << Format( + "Stream $0 will use local tserver optimization", stream_entry.first); streams_with_local_tserver_optimization_.insert(stream_entry.first); } if (stream_entry_pb.has_producer_schema()) { @@ -339,11 +341,10 @@ void CDCConsumer::UpdateInMemoryState(const cdc::ConsumerRegistryPB* consumer_re {producer_map.first, stream_entry.first, producer_tablet_id}); cdc::ConsumerTabletInfo consumer_tablet_info( {consumer_tablet_id, stream_entry_pb.consumer_table_id()}); - auto xCluster_tablet_info = cdc::XClusterTabletInfo { - .producer_tablet_info = producer_tablet_info, - .consumer_tablet_info = consumer_tablet_info, - .disable_stream = producer_entry_pb.disable_stream() - }; + auto xCluster_tablet_info = cdc::XClusterTabletInfo{ + .producer_tablet_info = producer_tablet_info, + .consumer_tablet_info = consumer_tablet_info, + .disable_stream = producer_entry_pb.disable_stream()}; producer_consumer_tablet_map_from_master_.emplace(xCluster_tablet_info); } } @@ -354,7 +355,7 @@ void CDCConsumer::UpdateInMemoryState(const cdc::ConsumerRegistryPB* consumer_re cond_.notify_all(); } -Result CDCConsumer::GetConsumerTableInfo( +Result XClusterConsumer::GetConsumerTableInfo( const TabletId& producer_tablet_id) { SharedLock lock(master_data_mutex_); const auto& index_by_tablet = producer_consumer_tablet_map_from_master_.get(); @@ -374,7 +375,7 @@ Result CDCConsumer::GetConsumerTableInfo( return it->consumer_tablet_info; } -void CDCConsumer::TriggerPollForNewTablets() { +void XClusterConsumer::TriggerPollForNewTablets() { std::lock_guard write_lock_master(master_data_mutex_); for (const auto& entry : producer_consumer_tablet_map_from_master_) { @@ -399,13 +400,13 @@ void CDCConsumer::TriggerPollForNewTablets() { if (status.ok()) { changed_master_addrs_.erase(uuid); } else { - LOG_WITH_PREFIX(WARNING) << "Problem Setting Master Addresses for " << uuid - << ": " << status.ToString(); + LOG_WITH_PREFIX(WARNING) + << "Problem Setting Master Addresses for " << uuid << ": " << status.ToString(); } } } if (start_polling) { - std::lock_guard write_lock_pollers(producer_pollers_map_mutex_); + std::lock_guard write_lock_pollers(producer_pollers_map_mutex_); // Check again, since we unlocked. start_polling = @@ -417,43 +418,44 @@ void CDCConsumer::TriggerPollForNewTablets() { if (!ContainsKey(remote_clients_, uuid)) { CHECK(ContainsKey(uuid_master_addrs_, uuid)); - auto remote_client = std::make_unique(); + auto remote_client = std::make_unique(); std::string dir; if (FLAGS_use_node_to_node_encryption) { rpc::MessengerBuilder messenger_builder("cdc-consumer"); if (!FLAGS_certs_for_cdc_dir.empty()) { - dir = JoinPathSegments(FLAGS_certs_for_cdc_dir, - cdc::GetOriginalReplicationUniverseId(uuid)); + dir = JoinPathSegments( + FLAGS_certs_for_cdc_dir, cdc::GetOriginalReplicationUniverseId(uuid)); } auto secure_context_result = server::SetupSecureContext( dir, "", "", server::SecureContextType::kInternal, &messenger_builder); if (!secure_context_result.ok()) { - LOG(WARNING) << "Could not create secure context for " << uuid - << ": " << secure_context_result.status().ToString(); - return; // Don't finish creation. Try again on the next heartbeat. + LOG(WARNING) << "Could not create secure context for " << uuid << ": " + << secure_context_result.status().ToString(); + return; // Don't finish creation. Try again on the next heartbeat. } remote_client->secure_context = std::move(*secure_context_result); auto messenger_result = messenger_builder.Build(); if (!messenger_result.ok()) { - LOG(WARNING) << "Could not build messenger for " << uuid - << ": " << secure_context_result.status().ToString(); - return; // Don't finish creation. Try again on the next heartbeat. + LOG(WARNING) << "Could not build messenger for " << uuid << ": " + << secure_context_result.status().ToString(); + return; // Don't finish creation. Try again on the next heartbeat. } remote_client->messenger = std::move(*messenger_result); } - auto client_result = yb::client::YBClientBuilder() - .set_client_name("CDCConsumerRemote") - .add_master_server_addr(uuid_master_addrs_[uuid]) - .skip_master_flagfile() - .default_rpc_timeout(MonoDelta::FromMilliseconds(FLAGS_cdc_read_rpc_timeout_ms)) - .Build(remote_client->messenger.get()); + auto client_result = + yb::client::YBClientBuilder() + .set_client_name("CDCConsumerRemote") + .add_master_server_addr(uuid_master_addrs_[uuid]) + .skip_master_flagfile() + .default_rpc_timeout(MonoDelta::FromMilliseconds(FLAGS_cdc_read_rpc_timeout_ms)) + .Build(remote_client->messenger.get()); if (!client_result.ok()) { - LOG(WARNING) << "Could not create a new YBClient for " << uuid - << ": " << client_result.status().ToString(); - return; // Don't finish creation. Try again on the next heartbeat. + LOG(WARNING) << "Could not create a new YBClient for " << uuid << ": " + << client_result.status().ToString(); + return; // Don't finish creation. Try again on the next heartbeat. } remote_client->client = std::move(*client_result); @@ -470,7 +472,7 @@ void CDCConsumer::TriggerPollForNewTablets() { bool use_local_tserver = streams_with_local_tserver_optimization_.find(producer_tablet_info.stream_id) != streams_with_local_tserver_optimization_.end(); - auto cdc_poller = std::make_shared( + auto xcluster_poller = std::make_shared( producer_tablet_info, consumer_tablet_info, thread_pool_.get(), rpcs_.get(), local_client_, remote_clients_[uuid], this, use_local_tserver, global_transaction_status_table_, enable_replicate_transaction_status_table_, @@ -478,26 +480,26 @@ void CDCConsumer::TriggerPollForNewTablets() { LOG_WITH_PREFIX(INFO) << Format( "Start polling for producer tablet $0, consumer tablet $1", producer_tablet_info, consumer_tablet_info.tablet_id); - producer_pollers_map_[producer_tablet_info] = cdc_poller; - cdc_poller->Poll(); + producer_pollers_map_[producer_tablet_info] = xcluster_poller; + xcluster_poller->Poll(); } } auto schema_version_iter = stream_to_schema_version_.find(producer_tablet_info.stream_id); if (schema_version_iter != stream_to_schema_version_.end()) { SharedLock read_lock_pollers(producer_pollers_map_mutex_); - auto cdc_poller_iter = producer_pollers_map_.find(producer_tablet_info); - if (cdc_poller_iter != producer_pollers_map_.end()) { - cdc_poller_iter->second->SetSchemaVersion(schema_version_iter->second.first, - schema_version_iter->second.second); + auto xcluster_poller_iter = producer_pollers_map_.find(producer_tablet_info); + if (xcluster_poller_iter != producer_pollers_map_.end()) { + xcluster_poller_iter->second->SetSchemaVersion( + schema_version_iter->second.first, schema_version_iter->second.second); } } } } -void CDCConsumer::TriggerDeletionOfOldPollers() { +void XClusterConsumer::TriggerDeletionOfOldPollers() { // Shutdown outside of master_data_mutex_ lock, to not block any heartbeats. - std::vector> clients_to_delete; - std::vector> pollers_to_shutdown; + std::vector> clients_to_delete; + std::vector> pollers_to_shutdown; { SharedLock read_lock_master(master_data_mutex_); std::lock_guard write_lock_pollers(producer_pollers_map_mutex_); @@ -534,34 +536,29 @@ void CDCConsumer::TriggerDeletionOfOldPollers() { } } -bool CDCConsumer::ShouldContinuePolling( +bool XClusterConsumer::ShouldContinuePolling( const ProducerTabletInfo producer_tablet_info, const cdc::ConsumerTabletInfo consumer_tablet_info) { const auto& it = producer_consumer_tablet_map_from_master_.find(producer_tablet_info); // We either no longer need to poll for this tablet, or a different tablet should be polling // for it now instead of this one (due to a local tablet split). if (it == producer_consumer_tablet_map_from_master_.end() || - it->consumer_tablet_info.tablet_id != consumer_tablet_info.tablet_id || - it->disable_stream) { + it->consumer_tablet_info.tablet_id != consumer_tablet_info.tablet_id || it->disable_stream) { // We no longer want to poll for this tablet, abort the cycle. return false; } return is_leader_for_tablet_(it->consumer_tablet_info.tablet_id); } -std::string CDCConsumer::LogPrefix() { - return log_prefix_; -} +std::string XClusterConsumer::LogPrefix() { return log_prefix_; } -int32_t CDCConsumer::cluster_config_version() const { +int32_t XClusterConsumer::cluster_config_version() const { return cluster_config_version_.load(std::memory_order_acquire); } -client::TransactionManager* CDCConsumer::TransactionManager() { - return transaction_manager_; -} +client::TransactionManager* XClusterConsumer::TransactionManager() { return transaction_manager_; } -Status CDCConsumer::ReloadCertificates() { +Status XClusterConsumer::ReloadCertificates() { if (local_client_->secure_context) { RETURN_NOT_OK(server::ReloadSecureContextKeysAndCertificates( local_client_->secure_context.get(), "" /* node_name */, "" /* root_dir*/, @@ -577,8 +574,8 @@ Status CDCConsumer::ReloadCertificates() { std::string cert_dir; if (!FLAGS_certs_for_cdc_dir.empty()) { - cert_dir = JoinPathSegments(FLAGS_certs_for_cdc_dir, - cdc::GetOriginalReplicationUniverseId(entry.first)); + cert_dir = JoinPathSegments( + FLAGS_certs_for_cdc_dir, cdc::GetOriginalReplicationUniverseId(entry.first)); } RETURN_NOT_OK(server::ReloadSecureContextKeysAndCertificates( client->secure_context.get(), cert_dir, "" /* node_name */)); @@ -587,7 +584,7 @@ Status CDCConsumer::ReloadCertificates() { return Status::OK(); } -Status CDCConsumer::PublishXClusterSafeTime() { +Status XClusterConsumer::PublishXClusterSafeTime() { if (consumer_role_ == cdc::XClusterRole::ACTIVE) { return Status::OK(); } @@ -657,20 +654,19 @@ Status CDCConsumer::PublishXClusterSafeTime() { return Status::OK(); } -void CDCConsumer::StoreReplicationError( +void XClusterConsumer::StoreReplicationError( const TabletId& tablet_id, const CDCStreamId& stream_id, const ReplicationErrorPb error, const std::string& detail) { - std::lock_guard lock(tablet_replication_error_map_lock_); tablet_replication_error_map_[tablet_id][stream_id][error] = detail; } -cdc::TabletReplicationErrorMap CDCConsumer::GetReplicationErrors() const { +cdc::TabletReplicationErrorMap XClusterConsumer::GetReplicationErrors() const { std::lock_guard lock(tablet_replication_error_map_lock_); return tablet_replication_error_map_; } -} // namespace tserver -} // namespace yb +} // namespace tserver +} // namespace yb diff --git a/src/yb/tserver/cdc_consumer.h b/src/yb/tserver/xcluster_consumer.h similarity index 91% rename from src/yb/tserver/cdc_consumer.h rename to src/yb/tserver/xcluster_consumer.h index 5ad7a0578a77..28ecb606ac5d 100644 --- a/src/yb/tserver/cdc_consumer.h +++ b/src/yb/tserver/xcluster_consumer.h @@ -51,34 +51,36 @@ class ConsumerRegistryPB; } // namespace cdc namespace tserver { -class CDCPoller; +class XClusterPoller; class TabletServer; -struct CDCClient { +struct XClusterClient { std::unique_ptr messenger; std::unique_ptr secure_context; std::shared_ptr client; - ~CDCClient(); + ~XClusterClient(); void Shutdown(); }; typedef std::pair SchemaVersionMapping; -class CDCConsumer { +class XClusterConsumer { public: - static Result> Create( + static Result> Create( std::function is_leader_for_tablet, rpc::ProxyCache* proxy_cache, TabletServer* tserver); - CDCConsumer(std::function is_leader_for_tablet, + XClusterConsumer( + std::function is_leader_for_tablet, rpc::ProxyCache* proxy_cache, const std::string& ts_uuid, - std::unique_ptr local_client, + std::unique_ptr + local_client, client::TransactionManager* transaction_manager); - ~CDCConsumer(); + ~XClusterConsumer(); void Shutdown() EXCLUDES(should_run_mutex_); // Refreshes the in memory state when we receive a new registry from master. @@ -87,7 +89,7 @@ class CDCConsumer { std::vector TEST_producer_tablets_running(); - std::vector> TEST_ListPollers(); + std::vector> TEST_ListPollers(); std::string LogPrefix(); @@ -183,7 +185,7 @@ class CDCConsumer { scoped_refptr run_trigger_poll_thread_; - std::unordered_map, + std::unordered_map, cdc::ProducerTabletInfo::Hash> producer_pollers_map_ GUARDED_BY(producer_pollers_map_mutex_); @@ -191,11 +193,11 @@ class CDCConsumer { std::unique_ptr rpcs_; std::string log_prefix_; - std::shared_ptr local_client_; + std::shared_ptr local_client_; // map: {universe_uuid : ...}. - std::unordered_map> remote_clients_ - GUARDED_BY(producer_pollers_map_mutex_); + std::unordered_map> remote_clients_ + GUARDED_BY(producer_pollers_map_mutex_); std::unordered_map uuid_master_addrs_ GUARDED_BY(master_data_mutex_); std::unordered_set changed_master_addrs_ GUARDED_BY(master_data_mutex_); diff --git a/src/yb/tserver/twodc_output_client.cc b/src/yb/tserver/xcluster_output_client.cc similarity index 83% rename from src/yb/tserver/twodc_output_client.cc rename to src/yb/tserver/xcluster_output_client.cc index 4fee79054de4..7866216d62be 100644 --- a/src/yb/tserver/twodc_output_client.cc +++ b/src/yb/tserver/xcluster_output_client.cc @@ -10,7 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations // under the License. -#include "yb/tserver/twodc_output_client.h" +#include "yb/tserver/xcluster_output_client.h" #include @@ -28,9 +28,9 @@ #include "yb/master/master_replication.pb.h" #include "yb/rpc/rpc.h" #include "yb/rpc/rpc_fwd.h" -#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/xcluster_consumer.h" #include "yb/tserver/tserver_service.proxy.h" -#include "yb/tserver/twodc_write_interface.h" +#include "yb/tserver/xcluster_write_interface.h" #include "yb/util/flags.h" #include "yb/util/logging.h" #include "yb/util/net/net_util.h" @@ -56,34 +56,35 @@ using namespace std::placeholders; namespace yb { namespace tserver { -class TwoDCOutputClient : public cdc::CDCOutputClient { +class XClusterOutputClient : public XClusterOutputClientIf { public: - TwoDCOutputClient( - CDCConsumer* cdc_consumer, + XClusterOutputClient( + XClusterConsumer* xcluster_consumer, const cdc::ConsumerTabletInfo& consumer_tablet_info, const cdc::ProducerTabletInfo& producer_tablet_info, - const std::shared_ptr& local_client, + const std::shared_ptr& local_client, ThreadPool* thread_pool, rpc::Rpcs* rpcs, - std::function apply_changes_clbk, + std::function + apply_changes_clbk, bool use_local_tserver, client::YBTablePtr global_transaction_status_table, - bool enable_replicate_transaction_status_table) : - cdc_consumer_(cdc_consumer), - consumer_tablet_info_(consumer_tablet_info), - producer_tablet_info_(producer_tablet_info), - local_client_(local_client), - thread_pool_(thread_pool), - rpcs_(rpcs), - write_handle_(rpcs->InvalidHandle()), - apply_changes_clbk_(std::move(apply_changes_clbk)), - use_local_tserver_(use_local_tserver), - all_tablets_result_(STATUS(Uninitialized, "Result has not been initialized.")), - global_transaction_status_table_(global_transaction_status_table), - enable_replicate_transaction_status_table_(enable_replicate_transaction_status_table) {} - - ~TwoDCOutputClient() { - VLOG_WITH_PREFIX_UNLOCKED(1) << "Destroying TwoDCOutputClient"; + bool enable_replicate_transaction_status_table) + : xcluster_consumer_(xcluster_consumer), + consumer_tablet_info_(consumer_tablet_info), + producer_tablet_info_(producer_tablet_info), + local_client_(local_client), + thread_pool_(thread_pool), + rpcs_(rpcs), + write_handle_(rpcs->InvalidHandle()), + apply_changes_clbk_(std::move(apply_changes_clbk)), + use_local_tserver_(use_local_tserver), + all_tablets_result_(STATUS(Uninitialized, "Result has not been initialized.")), + global_transaction_status_table_(global_transaction_status_table), + enable_replicate_transaction_status_table_(enable_replicate_transaction_status_table) {} + + ~XClusterOutputClient() { + VLOG_WITH_PREFIX_UNLOCKED(1) << "Destroying XClusterOutputClient"; DCHECK(shutdown_); } @@ -110,8 +111,8 @@ class TwoDCOutputClient : public cdc::CDCOutputClient { private: // Utility function since we are inheriting shared_from_this(). - std::shared_ptr SharedFromThis() { - return std::dynamic_pointer_cast(shared_from_this()); + std::shared_ptr SharedFromThis() { + return std::dynamic_pointer_cast(shared_from_this()); } std::string LogPrefixUnlocked() const { @@ -125,7 +126,7 @@ class TwoDCOutputClient : public cdc::CDCOutputClient { void SetLastCompatibleConsumerSchemaVersionUnlocked(uint32_t schema_version) REQUIRES(lock_); - // Process all records in twodc_resp_copy_ starting from the start index. If we find a ddl + // Process all records in xcluster_resp_copy_ starting from the start index. If we find a ddl // record, then we process the current changes first, wait for those to complete, then process // the ddl + other changes after. Status ProcessChangesStartingFromIndex(int start); @@ -172,24 +173,24 @@ class TwoDCOutputClient : public cdc::CDCOutputClient { // Returns true if all records are processed, false if there are still some pending records. bool IncProcessedRecordCount() REQUIRES(lock_); - cdc::OutputClientResponse PrepareResponse() REQUIRES(lock_); - void SendResponse(const cdc::OutputClientResponse& resp) EXCLUDES(lock_); + XClusterOutputClientResponse PrepareResponse() REQUIRES(lock_); + void SendResponse(const XClusterOutputClientResponse& resp) EXCLUDES(lock_); void HandleResponse() EXCLUDES(lock_); void HandleError(const Status& s) EXCLUDES(lock_); bool UseLocalTserver(); - CDCConsumer* cdc_consumer_; + XClusterConsumer* xcluster_consumer_; cdc::ConsumerTabletInfo consumer_tablet_info_; cdc::ProducerTabletInfo producer_tablet_info_; - std::shared_ptr local_client_; + std::shared_ptr local_client_; ThreadPool* thread_pool_; // Use threadpool so that callbacks aren't run on reactor threads. rpc::Rpcs* rpcs_; rpc::Rpcs::Handle write_handle_ GUARDED_BY(lock_); // Retain COMMIT rpcs in-flight as these need to be cleaned up on shutdown std::vector> external_transactions_; - std::function apply_changes_clbk_; + std::function apply_changes_clbk_; bool use_local_tserver_; @@ -209,7 +210,7 @@ class TwoDCOutputClient : public cdc::CDCOutputClient { SchemaVersion last_compatible_consumer_schema_version_ GUARDED_BY(lock_) = 0; // This will cache the response to an ApplyChanges() request. - cdc::GetChangesResponsePB twodc_resp_copy_; + cdc::GetChangesResponsePB xcluster_resp_copy_; // Store the result of the lookup for all the tablets. yb::Result>> all_tablets_result_; @@ -218,7 +219,7 @@ class TwoDCOutputClient : public cdc::CDCOutputClient { yb::MonoDelta timeout_ms_; - std::unique_ptr write_strategy_ GUARDED_BY(lock_); + std::unique_ptr write_strategy_ GUARDED_BY(lock_); bool enable_replicate_transaction_status_table_; }; @@ -239,7 +240,7 @@ class TwoDCOutputClient : public cdc::CDCOutputClient { return; \ } -void TwoDCOutputClient::SetLastCompatibleConsumerSchemaVersionUnlocked( +void XClusterOutputClient::SetLastCompatibleConsumerSchemaVersionUnlocked( SchemaVersion schema_version) { if (schema_version != cdc::kInvalidSchemaVersion && schema_version > last_compatible_consumer_schema_version_) { @@ -249,12 +250,12 @@ void TwoDCOutputClient::SetLastCompatibleConsumerSchemaVersionUnlocked( } } -void TwoDCOutputClient::SetLastCompatibleConsumerSchemaVersion(SchemaVersion schema_version) { +void XClusterOutputClient::SetLastCompatibleConsumerSchemaVersion(SchemaVersion schema_version) { std::lock_guard lock(lock_); SetLastCompatibleConsumerSchemaVersionUnlocked(schema_version); } -Status TwoDCOutputClient::ApplyChanges(const cdc::GetChangesResponsePB* poller_resp) { +Status XClusterOutputClient::ApplyChanges(const cdc::GetChangesResponsePB* poller_resp) { // ApplyChanges is called in a single threaded manner. // For all the changes in GetChangesResponsePB, we first fan out and find the tablet for // every record key. @@ -263,7 +264,7 @@ Status TwoDCOutputClient::ApplyChanges(const cdc::GetChangesResponsePB* poller_r // then either poll for next set of changes (in case of successful application) or will try to // re-apply. DCHECK(poller_resp->has_checkpoint()); - twodc_resp_copy_.Clear(); + xcluster_resp_copy_.Clear(); // Init class variables that threads will use. { @@ -296,7 +297,7 @@ Status TwoDCOutputClient::ApplyChanges(const cdc::GetChangesResponsePB* poller_r STATUS(TryAgain, "Failing ApplyChanges for transaction status table for test")); } - twodc_resp_copy_ = *poller_resp; + xcluster_resp_copy_ = *poller_resp; timeout_ms_ = MonoDelta::FromMilliseconds(FLAGS_cdc_read_rpc_timeout_ms); // Using this future as a barrier to get all the tablets before processing. Ordered iteration // matters: we need to ensure that each record is handled sequentially. @@ -308,13 +309,13 @@ Status TwoDCOutputClient::ApplyChanges(const cdc::GetChangesResponsePB* poller_r return Status::OK(); } -Status TwoDCOutputClient::ProcessChangesStartingFromIndex(int start) { +Status XClusterOutputClient::ProcessChangesStartingFromIndex(int start) { bool processed_write_record = false; - auto records_size = twodc_resp_copy_.records_size(); + auto records_size = xcluster_resp_copy_.records_size(); for (int i = start; i < records_size; i++) { // All KV-pairs within a single CDC record will be for the same row. // key(0).key() will contain the hash code for that row. We use this to lookup the tablet. - const auto& record = twodc_resp_copy_.records(i); + const auto& record = xcluster_resp_copy_.records(i); if (IsValidMetaOp(record)) { if (processed_write_record) { @@ -365,21 +366,21 @@ Status TwoDCOutputClient::ProcessChangesStartingFromIndex(int start) { return Status::OK(); } -Result TwoDCOutputClient::GetTargetTabletIdFromProducerTablet( +Result XClusterOutputClient::GetTargetTabletIdFromProducerTablet( const TabletId& producer_tablet_id) { - auto consumer_tablet_info = VERIFY_RESULT( - cdc_consumer_->GetConsumerTableInfo(producer_tablet_id)); + auto consumer_tablet_info = + VERIFY_RESULT(xcluster_consumer_->GetConsumerTableInfo(producer_tablet_id)); return consumer_tablet_info.tablet_id; } -Result> TwoDCOutputClient::GetInvolvedTargetTabletsFromCommitRecord( +Result> XClusterOutputClient::GetInvolvedTargetTabletsFromCommitRecord( const cdc::CDCRecordPB& record) { std::unordered_set involved_target_tablets; auto involved_producer_tablets = std::vector( record.transaction_state().tablets().begin(), record.transaction_state().tablets().end()); for (const auto& producer_tablet : involved_producer_tablets) { - auto consumer_tablet_info = cdc_consumer_->GetConsumerTableInfo(producer_tablet); + auto consumer_tablet_info = xcluster_consumer_->GetConsumerTableInfo(producer_tablet); // Ignore records for which we dont have any consumer tablets. if (!consumer_tablet_info.ok()) { if (consumer_tablet_info.status().IsNotFound()) { @@ -398,7 +399,7 @@ Result> TwoDCOutputClient::GetInvolvedTargetTabletsFromCom return std::vector(involved_target_tablets.begin(), involved_target_tablets.end()); } -Status TwoDCOutputClient::SendTransactionUpdates() { +Status XClusterOutputClient::SendTransactionUpdates() { std::vector transaction_metadatas; { std::lock_guard l(lock_); @@ -415,7 +416,7 @@ Status TwoDCOutputClient::SendTransactionUpdates() { continue; } // Create the ExternalTransactions and COMMIT them locally. - auto* transaction_manager = cdc_consumer_->TransactionManager(); + auto* transaction_manager = xcluster_consumer_->TransactionManager(); if (!transaction_manager) { return STATUS(InvalidArgument, "Could not commit transactions"); } @@ -471,7 +472,7 @@ Status TwoDCOutputClient::SendTransactionUpdates() { return Status::OK(); } -Status TwoDCOutputClient::SendUserTableWrites() { +Status XClusterOutputClient::SendUserTableWrites() { // Send out the buffered writes. std::unique_ptr write_request; { @@ -486,18 +487,17 @@ Status TwoDCOutputClient::SendUserTableWrites() { return Status::OK(); } -bool TwoDCOutputClient::UseLocalTserver() { +bool XClusterOutputClient::UseLocalTserver() { return use_local_tserver_ && !FLAGS_cdc_force_remote_tserver; } -Status TwoDCOutputClient::ProcessCreateRecord( - const std::string& status_tablet, - const cdc::CDCRecordPB& record) { +Status XClusterOutputClient::ProcessCreateRecord( + const std::string& status_tablet, const cdc::CDCRecordPB& record) { std::lock_guard l(lock_); return write_strategy_->ProcessCreateRecord(status_tablet, record); } -Status TwoDCOutputClient::ProcessCommitRecord( +Status XClusterOutputClient::ProcessCommitRecord( const std::string& status_tablet, const std::vector& involved_target_tablet_ids, const cdc::CDCRecordPB& record) { @@ -505,8 +505,8 @@ Status TwoDCOutputClient::ProcessCommitRecord( return write_strategy_->ProcessCommitRecord(status_tablet, involved_target_tablet_ids, record); } -Status TwoDCOutputClient::ProcessRecord(const std::vector& tablet_ids, - const cdc::CDCRecordPB& record) { +Status XClusterOutputClient::ProcessRecord( + const std::vector& tablet_ids, const cdc::CDCRecordPB& record) { std::lock_guard l(lock_); for (const auto& tablet_id : tablet_ids) { std::string status_tablet_id; @@ -535,13 +535,13 @@ Status TwoDCOutputClient::ProcessRecord(const std::vector& tablet_i return Status::OK(); } -Status TwoDCOutputClient::ProcessRecordForTablet( +Status XClusterOutputClient::ProcessRecordForTablet( const cdc::CDCRecordPB& record, const Result& tablet) { RETURN_NOT_OK(tablet); return ProcessRecord({tablet->get()->tablet_id()}, record); } -Status TwoDCOutputClient::ProcessRecordForTabletRange( +Status XClusterOutputClient::ProcessRecordForTabletRange( const cdc::CDCRecordPB& record, const Result>& tablets) { RETURN_NOT_OK(tablets); @@ -558,7 +558,7 @@ Status TwoDCOutputClient::ProcessRecordForTabletRange( return ProcessRecord(tablet_ids, record); } -Status TwoDCOutputClient::ProcessRecordForLocalTablet(const cdc::CDCRecordPB& record) { +Status XClusterOutputClient::ProcessRecordForLocalTablet(const cdc::CDCRecordPB& record) { const auto& operation = record.operation(); if (operation == cdc::CDCRecordPB::TRANSACTION_COMMITTED) { auto target_tablet_ids = VERIFY_RESULT(GetInvolvedTargetTabletsFromCommitRecord(record)); @@ -572,12 +572,12 @@ Status TwoDCOutputClient::ProcessRecordForLocalTablet(const cdc::CDCRecordPB& re return ProcessRecord({consumer_tablet_info_.tablet_id}, record); } -bool TwoDCOutputClient::IsValidMetaOp(const cdc::CDCRecordPB& record) { +bool XClusterOutputClient::IsValidMetaOp(const cdc::CDCRecordPB& record) { auto type = record.operation(); return type == cdc::CDCRecordPB::SPLIT_OP || type == cdc::CDCRecordPB::CHANGE_METADATA; } -Result TwoDCOutputClient::ProcessMetaOp(const cdc::CDCRecordPB& record) { +Result XClusterOutputClient::ProcessMetaOp(const cdc::CDCRecordPB& record) { uint32_t wait_for_version = 0; uint32_t last_compatible_consumer_schema_version = cdc::kInvalidSchemaVersion; if (record.operation() == cdc::CDCRecordPB::SPLIT_OP) { @@ -623,7 +623,7 @@ Result TwoDCOutputClient::ProcessMetaOp(const cdc::CDCRecordPB& record) { return done; } -void TwoDCOutputClient::SendNextCDCWriteToTablet(std::unique_ptr write_request) { +void XClusterOutputClient::SendNextCDCWriteToTablet(std::unique_ptr write_request) { // TODO: This should be parallelized for better performance with M:N setups. auto deadline = CoarseMonoClock::Now() + MonoDelta::FromMilliseconds(FLAGS_cdc_write_rpc_timeout_ms); @@ -638,7 +638,7 @@ void TwoDCOutputClient::SendNextCDCWriteToTablet(std::unique_ptr table_, local_client_->client.get(), write_request.get(), - std::bind(&TwoDCOutputClient::WriteCDCRecordDone, SharedFromThis(), _1, _2), + std::bind(&XClusterOutputClient::WriteCDCRecordDone, SharedFromThis(), _1, _2), UseLocalTserver()); (**write_handle_).SendRpc(); } else { @@ -646,7 +646,8 @@ void TwoDCOutputClient::SendNextCDCWriteToTablet(std::unique_ptr } } -void TwoDCOutputClient::WriteCDCRecordDone(const Status& status, const WriteResponsePB& response) { +void XClusterOutputClient::WriteCDCRecordDone( + const Status& status, const WriteResponsePB& response) { rpc::RpcCommandPtr retained; { std::lock_guard l(lock_); @@ -656,11 +657,12 @@ void TwoDCOutputClient::WriteCDCRecordDone(const Status& status, const WriteResp WARN_NOT_OK( thread_pool_->SubmitFunc(std::bind( - &TwoDCOutputClient::DoWriteCDCRecordDone, SharedFromThis(), status, std::move(response))), + &XClusterOutputClient::DoWriteCDCRecordDone, SharedFromThis(), status, + std::move(response))), "Could not submit DoWriteCDCRecordDone to thread pool"); } -void TwoDCOutputClient::DoWriteCDCRecordDone( +void XClusterOutputClient::DoWriteCDCRecordDone( const Status& status, const WriteResponsePB& response) { RETURN_WHEN_OFFLINE(); @@ -671,7 +673,7 @@ void TwoDCOutputClient::DoWriteCDCRecordDone( HandleError(StatusFromPB(response.error().status())); return; } - cdc_consumer_->IncrementNumSuccessfulWriteRpcs(); + xcluster_consumer_->IncrementNumSuccessfulWriteRpcs(); // See if we need to handle any more writes. std::unique_ptr write_request; @@ -705,7 +707,7 @@ void TwoDCOutputClient::DoWriteCDCRecordDone( } } -void TwoDCOutputClient::HandleError(const Status& s) { +void XClusterOutputClient::HandleError(const Status& s) { if (s.IsTryAgain()) { LOG(WARNING) << "Retrying applying replicated record for consumer tablet: " << consumer_tablet_info_.tablet_id << ", reason: " << s; @@ -725,8 +727,8 @@ void TwoDCOutputClient::HandleError(const Status& s) { HandleResponse(); } -cdc::OutputClientResponse TwoDCOutputClient::PrepareResponse() { - cdc::OutputClientResponse response; +XClusterOutputClientResponse XClusterOutputClient::PrepareResponse() { + XClusterOutputClientResponse response; response.status = error_status_; if (response.status.ok()) { response.last_applied_op_id = op_id_; @@ -738,14 +740,14 @@ cdc::OutputClientResponse TwoDCOutputClient::PrepareResponse() { return response; } -void TwoDCOutputClient::SendResponse(const cdc::OutputClientResponse& resp) { +void XClusterOutputClient::SendResponse(const XClusterOutputClientResponse& resp) { // If we're shutting down, then don't try to call the callback as that object might be deleted. RETURN_WHEN_OFFLINE(); apply_changes_clbk_(resp); } -void TwoDCOutputClient::HandleResponse() { - cdc::OutputClientResponse response; +void XClusterOutputClient::HandleResponse() { + XClusterOutputClientResponse response; { std::lock_guard l(lock_); response = PrepareResponse(); @@ -753,7 +755,7 @@ void TwoDCOutputClient::HandleResponse() { SendResponse(response); } -bool TwoDCOutputClient::IncProcessedRecordCount() { +bool XClusterOutputClient::IncProcessedRecordCount() { processed_record_count_++; if (processed_record_count_ == record_count_) { done_processing_ = true; @@ -762,20 +764,21 @@ bool TwoDCOutputClient::IncProcessedRecordCount() { return done_processing_; } -std::shared_ptr CreateTwoDCOutputClient( - CDCConsumer* cdc_consumer, +std::shared_ptr CreateXClusterOutputClient( + XClusterConsumer* xcluster_consumer, const cdc::ConsumerTabletInfo& consumer_tablet_info, const cdc::ProducerTabletInfo& producer_tablet_info, - const std::shared_ptr& local_client, + const std::shared_ptr& local_client, ThreadPool* thread_pool, rpc::Rpcs* rpcs, - std::function apply_changes_clbk, + std::function + apply_changes_clbk, bool use_local_tserver, client::YBTablePtr global_transaction_status_table, bool enable_replicate_transaction_status_table) { - return std::make_unique( - cdc_consumer, consumer_tablet_info, producer_tablet_info, local_client, thread_pool, rpcs, - std::move(apply_changes_clbk), use_local_tserver, global_transaction_status_table, + return std::make_unique( + xcluster_consumer, consumer_tablet_info, producer_tablet_info, local_client, thread_pool, + rpcs, std::move(apply_changes_clbk), use_local_tserver, global_transaction_status_table, enable_replicate_transaction_status_table); } diff --git a/src/yb/tserver/twodc_output_client.h b/src/yb/tserver/xcluster_output_client.h similarity index 74% rename from src/yb/tserver/twodc_output_client.h rename to src/yb/tserver/xcluster_output_client.h index c0e9b1f4da65..ab5240c40821 100644 --- a/src/yb/tserver/twodc_output_client.h +++ b/src/yb/tserver/xcluster_output_client.h @@ -10,7 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations // under the License. -#include "yb/cdc/cdc_output_client_interface.h" +#include "yb/tserver/xcluster_output_client_interface.h" #include "yb/cdc/cdc_util.h" #include "yb/client/client_fwd.h" #include "yb/rpc/rpc_fwd.h" @@ -23,17 +23,18 @@ class ThreadPool; namespace tserver { -class CDCConsumer; -struct CDCClient; +class XClusterConsumer; +struct XClusterClient; -std::shared_ptr CreateTwoDCOutputClient( - CDCConsumer* cdc_consumer, +std::shared_ptr CreateXClusterOutputClient( + XClusterConsumer* xcluster_consumer, const cdc::ConsumerTabletInfo& consumer_tablet_info, const cdc::ProducerTabletInfo& producer_tablet_info, - const std::shared_ptr& local_client, + const std::shared_ptr& local_client, ThreadPool* thread_pool, rpc::Rpcs* rpcs, - std::function apply_changes_clbk, + std::function + apply_changes_clbk, bool use_local_tserver, client::YBTablePtr global_transaction_status_table, bool enable_replicate_transaction_status_table); diff --git a/src/yb/cdc/cdc_output_client_interface.h b/src/yb/tserver/xcluster_output_client_interface.h similarity index 82% rename from src/yb/cdc/cdc_output_client_interface.h rename to src/yb/tserver/xcluster_output_client_interface.h index 794d0d3eb847..d9cac0446220 100644 --- a/src/yb/cdc/cdc_output_client_interface.h +++ b/src/yb/tserver/xcluster_output_client_interface.h @@ -26,20 +26,18 @@ namespace client { class YBTableName; -} // namespace client +} // namespace client -namespace cdc { - -struct OutputClientResponse { +struct XClusterOutputClientResponse { Status status; OpIdPB last_applied_op_id; uint32_t processed_record_count; - uint32_t wait_for_version { 0 }; + uint32_t wait_for_version{0}; }; -class CDCOutputClient : public std::enable_shared_from_this { +class XClusterOutputClientIf : public std::enable_shared_from_this { public: - virtual ~CDCOutputClient() {} + virtual ~XClusterOutputClientIf() {} virtual void Shutdown() {} // Sets the last compatible consumer schema version @@ -49,5 +47,4 @@ class CDCOutputClient : public std::enable_shared_from_this { virtual Status ApplyChanges(const cdc::GetChangesResponsePB* resp) = 0; }; -} // namespace cdc -} // namespace yb +} // namespace yb diff --git a/src/yb/tserver/cdc_poller.cc b/src/yb/tserver/xcluster_poller.cc similarity index 79% rename from src/yb/tserver/cdc_poller.cc rename to src/yb/tserver/xcluster_poller.cc index d326a676e59d..e0578d9681b2 100644 --- a/src/yb/tserver/cdc_poller.cc +++ b/src/yb/tserver/xcluster_poller.cc @@ -11,11 +11,11 @@ // under the License. // -#include "yb/tserver/cdc_poller.h" +#include "yb/tserver/xcluster_poller.h" #include "yb/client/client_fwd.h" #include "yb/gutil/strings/split.h" -#include "yb/tserver/cdc_consumer.h" -#include "yb/tserver/twodc_output_client.h" +#include "yb/tserver/xcluster_consumer.h" +#include "yb/tserver/xcluster_output_client.h" #include "yb/cdc/cdc_rpc.h" #include "yb/cdc/cdc_service.pb.h" @@ -66,14 +66,14 @@ using namespace std::placeholders; namespace yb { namespace tserver { -CDCPoller::CDCPoller( +XClusterPoller::XClusterPoller( const cdc::ProducerTabletInfo& producer_tablet_info, const cdc::ConsumerTabletInfo& consumer_tablet_info, ThreadPool* thread_pool, rpc::Rpcs* rpcs, - const std::shared_ptr& local_client, - const std::shared_ptr& producer_client, - CDCConsumer* cdc_consumer, + const std::shared_ptr& local_client, + const std::shared_ptr& producer_client, + XClusterConsumer* xcluster_consumer, bool use_local_tserver, client::YBTablePtr global_transaction_status_table, bool enable_replicate_transaction_status_table, @@ -84,14 +84,14 @@ CDCPoller::CDCPoller( validated_schema_version_(0), last_compatible_consumer_schema_version_(last_compatible_consumer_schema_version), resp_(std::make_unique()), - output_client_(CreateTwoDCOutputClient( - cdc_consumer, + output_client_(CreateXClusterOutputClient( + xcluster_consumer, consumer_tablet_info, producer_tablet_info, local_client, thread_pool, rpcs, - std::bind(&CDCPoller::HandleApplyChanges, this, _1), + std::bind(&XClusterPoller::HandleApplyChanges, this, _1), use_local_tserver, global_transaction_status_table, enable_replicate_transaction_status_table)), @@ -99,19 +99,21 @@ CDCPoller::CDCPoller( thread_pool_(thread_pool), rpcs_(rpcs), poll_handle_(rpcs_->InvalidHandle()), - cdc_consumer_(cdc_consumer), + xcluster_consumer_(xcluster_consumer), producer_safe_time_(HybridTime::kInvalid) {} -CDCPoller::~CDCPoller() { - VLOG(1) << "Destroying CDCPoller"; +XClusterPoller::~XClusterPoller() { + VLOG(1) << "Destroying XClusterPoller"; DCHECK(shutdown_); } -void CDCPoller::Shutdown() { +void XClusterPoller::Shutdown() { // The poller is shutdown in two cases: - // 1. The regular case where the poller is deleted via CDCConsumer's TriggerDeletionOfOldPollers. + // 1. The regular case where the poller is deleted via XClusterConsumer's + // TriggerDeletionOfOldPollers. // This happens when the stream is deleted or the consumer tablet leader changes. - // 2. During CDCConsumer::Shutdown(). Note that in this scenario, we may still be processing a + // 2. During XClusterConsumer::Shutdown(). Note that in this scenario, we may still be processing + // a // GetChanges request / handle callback, so we shutdown what we can here (note that // thread_pool_ is shutdown before we shutdown the pollers, so that will force most // codepaths to exit early), and then using shared_from_this, destroy the object once all @@ -132,7 +134,7 @@ void CDCPoller::Shutdown() { } } -std::string CDCPoller::LogPrefixUnlocked() const { +std::string XClusterPoller::LogPrefixUnlocked() const { return strings::Substitute("P [$0:$1] C [$2:$3]: ", producer_tablet_info_.stream_id, producer_tablet_info_.tablet_id, @@ -140,7 +142,7 @@ std::string CDCPoller::LogPrefixUnlocked() const { consumer_tablet_info_.tablet_id); } -bool CDCPoller::CheckOffline() { return shutdown_.load(); } +bool XClusterPoller::CheckOffline() { return shutdown_.load(); } #define RETURN_WHEN_OFFLINE() \ if (CheckOffline()) { \ @@ -152,7 +154,7 @@ bool CDCPoller::CheckOffline() { return shutdown_.load(); } RETURN_WHEN_OFFLINE(); \ std::lock_guard l(data_mutex_); -void CDCPoller::SetSchemaVersion( +void XClusterPoller::SetSchemaVersion( SchemaVersion cur_version, SchemaVersion last_compatible_consumer_schema_version) { RETURN_WHEN_OFFLINE(); @@ -160,14 +162,14 @@ void CDCPoller::SetSchemaVersion( validated_schema_version_ < cur_version) { WARN_NOT_OK( thread_pool_->SubmitFunc(std::bind( - &CDCPoller::DoSetSchemaVersion, shared_from_this(), cur_version, + &XClusterPoller::DoSetSchemaVersion, shared_from_this(), cur_version, last_compatible_consumer_schema_version)), "Could not submit SetSchemaVersion to thread pool"); } } -void CDCPoller::DoSetSchemaVersion(SchemaVersion cur_version, - SchemaVersion current_consumer_schema_version) { +void XClusterPoller::DoSetSchemaVersion( + SchemaVersion cur_version, SchemaVersion current_consumer_schema_version) { ACQUIRE_MUTEX_IF_ONLINE(); if (last_compatible_consumer_schema_version_ < current_consumer_schema_version) { @@ -182,20 +184,23 @@ void CDCPoller::DoSetSchemaVersion(SchemaVersion cur_version, LOG(INFO) << "Restarting polling on " << producer_tablet_info_.tablet_id << " Producer schema version : " << validated_schema_version_ << " Consumer schema version : " << last_compatible_consumer_schema_version_; - WARN_NOT_OK(thread_pool_->SubmitFunc(std::bind(&CDCPoller::DoPoll, shared_from_this())), - "Could not submit Poll to thread pool"); + WARN_NOT_OK( + thread_pool_->SubmitFunc(std::bind(&XClusterPoller::DoPoll, shared_from_this())), + "Could not submit Poll to thread pool"); } } } -HybridTime CDCPoller::GetSafeTime() const { +HybridTime XClusterPoller::GetSafeTime() const { SharedLock lock(safe_time_lock_); return producer_safe_time_; } -cdc::ConsumerTabletInfo CDCPoller::GetConsumerTabletInfo() const { return consumer_tablet_info_; } +cdc::ConsumerTabletInfo XClusterPoller::GetConsumerTabletInfo() const { + return consumer_tablet_info_; +} -void CDCPoller::UpdateSafeTime(int64 new_time) { +void XClusterPoller::UpdateSafeTime(int64 new_time) { HybridTime new_hybrid_time(new_time); if (!new_hybrid_time.is_special()) { std::lock_guard l(safe_time_lock_); @@ -205,13 +210,14 @@ void CDCPoller::UpdateSafeTime(int64 new_time) { } } -void CDCPoller::Poll() { +void XClusterPoller::Poll() { RETURN_WHEN_OFFLINE(); - WARN_NOT_OK(thread_pool_->SubmitFunc(std::bind(&CDCPoller::DoPoll, shared_from_this())), - "Could not submit Poll to thread pool"); + WARN_NOT_OK( + thread_pool_->SubmitFunc(std::bind(&XClusterPoller::DoPoll, shared_from_this())), + "Could not submit Poll to thread pool"); } -void CDCPoller::DoPoll() { +void XClusterPoller::DoPoll() { ACQUIRE_MUTEX_IF_ONLINE(); if (PREDICT_FALSE(FLAGS_TEST_cdc_skip_replication_poll)) { @@ -284,11 +290,11 @@ void CDCPoller::DoPoll() { nullptr, /* RemoteTablet: will get this from 'req' */ producer_client_->client.get(), &req, - std::bind(&CDCPoller::HandlePoll, shared_from_this(), _1, _2)); + std::bind(&XClusterPoller::HandlePoll, shared_from_this(), _1, _2)); (**poll_handle_).SendRpc(); } -void CDCPoller::HandlePoll(const Status& status, cdc::GetChangesResponsePB&& resp) { +void XClusterPoller::HandlePoll(const Status& status, cdc::GetChangesResponsePB&& resp) { rpc::RpcCommandPtr retained; { std::lock_guard l(data_mutex_); @@ -298,11 +304,11 @@ void CDCPoller::HandlePoll(const Status& status, cdc::GetChangesResponsePB&& res auto new_resp = std::make_shared(std::move(resp)); WARN_NOT_OK( thread_pool_->SubmitFunc( - std::bind(&CDCPoller::DoHandlePoll, shared_from_this(), status, new_resp)), + std::bind(&XClusterPoller::DoHandlePoll, shared_from_this(), status, new_resp)), "Could not submit HandlePoll to thread pool"); } -void CDCPoller::DoHandlePoll(Status status, std::shared_ptr resp) { +void XClusterPoller::DoHandlePoll(Status status, std::shared_ptr resp) { ACQUIRE_MUTEX_IF_ONLINE(); status_ = status; @@ -310,23 +316,23 @@ void CDCPoller::DoHandlePoll(Status status, std::shared_ptrhas_error()) { - LOG_WITH_PREFIX_UNLOCKED(WARNING) << "CDCPoller failure response: code=" - << resp_->error().code() - << ", status=" << resp->error().status().DebugString(); + LOG_WITH_PREFIX_UNLOCKED(WARNING) + << "XClusterPoller failure response: code=" << resp_->error().code() + << ", status=" << resp->error().status().DebugString(); failed = true; if (resp_->error().code() == cdc::CDCErrorPB::CHECKPOINT_TOO_OLD) { - cdc_consumer_->StoreReplicationError( - consumer_tablet_info_.tablet_id, - producer_tablet_info_.stream_id, - ReplicationErrorPb::REPLICATION_MISSING_OP_ID, - "Unable to find expected op id on the producer"); + xcluster_consumer_->StoreReplicationError( + consumer_tablet_info_.tablet_id, + producer_tablet_info_.stream_id, + ReplicationErrorPb::REPLICATION_MISSING_OP_ID, + "Unable to find expected op id on the producer"); } } else if (!resp_->has_checkpoint()) { - LOG_WITH_PREFIX_UNLOCKED(ERROR) << "CDCPoller failure: no checkpoint"; + LOG_WITH_PREFIX_UNLOCKED(ERROR) << "XClusterPoller failure: no checkpoint"; failed = true; } if (failed) { @@ -342,14 +348,15 @@ void CDCPoller::DoHandlePoll(Status status, std::shared_ptrApplyChanges(resp_.get()), "Could not ApplyChanges"); } -void CDCPoller::HandleApplyChanges(cdc::OutputClientResponse response) { +void XClusterPoller::HandleApplyChanges(XClusterOutputClientResponse response) { RETURN_WHEN_OFFLINE(); - WARN_NOT_OK(thread_pool_->SubmitFunc( - std::bind(&CDCPoller::DoHandleApplyChanges, shared_from_this(), response)), - "Could not submit HandleApplyChanges to thread pool"); + WARN_NOT_OK( + thread_pool_->SubmitFunc( + std::bind(&XClusterPoller::DoHandleApplyChanges, shared_from_this(), response)), + "Could not submit HandleApplyChanges to thread pool"); } -void CDCPoller::DoHandleApplyChanges(cdc::OutputClientResponse response) { +void XClusterPoller::DoHandleApplyChanges(XClusterOutputClientResponse response) { ACQUIRE_MUTEX_IF_ONLINE(); if (!response.status.ok()) { diff --git a/src/yb/tserver/cdc_poller.h b/src/yb/tserver/xcluster_poller.h similarity index 83% rename from src/yb/tserver/cdc_poller.h rename to src/yb/tserver/xcluster_poller.h index 082874a7e2d2..dd299e08be66 100644 --- a/src/yb/tserver/cdc_poller.h +++ b/src/yb/tserver/xcluster_poller.h @@ -15,10 +15,10 @@ #include #include "yb/cdc/cdc_util.h" -#include "yb/cdc/cdc_output_client_interface.h" +#include "yb/tserver/xcluster_output_client_interface.h" #include "yb/common/hybrid_time.h" #include "yb/rpc/rpc.h" -#include "yb/tserver/cdc_consumer.h" +#include "yb/tserver/xcluster_consumer.h" #include "yb/tserver/tablet_server.h" #include "yb/util/locks.h" #include "yb/util/status_fwd.h" @@ -43,24 +43,23 @@ class CDCServiceProxy; namespace tserver { -class CDCConsumer; +class XClusterConsumer; - -class CDCPoller : public std::enable_shared_from_this { +class XClusterPoller : public std::enable_shared_from_this { public: - CDCPoller( + XClusterPoller( const cdc::ProducerTabletInfo& producer_tablet_info, const cdc::ConsumerTabletInfo& consumer_tablet_info, ThreadPool* thread_pool, rpc::Rpcs* rpcs, - const std::shared_ptr& local_client, - const std::shared_ptr& producer_client, - CDCConsumer* cdc_consumer, + const std::shared_ptr& local_client, + const std::shared_ptr& producer_client, + XClusterConsumer* xcluster_consumer, bool use_local_tserver, client::YBTablePtr global_transaction_status_table, bool enable_replicate_transaction_status_table, SchemaVersion last_compatible_consumer_schema_version); - ~CDCPoller(); + ~XClusterPoller(); void Shutdown(); @@ -89,9 +88,9 @@ class CDCPoller : public std::enable_shared_from_this { void HandlePoll(const Status& status, cdc::GetChangesResponsePB&& resp); void DoHandlePoll(Status status, std::shared_ptr resp); // Async handler for the response from output client. - void HandleApplyChanges(cdc::OutputClientResponse response); + void HandleApplyChanges(XClusterOutputClientResponse response); // Does the work of polling for new changes. - void DoHandleApplyChanges(cdc::OutputClientResponse response); + void DoHandleApplyChanges(XClusterOutputClientResponse response); void UpdateSafeTime(int64 new_time) EXCLUDES(safe_time_lock_); cdc::ProducerTabletInfo producer_tablet_info_; @@ -112,13 +111,13 @@ class CDCPoller : public std::enable_shared_from_this { Status status_ GUARDED_BY(data_mutex_); std::shared_ptr resp_ GUARDED_BY(data_mutex_); - std::shared_ptr output_client_; - std::shared_ptr producer_client_; + std::shared_ptr output_client_; + std::shared_ptr producer_client_; ThreadPool* thread_pool_; rpc::Rpcs* rpcs_; rpc::Rpcs::Handle poll_handle_ GUARDED_BY(data_mutex_); - CDCConsumer* cdc_consumer_ GUARDED_BY(data_mutex_); + XClusterConsumer* xcluster_consumer_ GUARDED_BY(data_mutex_); mutable rw_spinlock safe_time_lock_; HybridTime producer_safe_time_ GUARDED_BY(safe_time_lock_); diff --git a/src/yb/tserver/twodc_write_implementations.cc b/src/yb/tserver/xcluster_write_implementations.cc similarity index 96% rename from src/yb/tserver/twodc_write_implementations.cc rename to src/yb/tserver/xcluster_write_implementations.cc index cd3d2947018a..cd36133f1d27 100644 --- a/src/yb/tserver/twodc_write_implementations.cc +++ b/src/yb/tserver/xcluster_write_implementations.cc @@ -24,7 +24,7 @@ #include "yb/docdb/packed_row.h" #include "yb/docdb/rocksdb_writer.h" -#include "yb/tserver/twodc_write_interface.h" +#include "yb/tserver/xcluster_write_interface.h" #include "yb/tserver/tserver.pb.h" #include "yb/cdc/cdc_service.pb.h" @@ -44,8 +44,9 @@ DEFINE_RUNTIME_int32(cdc_max_apply_batch_num_records, 0, DEFINE_RUNTIME_uint32(cdc_max_apply_batch_size_bytes, 0, "Max CDC write request batch size in kb. If 0, default to no max batch size."); -DEFINE_test_flag(bool, twodc_write_hybrid_time, false, - "Override external_hybrid_time with initialHybridTimeValue for testing."); +DEFINE_test_flag( + bool, xcluster_write_hybrid_time, false, + "Override external_hybrid_time with initialHybridTimeValue for testing."); DECLARE_uint64(consensus_max_batch_size_bytes); @@ -203,7 +204,7 @@ Status AddRecord(const ProcessRecordInfo& process_record_info, const Slice& updated_value_slice = updated_value.AsSlice(); write_pair->set_value(updated_value_slice.cdata(), updated_value_slice.size()); - if (PREDICT_FALSE(FLAGS_TEST_twodc_write_hybrid_time)) { + if (PREDICT_FALSE(FLAGS_TEST_xcluster_write_hybrid_time)) { // Used only for testing external hybrid time. write_pair->set_external_hybrid_time(yb::kInitialHybridTimeValue); } else { @@ -229,7 +230,7 @@ Status AddRecord(const ProcessRecordInfo& process_record_info, // is cdc_max_apply_batch_size_kb. Batches are not sent by opid order, since a GetChangesResponse // can contain interleaved records to multiple tablets. Rather, we send batches to each tablet // in order for that tablet, before moving on to the next tablet. -class BatchedWriteImplementation : public TwoDCWriteInterface { +class BatchedWriteImplementation : public XClusterWriteInterface { ~BatchedWriteImplementation() = default; Status ProcessRecord( @@ -318,7 +319,7 @@ class BatchedWriteImplementation : public TwoDCWriteInterface { std::vector transaction_metadatas_; }; -void ResetWriteInterface(std::unique_ptr* write_strategy) { +void ResetWriteInterface(std::unique_ptr* write_strategy) { write_strategy->reset(new BatchedWriteImplementation()); } diff --git a/src/yb/tserver/twodc_write_interface.h b/src/yb/tserver/xcluster_write_interface.h similarity index 85% rename from src/yb/tserver/twodc_write_interface.h rename to src/yb/tserver/xcluster_write_interface.h index 5c3016e14d6c..20fc9dc78b33 100644 --- a/src/yb/tserver/twodc_write_interface.h +++ b/src/yb/tserver/xcluster_write_interface.h @@ -38,15 +38,14 @@ struct ProcessRecordInfo { SchemaVersion last_compatible_consumer_schema_version; }; -class TwoDCWriteInterface { +class XClusterWriteInterface { public: - virtual ~TwoDCWriteInterface() {} + virtual ~XClusterWriteInterface() {} virtual std::unique_ptr GetNextWriteRequest() = 0; virtual Status ProcessRecord( const ProcessRecordInfo& process_record_info, const cdc::CDCRecordPB& record) = 0; virtual Status ProcessCreateRecord( - const std::string& status_tablet, - const cdc::CDCRecordPB& record) = 0; + const std::string& status_tablet, const cdc::CDCRecordPB& record) = 0; virtual Status ProcessCommitRecord( const std::string& status_tablet, const std::vector& involved_target_tablet_ids, @@ -54,7 +53,7 @@ class TwoDCWriteInterface { virtual std::vector& GetTransactionMetadatas() = 0; }; -void ResetWriteInterface(std::unique_ptr* write_strategy); +void ResetWriteInterface(std::unique_ptr* write_strategy); -} // namespace tserver -} // namespace yb +} // namespace tserver +} // namespace yb From 1fdabc0b88e0dc70291de65ed39313c2fb6cf886 Mon Sep 17 00:00:00 2001 From: Sergei Politov Date: Wed, 1 Mar 2023 12:09:45 +0300 Subject: [PATCH 26/81] [#16254] DocDB: Pack control flags for YCQL columns of fixed size Summary: All YCQL columns should be interpreted as varlen for row packing. Since they could contain TTL and user timestamp. Test Plan: DocOperationTest.WritePackedNulls Reviewers: rthallam, qhu Reviewed By: qhu Subscribers: ybase, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D23209 --- .../yb/master/restore_sys_catalog_state.cc | 2 +- src/yb/cdc/cdcsdk_producer.cc | 10 +- src/yb/consensus/log-dump.cc | 7 +- src/yb/docdb/cql_operation.cc | 4 +- src/yb/docdb/doc_operation-test.cc | 52 +++-- src/yb/docdb/doc_read_context.cc | 11 +- src/yb/docdb/doc_read_context.h | 7 +- src/yb/docdb/doc_reader.cc | 2 +- src/yb/docdb/doc_write_batch.cc | 2 +- src/yb/docdb/docdb-test.cc | 6 +- src/yb/docdb/docdb_test_util.cc | 9 +- src/yb/docdb/docdb_util.cc | 47 ++++- src/yb/docdb/docdb_util.h | 19 +- src/yb/docdb/docrowwiseiterator-test.cc | 192 ++++++++---------- src/yb/docdb/intent_iterator-test.cc | 30 +-- src/yb/docdb/packed_row-test.cc | 2 +- src/yb/docdb/pgsql_operation.cc | 6 +- src/yb/docdb/randomized_docdb-test.cc | 4 + src/yb/docdb/schema_packing.cc | 17 +- src/yb/docdb/schema_packing.h | 12 +- src/yb/master/sys_catalog.cc | 2 +- src/yb/master/system_tablet.cc | 2 +- src/yb/tablet/tablet-split-test.cc | 5 +- src/yb/tablet/tablet.cc | 2 +- src/yb/tablet/tablet_metadata.cc | 12 +- src/yb/tablet/tablet_metadata.h | 2 +- src/yb/tablet/tablet_snapshots.cc | 3 +- src/yb/tools/bulk_load_docdb_util.h | 1 + src/yb/tools/data-patcher.cc | 4 +- src/yb/tools/sst_dump.cc | 2 +- src/yb/tools/yb-bulk_load.cc | 2 +- src/yb/util/tostring.h | 5 + 32 files changed, 264 insertions(+), 219 deletions(-) diff --git a/ent/src/yb/master/restore_sys_catalog_state.cc b/ent/src/yb/master/restore_sys_catalog_state.cc index 3068afc678b6..3261b933615e 100644 --- a/ent/src/yb/master/restore_sys_catalog_state.cc +++ b/ent/src/yb/master/restore_sys_catalog_state.cc @@ -58,7 +58,7 @@ Status ApplyWriteRequest( docdb::DocWriteBatch* write_batch) { const std::string kLogPrefix = "restored tablet: "; auto doc_read_context = std::make_shared( - kLogPrefix, schema, kSysCatalogSchemaVersion); + kLogPrefix, TableType::YQL_TABLE_TYPE, schema, write_request.schema_version()); docdb::DocOperationApplyData apply_data{ .doc_write_batch = write_batch, .deadline = {}, .read_time = {}, .restart_read_ht = nullptr}; IndexMap index_map; diff --git a/src/yb/cdc/cdcsdk_producer.cc b/src/yb/cdc/cdcsdk_producer.cc index 2070d18b29bc..1756fc0f8d81 100644 --- a/src/yb/cdc/cdcsdk_producer.cc +++ b/src/yb/cdc/cdcsdk_producer.cc @@ -227,7 +227,7 @@ Status PopulateBeforeImage( auto docdb = tablet->doc_db(); const auto log_prefix = tablet->LogPrefix(); - docdb::DocReadContext doc_read_context(log_prefix, schema, schema_version); + docdb::DocReadContext doc_read_context(log_prefix, tablet->table_type(), schema, schema_version); docdb::DocRowwiseIterator iter( schema, doc_read_context, TransactionOperationContext(), docdb, CoarseTimePoint::max() /* deadline */, read_time); @@ -351,7 +351,7 @@ Status PopulateCDCSDKIntentRecord( bool colocated = tablet->metadata()->colocated(); Schema& schema = *old_schema; std::string table_name = tablet->metadata()->table_name(); - SchemaPackingStorage schema_packing_storage; + SchemaPackingStorage schema_packing_storage(tablet->table_type()); schema_packing_storage.AddSchema(schema_version, schema); Slice prev_key; CDCSDKProtoRecordPB proto_record; @@ -461,7 +461,7 @@ Status PopulateCDCSDKIntentRecord( schema = *tablet->metadata()->schema("", colocation_id); schema_version = tablet->metadata()->schema_version("", colocation_id); table_name = tablet->metadata()->table_name("", colocation_id); - schema_packing_storage = SchemaPackingStorage(); + schema_packing_storage = SchemaPackingStorage(tablet->table_type()); schema_packing_storage.AddSchema(schema_version, schema); } @@ -691,7 +691,7 @@ Status PopulateCDCSDKWriteRecord( Schema schema = current_schema; SchemaVersion schema_version = current_schema_version; std::string table_name = tablet_ptr->metadata()->table_name(); - SchemaPackingStorage schema_packing_storage; + SchemaPackingStorage schema_packing_storage(tablet_ptr->table_type()); schema_packing_storage.AddSchema(schema_version, schema); // TODO: This function and PopulateCDCSDKIntentRecord have a lot of code in common. They should // be refactored to use some common row-column iterator. @@ -722,7 +722,7 @@ Status PopulateCDCSDKWriteRecord( schema = *tablet_ptr->metadata()->schema("", colocation_id); schema_version = tablet_ptr->metadata()->schema_version("", colocation_id); table_name = tablet_ptr->metadata()->table_name("", colocation_id); - schema_packing_storage = SchemaPackingStorage(); + schema_packing_storage = SchemaPackingStorage(tablet_ptr->table_type()); schema_packing_storage.AddSchema(schema_version, schema); } diff --git a/src/yb/consensus/log-dump.cc b/src/yb/consensus/log-dump.cc index c4f4da67053f..c383e2eed3f1 100644 --- a/src/yb/consensus/log-dump.cc +++ b/src/yb/consensus/log-dump.cc @@ -177,10 +177,9 @@ Status PrintDecodedWriteRequestPB(const string& indent, cout << indent << indent << indent << indent << "Key: " << formatted_key << endl; } if (kv.has_value()) { - static ::yb::docdb::SchemaPackingStorage schema_packing_storage; - Result formatted_value = - DocDBValueToDebugStr(kv.key(), ::yb::docdb::StorageDbType::kRegular, - kv.value(), schema_packing_storage); + static docdb::SchemaPackingStorage schema_packing_storage(TableType::YQL_TABLE_TYPE); + Result formatted_value = DocDBValueToDebugStr( + kv.key(), ::yb::docdb::StorageDbType::kRegular, kv.value(), schema_packing_storage); cout << indent << indent << indent << indent << "Value: " << formatted_value << endl; } cout << indent << indent << indent << "}" << endl; // write_pairs { diff --git a/src/yb/docdb/cql_operation.cc b/src/yb/docdb/cql_operation.cc index bc5d153720f9..f4495a9fc510 100644 --- a/src/yb/docdb/cql_operation.cc +++ b/src/yb/docdb/cql_operation.cc @@ -556,7 +556,7 @@ Status QLWriteOperation::PopulateStatusRow(const DocOperationApplyData& data, // Check if a duplicate value is inserted into a unique index. Result QLWriteOperation::HasDuplicateUniqueIndexValue(const DocOperationApplyData& data) { VLOG(3) << "Looking for collisions in\n" << docdb::DocDBDebugDumpToStr( - data.doc_write_batch->doc_db(), SchemaPackingStorage()); + data.doc_write_batch->doc_db(), SchemaPackingStorage(TableType::YQL_TABLE_TYPE)); // We need to check backwards only for backfilled entries. bool ret = VERIFY_RESULT(HasDuplicateUniqueIndexValue(data, Direction::kForward)) || @@ -635,7 +635,7 @@ Result QLWriteOperation::HasDuplicateUniqueIndexValue( << " vs New: " << yb::ToString(new_value) << "\nUsed read time as " << yb::ToString(data.read_time); DVLOG(3) << "DocDB is now:\n" << docdb::DocDBDebugDumpToStr( - data.doc_write_batch->doc_db(), SchemaPackingStorage()); + data.doc_write_batch->doc_db(), SchemaPackingStorage(TableType::YQL_TABLE_TYPE)); return true; } } diff --git a/src/yb/docdb/doc_operation-test.cc b/src/yb/docdb/doc_operation-test.cc index 3da32260cf35..b0569f0ebb51 100644 --- a/src/yb/docdb/doc_operation-test.cc +++ b/src/yb/docdb/doc_operation-test.cc @@ -44,6 +44,7 @@ using std::vector; +DECLARE_bool(ycql_enable_packed_row); DECLARE_uint64(rocksdb_max_file_size_for_compaction); DECLARE_int32(rocksdb_level0_slowdown_writes_trigger); DECLARE_int32(rocksdb_level0_stop_writes_trigger); @@ -145,15 +146,14 @@ class DocOperationTest : public DocDBTestBase { SeedRandom(); } - Schema CreateSchema() { + Schema CreateSchema() override { ColumnSchema hash_column_schema("k", INT32, false, true); ColumnSchema column1_schema("c1", INT32, false, false); ColumnSchema column2_schema("c2", INT32, false, false); ColumnSchema column3_schema("c3", INT32, false, false); const vector columns({hash_column_schema, column1_schema, column2_schema, column3_schema}); - Schema schema(columns, CreateColumnIds(columns.size()), 1); - return schema; + return Schema(columns, CreateColumnIds(columns.size()), 1); } void AddPrimaryKeyColumn(yb::QLWriteRequestPB* ql_writereq_pb, int32_t value) { @@ -347,6 +347,8 @@ SubDocKey(DocKey(0x0000, [1], []), [ColumnId(3); HT{ w: 2 }]) -> 4 // Normally, max_file_size_for_compaction is only used for tables with TTL. max_file_size_for_compaction_ = MakeMaxFileSizeFunction(); } + + void TestWriteNulls(); }; TEST_F(DocOperationTest, TestRedisSetKVWithTTL) { @@ -398,7 +400,7 @@ TEST_F(DocOperationTest, TestQLUpdateWithoutTTL) { RunTestQLInsertUpdate(QLWriteRequestPB_QLStmtType_QL_STMT_UPDATE); } -TEST_F(DocOperationTest, TestQLWriteNulls) { +void DocOperationTest::TestWriteNulls() { yb::QLWriteRequestPB ql_writereq_pb; yb::QLResponsePB ql_writeresp_pb; @@ -420,7 +422,10 @@ TEST_F(DocOperationTest, TestQLWriteNulls) { // Write to docdb. WriteQL(ql_writereq_pb, schema, &ql_writeresp_pb); +} +TEST_F(DocOperationTest, TestQLWriteNulls) { + TestWriteNulls(); // Null columns are converted to tombstones. AssertDocDbDebugDumpStrEq(R"#( SubDocKey(DocKey(0x0000, [1], []), [SystemColumnId(0); HT]) -> null @@ -430,6 +435,14 @@ SubDocKey(DocKey(0x0000, [1], []), [ColumnId(3); HT{ w: 3 }]) -> DEL )#"); } +TEST_F(DocOperationTest, WritePackedNulls) { + FLAGS_ycql_enable_packed_row = true; + TestWriteNulls(); + AssertDocDbDebugDumpStrEq(R"#( +SubDocKey(DocKey(0x0000, [1], []), [HT]) -> { 1: DEL 2: DEL 3: DEL } + )#"); +} + TEST_F(DocOperationTest, TestQLReadWriteSimple) { yb::QLWriteRequestPB ql_writereq_pb; yb::QLResponsePB ql_writeresp_pb; @@ -781,14 +794,13 @@ class DocOperationScanTest : public DocOperationTest { } } - void InitSchema(SortingType range_column_sorting) { - range_column_sorting_type_ = range_column_sorting; + Schema CreateSchema() override { ColumnSchema hash_column("k", INT32, false, true); - ColumnSchema range_column("r", INT32, false, false, false, false, 1, range_column_sorting); + ColumnSchema range_column( + "r", INT32, false, false, false, false, 1, range_column_sorting_type_); ColumnSchema value_column("v", INT32, false, false); auto columns = { hash_column, range_column, value_column }; - doc_read_context_ = DocReadContext::TEST_Create( - Schema(columns, CreateColumnIds(columns.size()), 2)); + return Schema(columns, CreateColumnIds(columns.size()), 2); } void InsertRows(const size_t num_rows_per_key, @@ -821,7 +833,7 @@ class DocOperationScanTest : public DocOperationTest { } } WriteQLRow(QLWriteRequestPB_QLStmtType_QL_STMT_INSERT, - doc_read_context_.schema, + doc_read_context().schema, { row_data.k, row_data.r, row_data.v }, 1000, ht, @@ -834,8 +846,10 @@ class DocOperationScanTest : public DocOperationTest { ASSERT_OK(FlushRocksDbAndWait()); } - DumpRocksDBToLog(rocksdb(), docdb::SchemaPackingStorage(), StorageDbType::kRegular); - DumpRocksDBToLog(intents_db(), docdb::SchemaPackingStorage(), StorageDbType::kIntents); + DumpRocksDBToLog( + rocksdb(), doc_read_context().schema_packing_storage, StorageDbType::kRegular); + DumpRocksDBToLog( + intents_db(), doc_read_context().schema_packing_storage, StorageDbType::kIntents); } void PerformScans(const bool is_forward_scan, @@ -893,10 +907,10 @@ class DocOperationScanTest : public DocOperationTest { std::reverse(expected_rows.begin(), expected_rows.end()); } DocQLScanSpec ql_scan_spec( - doc_read_context_.schema, kFixedHashCode, kFixedHashCode, hashed_components, + doc_read_context().schema, kFixedHashCode, kFixedHashCode, hashed_components, &condition, nullptr /* if_ req */, rocksdb::kDefaultQueryId, is_forward_scan); DocRowwiseIterator ql_iter( - doc_read_context_.schema, doc_read_context_, txn_op_context, doc_db(), + doc_read_context().schema, doc_read_context(), txn_op_context, doc_db(), CoarseTimePoint::max() /* deadline */, read_ht); ASSERT_OK(ql_iter.Init(ql_scan_spec)); LOG(INFO) << "Expected rows: " << AsString(expected_rows); @@ -932,13 +946,17 @@ class DocOperationScanTest : public DocOperationTest { virtual void DoTestWithSortingType(SortingType schema_type, bool is_forward_scan, size_t num_rows_per_key) = 0; + void SetSortingType(SortingType value) { + range_column_sorting_type_ = value; + CHECK_EQ(doc_read_context().schema.column(1).sorting_type(), value); + } + constexpr static int32_t kNumKeys = 20; constexpr static uint32_t kMinTime = 500; constexpr static uint32_t kMaxTime = 1500; std::mt19937_64 rng_; SortingType range_column_sorting_type_; - DocReadContext doc_read_context_ = DocReadContext::TEST_Create(Schema()); int32_t h_key_; std::vector rows_; }; @@ -955,7 +973,7 @@ void DocOperationRangeFilterTest::DoTestWithSortingType(SortingType sorting_type const bool is_forward_scan, const size_t num_rows_per_key) { ASSERT_OK(DisableCompactions()); - InitSchema(sorting_type); + SetSortingType(sorting_type); InsertRows(num_rows_per_key); { @@ -999,7 +1017,7 @@ class DocOperationTxnScanTest : public DocOperationScanTest { size_t num_rows_per_key) override { ASSERT_OK(DisableCompactions()); - InitSchema(sorting_type); + SetSortingType(sorting_type); TransactionStatusManagerMock txn_status_manager; SetTransactionIsolationLevel(IsolationLevel::SNAPSHOT_ISOLATION); diff --git a/src/yb/docdb/doc_read_context.cc b/src/yb/docdb/doc_read_context.cc index 2faa02804eca..a832cc4edb54 100644 --- a/src/yb/docdb/doc_read_context.cc +++ b/src/yb/docdb/doc_read_context.cc @@ -17,13 +17,14 @@ namespace yb::docdb { -DocReadContext::DocReadContext(const std::string& log_prefix) - : log_prefix_(log_prefix) { +DocReadContext::DocReadContext(const std::string& log_prefix, TableType table_type) + : schema_packing_storage(table_type), log_prefix_(log_prefix) { } -DocReadContext::DocReadContext(const std::string& log_prefix, const Schema& schema_, - SchemaVersion schema_version) - : schema(schema_), log_prefix_(log_prefix) { +DocReadContext::DocReadContext( + const std::string& log_prefix, TableType table_type, const Schema& schema_, + SchemaVersion schema_version) + : schema(schema_), schema_packing_storage(table_type), log_prefix_(log_prefix) { schema_packing_storage.AddSchema(schema_version, schema_); LOG_IF_WITH_PREFIX(INFO, schema_version != 0) << "DocReadContext, from schema, version: " << schema_version; diff --git a/src/yb/docdb/doc_read_context.h b/src/yb/docdb/doc_read_context.h index 0a993b2fc220..87c53c2a26eb 100644 --- a/src/yb/docdb/doc_read_context.h +++ b/src/yb/docdb/doc_read_context.h @@ -21,10 +21,11 @@ namespace yb { namespace docdb { struct DocReadContext { - explicit DocReadContext(const std::string& log_prefix); + explicit DocReadContext(const std::string& log_prefix, TableType table_type); DocReadContext( - const std::string& log_prefix, const Schema& schema_, SchemaVersion schema_version); + const std::string& log_prefix, TableType table_type, const Schema& schema_, + SchemaVersion schema_version); DocReadContext(const DocReadContext& rhs, const Schema& schema_, SchemaVersion schema_version); @@ -61,7 +62,7 @@ struct DocReadContext { } static DocReadContext TEST_Create(const Schema& schema) { - return DocReadContext("TEST: ", schema, 1); + return DocReadContext("TEST: ", TableType::YQL_TABLE_TYPE, schema, 0); } Schema schema; diff --git a/src/yb/docdb/doc_reader.cc b/src/yb/docdb/doc_reader.cc index 78580cdd7dc8..c25de261a184 100644 --- a/src/yb/docdb/doc_reader.cc +++ b/src/yb/docdb/doc_reader.cc @@ -135,7 +135,7 @@ Result> TEST_GetSubDocument( DOCDB_DEBUG_LOG("GetSubDocument for key $0 @ $1", sub_doc_key.ToDebugHexString(), iter->read_time().ToString()); iter->SeekToLastDocKey(); - SchemaPackingStorage schema_packing_storage; + SchemaPackingStorage schema_packing_storage(TableType::YQL_TABLE_TYPE); DocDBTableReader doc_reader( iter.get(), deadline, projection, TableType::YQL_TABLE_TYPE, schema_packing_storage); RETURN_NOT_OK(doc_reader.UpdateTableTombstoneTime(sub_doc_key)); diff --git a/src/yb/docdb/doc_write_batch.cc b/src/yb/docdb/doc_write_batch.cc index fc8b13ea983c..b9fd1fe419f9 100644 --- a/src/yb/docdb/doc_write_batch.cc +++ b/src/yb/docdb/doc_write_batch.cc @@ -848,7 +848,7 @@ std::string DocWriteBatchFormatter::FormatKey(const Slice& key) { std::string DocWriteBatchFormatter::FormatValue(const Slice& key, const Slice& value) { auto key_type = GetKeyType(key, storage_db_type_); const auto value_result = DocDBValueToDebugStr( - key_type, key, value, SchemaPackingStorage()); + key_type, key, value, SchemaPackingStorage(TableType::YQL_TABLE_TYPE)); if (value_result.ok()) { return *value_result; } diff --git a/src/yb/docdb/docdb-test.cc b/src/yb/docdb/docdb-test.cc index 2b87f8ff4a5b..952545a1c5d4 100644 --- a/src/yb/docdb/docdb-test.cc +++ b/src/yb/docdb/docdb-test.cc @@ -73,9 +73,9 @@ using rocksdb::WriteOptions; using namespace std::literals; +DECLARE_bool(TEST_docdb_sort_weak_intents); DECLARE_bool(use_docdb_aware_bloom_filter); DECLARE_int32(max_nexts_to_avoid_seek); -DECLARE_bool(TEST_docdb_sort_weak_intents); #define ASSERT_DOC_DB_DEBUG_DUMP_STR_EQ(str) ASSERT_NO_FATALS(AssertDocDbDebugDumpStrEq(str)) @@ -122,6 +122,10 @@ class DocDBTest : public DocDBTestBase { ~DocDBTest() override { } + Schema CreateSchema() override { + return Schema(); + } + virtual void GetSubDoc( const KeyBytes& subdoc_key, SubDocument* result, bool* found_result, const TransactionOperationContext& txn_op_context = TransactionOperationContext(), diff --git a/src/yb/docdb/docdb_test_util.cc b/src/yb/docdb/docdb_test_util.cc index 91955e031a84..06165e7346e3 100644 --- a/src/yb/docdb/docdb_test_util.cc +++ b/src/yb/docdb/docdb_test_util.cc @@ -341,7 +341,8 @@ void LogicalRocksDBDebugSnapshot::Capture(rocksdb::DB* rocksdb) { } // Save the DocDB debug dump as a string so we can check that we've properly restored the snapshot // in RestoreTo. - docdb_debug_dump_str = DocDBDebugDumpToStr(rocksdb, SchemaPackingStorage()); + docdb_debug_dump_str = DocDBDebugDumpToStr( + rocksdb, SchemaPackingStorage(TableType::YQL_TABLE_TYPE)); } void LogicalRocksDBDebugSnapshot::RestoreTo(rocksdb::DB *rocksdb) const { @@ -357,7 +358,8 @@ void LogicalRocksDBDebugSnapshot::RestoreTo(rocksdb::DB *rocksdb) const { ASSERT_OK(rocksdb->Put(write_options, kv.first, kv.second)); } ASSERT_OK(FullyCompactDB(rocksdb)); - ASSERT_EQ(docdb_debug_dump_str, DocDBDebugDumpToStr(rocksdb, SchemaPackingStorage())); + ASSERT_EQ(docdb_debug_dump_str, + DocDBDebugDumpToStr(rocksdb, SchemaPackingStorage(TableType::YQL_TABLE_TYPE))); } // ------------------------------------------------------------------------------------------------ @@ -451,7 +453,8 @@ void DocDBLoadGenerator::PerformOperation(bool compact_history) { ASSERT_OK(in_mem_docdb_.SetPrimitive(doc_path, pv)); const auto set_primitive_status = dwb.SetPrimitive(doc_path, value); if (!set_primitive_status.ok()) { - DocDBDebugDump(rocksdb(), std::cerr, SchemaPackingStorage(), StorageDbType::kRegular); + DocDBDebugDump(rocksdb(), std::cerr, SchemaPackingStorage(TableType::YQL_TABLE_TYPE), + StorageDbType::kRegular); LOG(INFO) << "doc_path=" << doc_path.ToString(); } ASSERT_OK(set_primitive_status); diff --git a/src/yb/docdb/docdb_util.cc b/src/yb/docdb/docdb_util.cc index 564bd1ba63d2..3097b26d48d0 100644 --- a/src/yb/docdb/docdb_util.cc +++ b/src/yb/docdb/docdb_util.cc @@ -57,10 +57,18 @@ Status SetValueFromQLBinaryWrapper( ql_value, pg_data_type, enum_oid_label_map, composite_atts_map, cdc_datum_message); } -DocDBRocksDBUtil::DocDBRocksDBUtil() : doc_read_context_(kEmptyLogPrefix, Schema(), 1) {} +DocDBRocksDBUtil::DocDBRocksDBUtil() {} DocDBRocksDBUtil::DocDBRocksDBUtil(InitMarkerBehavior init_marker_behavior) - : doc_read_context_(kEmptyLogPrefix, Schema(), 1), init_marker_behavior_(init_marker_behavior) { + : init_marker_behavior_(init_marker_behavior) { +} + +DocReadContext& DocDBRocksDBUtil::doc_read_context() { + if (!doc_read_context_) { + doc_read_context_ = std::make_shared( + kEmptyLogPrefix, TableType::YQL_TABLE_TYPE, CreateSchema(), 0); + } + return *doc_read_context_; } rocksdb::DB* DocDBRocksDBUtil::rocksdb() { @@ -291,14 +299,14 @@ void DocDBRocksDBUtil::SetHistoryCutoffHybridTime(HybridTime history_cutoff) { } void DocDBRocksDBUtil::SetTableTTL(uint64_t ttl_msec) { - doc_read_context_.schema.SetDefaultTimeToLive(ttl_msec); + doc_read_context().schema.SetDefaultTimeToLive(ttl_msec); retention_policy_->SetTableTTLForTests(MonoDelta::FromMilliseconds(ttl_msec)); } string DocDBRocksDBUtil::DocDBDebugDumpToStr() { - return docdb::DocDBDebugDumpToStr(rocksdb(), SchemaPackingStorage()) + + return docdb::DocDBDebugDumpToStr(rocksdb(), doc_read_context().schema_packing_storage) + docdb::DocDBDebugDumpToStr( - intents_db(), SchemaPackingStorage(), StorageDbType::kIntents); + intents_db(), doc_read_context().schema_packing_storage, StorageDbType::kIntents); } Status DocDBRocksDBUtil::SetPrimitive( @@ -454,9 +462,10 @@ Status DocDBRocksDBUtil::DeleteSubDoc( return WriteToRocksDB(dwb, hybrid_time); } -void DocDBRocksDBUtil::DocDBDebugDumpToConsole(const SchemaPackingStorage& schema_packing_storage) { +void DocDBRocksDBUtil::DocDBDebugDumpToConsole() { DocDBDebugDump( - regular_db_.get(), std::cerr, schema_packing_storage, StorageDbType::kRegular); + regular_db_.get(), std::cerr, doc_read_context().schema_packing_storage, + StorageDbType::kRegular); } Status DocDBRocksDBUtil::FlushRocksDbAndWait() { @@ -479,7 +488,7 @@ Status DocDBRocksDBUtil::ReinitDBOptions() { [this](const std::vector&) { return delete_marker_retention_time_; } , - /* schema_packing_provider= */ nullptr); + this); regular_db_options_.compaction_file_filter_factory = compaction_file_filter_factory_; regular_db_options_.max_file_size_for_compaction = @@ -515,5 +524,27 @@ void DocDBRocksDBUtil::SetInitMarkerBehavior(InitMarkerBehavior init_marker_beha } } +Result DocDBRocksDBUtil::CotablePacking( + const Uuid& table_id, uint32_t schema_version, HybridTime history_cutoff) { + if (schema_version == docdb::kLatestSchemaVersion) { + schema_version = 0; + } + auto& packing = VERIFY_RESULT_REF( + doc_read_context().schema_packing_storage.GetPacking(schema_version)); + return docdb::CompactionSchemaInfo { + .table_type = TableType::YQL_TABLE_TYPE, + .schema_version = schema_version, + .schema_packing = rpc::SharedField(doc_read_context_, &packing), + .cotable_id = table_id, + .deleted_cols = {}, + .enabled = docdb::PackedRowEnabled(TableType::YQL_TABLE_TYPE, false) + }; +} + +Result DocDBRocksDBUtil::ColocationPacking( + ColocationId colocation_id, uint32_t schema_version, HybridTime history_cutoff) { + return CotablePacking(Uuid::Nil(), schema_version, history_cutoff); +} + } // namespace docdb } // namespace yb diff --git a/src/yb/docdb/docdb_util.h b/src/yb/docdb/docdb_util.h index 81887787c3b7..4f6754f3f2be 100644 --- a/src/yb/docdb/docdb_util.h +++ b/src/yb/docdb/docdb_util.h @@ -47,8 +47,7 @@ struct ExternalIntent { // compacting the history until a certain point. This is used in the bulk load tool. This is also // convenient base class for GTest test classes, because it exposes member functions such as // rocksdb() and write_options(). -class DocDBRocksDBUtil { - +class DocDBRocksDBUtil : public SchemaPackingProvider { public: DocDBRocksDBUtil(); explicit DocDBRocksDBUtil(InitMarkerBehavior init_marker_behavior); @@ -60,6 +59,7 @@ class DocDBRocksDBUtil { // function BlockCacheSize. virtual Status InitRocksDBOptions() = 0; virtual std::string tablet_id() = 0; + virtual Schema CreateSchema() = 0; // Size of block cache for RocksDB, 0 means don't use block cache. virtual size_t block_cache_size() const { return 16 * 1024 * 1024; } @@ -179,8 +179,7 @@ class DocDBRocksDBUtil { HybridTime hybrid_time, const ReadHybridTime& read_ht = ReadHybridTime::Max()); - void DocDBDebugDumpToConsole( - const SchemaPackingStorage& schema_packing_storage = SchemaPackingStorage()); + void DocDBDebugDumpToConsole(); Status FlushRocksDbAndWait(); @@ -219,6 +218,8 @@ class DocDBRocksDBUtil { // Could be used when single DocWriteBatch is enough. DocWriteBatch& DefaultDocWriteBatch(); + DocReadContext& doc_read_context(); + protected: std::string IntentsDBDir(); @@ -239,7 +240,6 @@ class DocDBRocksDBUtil { std::shared_ptr> max_file_size_for_compaction_; rocksdb::WriteOptions write_options_; - DocReadContext doc_read_context_; boost::optional current_txn_id_; mutable IntraTxnWriteId intra_txn_write_id_ = 0; IsolationLevel txn_isolation_level_ = IsolationLevel::NON_TRANSACTIONAL; @@ -247,8 +247,15 @@ class DocDBRocksDBUtil { HybridTime delete_marker_retention_time_ = HybridTime::kMax; private: + Result CotablePacking( + const Uuid& table_id, uint32_t schema_version, HybridTime history_cutoff) override; + + Result ColocationPacking( + ColocationId colocation_id, uint32_t schema_version, HybridTime history_cutoff) override; + std::atomic monotonic_counter_{0}; - boost::optional doc_write_batch_; + std::optional doc_write_batch_; + std::shared_ptr doc_read_context_; }; } // namespace docdb diff --git a/src/yb/docdb/docrowwiseiterator-test.cc b/src/yb/docdb/docrowwiseiterator-test.cc index 0b8f7ad426af..ca100c45c24f 100644 --- a/src/yb/docdb/docrowwiseiterator-test.cc +++ b/src/yb/docdb/docrowwiseiterator-test.cc @@ -54,19 +54,22 @@ class DocRowwiseIteratorTest : public DocDBTestBase { // TODO Could define them out of class, so one line would be enough for them. static const KeyBytes kEncodedDocKey1; static const KeyBytes kEncodedDocKey2; - static const Schema kSchemaForIteratorTests; - static Schema kProjectionForIteratorTests; void SetUp() override { FLAGS_TEST_docdb_sort_weak_intents = true; DocDBTestBase::SetUp(); } - static void SetUpTestCase() { - ASSERT_OK(kSchemaForIteratorTests.CreateProjectionByNames({"c", "d", "e"}, - &kProjectionForIteratorTests)); + const Schema& projection() { + if (!projection_) { + projection_.emplace(); + CHECK_OK(doc_read_context().schema.CreateProjectionByNames({"c", "d", "e"}, &*projection_)); + } + return *projection_; } + Schema CreateSchema() override; + void InsertPopulationData(); void InsertTestRangeData(); @@ -157,6 +160,8 @@ class DocRowwiseIteratorTest : public DocDBTestBase { // Restore doesn't use delete tombstones for rows, instead marks all columns // as deleted. void TestDeletedDocumentUsingLivenessColumnDelete(); + + std::optional projection_; }; const std::string kStrKey1 = "row1"; @@ -170,7 +175,8 @@ const KeyBytes DocRowwiseIteratorTest::kEncodedDocKey1( const KeyBytes DocRowwiseIteratorTest::kEncodedDocKey2( DocKey(KeyEntryValues(kStrKey2, kIntKey2)).Encode()); -const Schema DocRowwiseIteratorTest::kSchemaForIteratorTests({ +Schema DocRowwiseIteratorTest::CreateSchema() { + return Schema({ ColumnSchema("a", DataType::STRING, /* is_nullable = */ false), ColumnSchema("b", DataType::INT64, false), // Non-key columns @@ -184,8 +190,7 @@ const Schema DocRowwiseIteratorTest::kSchemaForIteratorTests({ 40_ColId, 50_ColId }, 2); - -Schema DocRowwiseIteratorTest::kProjectionForIteratorTests; +} constexpr int32_t kFixedHashCode = 0; @@ -971,15 +976,13 @@ void DocRowwiseIteratorTest::SetupDocRowwiseIteratorData() { void DocRowwiseIteratorTest::TestDocRowwiseIterator() { SetupDocRowwiseIteratorData(); - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; + const auto& projection = this->projection(); QLTableRow row; QLValue value; - auto doc_read_context = DocReadContext::TEST_Create(schema); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); @@ -1018,7 +1021,7 @@ void DocRowwiseIteratorTest::TestDocRowwiseIterator() { { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(5000))); ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); @@ -1061,14 +1064,12 @@ void DocRowwiseIteratorTest::TestDocRowwiseIterator() { void DocRowwiseIteratorTest::TestDocRowwiseIteratorCallbackAPI() { SetupDocRowwiseIteratorData(); - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; QLValue value; - auto doc_read_context = DocReadContext::TEST_Create(schema); + const auto& projection = this->projection(); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); size_t row_idx = 0; @@ -1116,7 +1117,7 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorCallbackAPI() { { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(5000))); size_t row_idx = 0; @@ -1165,7 +1166,7 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorCallbackAPI() { // Validate the callback function result. { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(5000))); YQLScanCallback callback = [&](const QLTableRow& row) -> Result { @@ -1178,7 +1179,7 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorCallbackAPI() { { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(5000))); size_t row_idx = 0; @@ -1219,13 +1220,11 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorDeletedDocument() { SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(40); HT{ physical: 2000 }]) -> 20000 )#"); - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; - auto doc_read_context = DocReadContext::TEST_Create(schema); + const auto& projection = this->projection(); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2500))); QLTableRow row; @@ -1276,13 +1275,11 @@ SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 2800 }]) -> SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(40); HT{ physical: 2800 w: 1 }]) -> 20000 )#"); - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; - auto doc_read_context = DocReadContext::TEST_Create(schema); + const auto& projection = this->projection(); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); QLTableRow row; @@ -1490,14 +1487,11 @@ SubDocKey(DocKey([], ["row1", 11111]), [HT{ physical: 2500 }]) -> DEL SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(40); HT{ physical: 1000 }]) -> 10000 SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 2800 }]) -> "row1_e" )#"); - - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; - auto doc_read_context = DocReadContext::TEST_Create(schema); + const auto& projection = this->projection(); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); QLTableRow row; @@ -1539,14 +1533,12 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorIncompleteProjection() { SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(40); HT{ physical: 1000 w: 2 }]) -> 20000 )#"); - const Schema &schema = kSchemaForIteratorTests; Schema projection; - ASSERT_OK(kSchemaForIteratorTests.CreateProjectionByNames({"c", "d"}, &projection)); - auto doc_read_context = DocReadContext::TEST_Create(schema); + ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"c", "d"}, &projection)); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); QLTableRow row; @@ -1600,7 +1592,7 @@ SubDocKey(DocKey(ColocationId=16385, [], []), [HT{ physical: 2000 }]) -> DEL SubDocKey(DocKey(ColocationId=16385, [], ["row1", 11111]), [SystemColumnId(0); \ HT{ physical: 1000 }]) -> null )#"); - Schema schema_copy = kSchemaForIteratorTests; + Schema schema_copy = doc_read_context().schema; schema_copy.set_colocation_id(colocation_id); Schema projection; auto doc_read_context = DocReadContext::TEST_Create(schema_copy); @@ -1672,14 +1664,12 @@ SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(50); HT{ physical: 2800 w: 3 }] "row2_e"; ttl: 0.003s )#"); - const Schema &schema = kSchemaForIteratorTests; Schema projection; - ASSERT_OK(kSchemaForIteratorTests.CreateProjectionByNames({"c", "e"}, &projection)); - auto doc_read_context = DocReadContext::TEST_Create(schema); + ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"c", "e"}, &projection)); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, read_time)); QLTableRow row; @@ -1738,14 +1728,12 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorValidColumnNotInProjection() SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(50); HT{ physical: 2000 }]) -> "row2_e" )#"); - const Schema &schema = kSchemaForIteratorTests; Schema projection; - ASSERT_OK(kSchemaForIteratorTests.CreateProjectionByNames({"c", "d"}, &projection)); - auto doc_read_context = DocReadContext::TEST_Create(schema); + ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"c", "d"}, &projection)); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); QLTableRow row; @@ -1793,15 +1781,12 @@ SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(40); HT{ physical: 1000 }]) -> SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 1000 w: 1 }]) -> "row1_e" )#"); - const Schema &schema = kSchemaForIteratorTests; Schema projection; - ASSERT_OK(kSchemaForIteratorTests.CreateProjectionByNames({"a", "b"}, - &projection, 2)); - auto doc_read_context = DocReadContext::TEST_Create(schema); + ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"a", "b"}, &projection, 2)); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); QLTableRow row; @@ -1952,17 +1937,15 @@ TXN REV 30303030-3030-3030-3030-303030303032 HT{ physical: 4000 w: 3 } -> \ SubDocKey(DocKey([], ["row2", 22222]), []) [kWeakRead, kWeakWrite] HT{ physical: 4000 w: 3 } )#"); - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; + const auto& projection = this->projection(); const auto txn_context = TransactionOperationContext( TransactionId::GenerateRandom(), &txn_status_manager); - auto doc_read_context = DocReadContext::TEST_Create(schema); LOG(INFO) << "=============================================== ReadTime-2000"; { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, txn_context, doc_db(), CoarseTimePoint::max() /* deadline */, - ReadHybridTime::FromMicros(2000))); + projection, doc_read_context(), txn_context, doc_db(), + CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); QLTableRow row; QLValue value; @@ -2004,8 +1987,8 @@ TXN REV 30303030-3030-3030-3030-303030303032 HT{ physical: 4000 w: 3 } -> \ LOG(INFO) << "=============================================== ReadTime-5000"; { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, txn_context, doc_db(), CoarseTimePoint::max() /* deadline */, - ReadHybridTime::FromMicros(5000))); + projection, doc_read_context(), txn_context, doc_db(), + CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(5000))); QLTableRow row; QLValue value; @@ -2045,8 +2028,8 @@ TXN REV 30303030-3030-3030-3030-303030303032 HT{ physical: 4000 w: 3 } -> \ LOG(INFO) << "=============================================== ReadTime-6000"; { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, txn_context, doc_db(), CoarseTimePoint::max() /* deadline */, - ReadHybridTime::FromMicros(6000))); + projection, doc_read_context(), txn_context, doc_db(), + CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(6000))); QLTableRow row; QLValue value; @@ -2203,11 +2186,10 @@ void DocRowwiseIteratorTest::TestScanWithinTheSameTxn() { LOG(INFO) << "Dump:\n" << DocDBDebugDumpToStr(); const auto txn_context = TransactionOperationContext(*txn, &txn_status_manager); - const Schema &projection = kProjectionForIteratorTests; - auto doc_read_context = DocReadContext::TEST_Create(kSchemaForIteratorTests); + const auto& projection = this->projection(); auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, txn_context, doc_db(), CoarseTimePoint::max() /* deadline */, + projection, doc_read_context(), txn_context, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1000))); QLTableRow row; @@ -2267,15 +2249,13 @@ void DocRowwiseIteratorTest::TestLargeKeys() { DocDBDebugDumpToConsole(); - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; + const auto& projection = this->projection(); QLTableRow row; QLValue value; - auto doc_read_context = DocReadContext::TEST_Create(schema); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); @@ -2298,9 +2278,9 @@ void DocRowwiseIteratorTest::TestLargeKeys() { } void DocRowwiseIteratorTest::TestPackedRow() { - constexpr int kVersion = 1; - const Schema &schema = kSchemaForIteratorTests; - SchemaPacking schema_packing(schema); + constexpr int kVersion = 0; + auto& schema_packing = ASSERT_RESULT( + doc_read_context().schema_packing_storage.GetPacking(kVersion)).get(); { Slice row1_packed_row; @@ -2337,19 +2317,15 @@ void DocRowwiseIteratorTest::TestPackedRow() { HybridTime::FromMicros(1000))); } - SchemaPackingStorage schema_packing_storage; - schema_packing_storage.AddSchema(kVersion, schema); - - DocDBDebugDumpToConsole(schema_packing_storage); + DocDBDebugDumpToConsole(); - const Schema &projection = kProjectionForIteratorTests; + const Schema &projection = this->projection(); QLTableRow row; QLValue value; - auto doc_read_context = DocReadContext::TEST_Create(schema); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); @@ -2428,15 +2404,13 @@ void DocRowwiseIteratorTest::TestDeletedDocumentUsingLivenessColumnDelete() { SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 1000 }]) -> "row1_e" )#"); - const Schema &schema = kSchemaForIteratorTests; - const Schema &projection = kProjectionForIteratorTests; + const auto& projection = this->projection(); QLTableRow row; QLValue value; - auto doc_read_context = DocReadContext::TEST_Create(schema); { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1000), nullptr)); ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); @@ -2460,7 +2434,7 @@ void DocRowwiseIteratorTest::TestDeletedDocumentUsingLivenessColumnDelete() { LOG(INFO) << "Validate one deleted column is removed"; { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1100), nullptr)); ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); @@ -2483,7 +2457,7 @@ void DocRowwiseIteratorTest::TestDeletedDocumentUsingLivenessColumnDelete() { LOG(INFO) << "Validate that row is not visible when liveness column is tombstoned"; { auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context, kNonTransactionalOperationContext, doc_db(), + projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1500), nullptr)); ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); @@ -2491,107 +2465,107 @@ void DocRowwiseIteratorTest::TestDeletedDocumentUsingLivenessColumnDelete() { } TEST_F(DocRowwiseIteratorTest, ClusteredFilterTestRange) { - TestClusteredFilterRange(); + TestClusteredFilterRange(); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterRangeWithTableTombstone) { - TestClusteredFilterRangeWithTableTombstone(); + TestClusteredFilterRangeWithTableTombstone(); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterRangeWithTableTombstoneReverseScan) { - TestClusteredFilterRangeWithTableTombstoneReverseScan(); + TestClusteredFilterRangeWithTableTombstoneReverseScan(); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterHybridScanTest) { - TestClusteredFilterHybridScan(); + TestClusteredFilterHybridScan(); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterSubsetColTest) { - TestClusteredFilterSubsetCol(); + TestClusteredFilterSubsetCol(); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterSubsetColTest2) { - TestClusteredFilterSubsetCol2(); + TestClusteredFilterSubsetCol2(); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterMultiInTest) { - TestClusteredFilterMultiIn(); + TestClusteredFilterMultiIn(); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterEmptyInTest) { - TestClusteredFilterEmptyIn(); + TestClusteredFilterEmptyIn(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorTest) { - TestDocRowwiseIterator(); + TestDocRowwiseIterator(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorTestCallbackAPI) { - TestDocRowwiseIteratorCallbackAPI(); +TestDocRowwiseIteratorCallbackAPI(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorDeletedDocumentTest) { - TestDocRowwiseIteratorDeletedDocument(); + TestDocRowwiseIteratorDeletedDocument(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorTestRowDeletes) { - TestDocRowwiseIteratorWithRowDeletes(); + TestDocRowwiseIteratorWithRowDeletes(); } TEST_F(DocRowwiseIteratorTest, BackfillInsert) { - TestBackfillInsert(); + TestBackfillInsert(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorHasNextIdempotence) { - TestDocRowwiseIteratorHasNextIdempotence(); + TestDocRowwiseIteratorHasNextIdempotence(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorIncompleteProjection) { - TestDocRowwiseIteratorIncompleteProjection(); + TestDocRowwiseIteratorIncompleteProjection(); } TEST_F(DocRowwiseIteratorTest, ColocatedTableTombstoneTest) { - TestColocatedTableTombstone(); + TestColocatedTableTombstone(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorMultipleDeletes) { - TestDocRowwiseIteratorMultipleDeletes(); + TestDocRowwiseIteratorMultipleDeletes(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorValidColumnNotInProjection) { - TestDocRowwiseIteratorValidColumnNotInProjection(); + TestDocRowwiseIteratorValidColumnNotInProjection(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorKeyProjection) { - TestDocRowwiseIteratorKeyProjection(); + TestDocRowwiseIteratorKeyProjection(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorResolveWriteIntents) { - TestDocRowwiseIteratorResolveWriteIntents(); + TestDocRowwiseIteratorResolveWriteIntents(); } TEST_F(DocRowwiseIteratorTest, IntentAwareIteratorSeek) { - TestIntentAwareIteratorSeek(); + TestIntentAwareIteratorSeek(); } TEST_F(DocRowwiseIteratorTest, SeekTwiceWithinTheSameTxn) { - TestSeekTwiceWithinTheSameTxn(); + TestSeekTwiceWithinTheSameTxn(); } TEST_F(DocRowwiseIteratorTest, ScanWithinTheSameTxn) { - TestScanWithinTheSameTxn(); + TestScanWithinTheSameTxn(); } TEST_F(DocRowwiseIteratorTest, LargeKeysTest) { - TestLargeKeys(); + TestLargeKeys(); } TEST_F(DocRowwiseIteratorTest, BasicPackedRowTest) { - TestPackedRow(); + TestPackedRow(); } TEST_F(DocRowwiseIteratorTest, DeletedDocumentUsingLivenessColumnDeleteTest) { - TestDeletedDocumentUsingLivenessColumnDelete(); + TestDeletedDocumentUsingLivenessColumnDelete(); } } // namespace docdb diff --git a/src/yb/docdb/intent_iterator-test.cc b/src/yb/docdb/intent_iterator-test.cc index 27454dcff114..c43966ea5c5a 100644 --- a/src/yb/docdb/intent_iterator-test.cc +++ b/src/yb/docdb/intent_iterator-test.cc @@ -19,7 +19,6 @@ #include "yb/common/transaction-test-util.h" #include "yb/docdb/doc_key.h" -#include "yb/docdb/doc_read_context.h" #include "yb/docdb/docdb_test_base.h" #include "yb/docdb/intent_iterator.h" @@ -37,7 +36,15 @@ class IntentIteratorTest : public DocDBTestBase { DocDBTestBase::SetUp(); } - static void SetUpTestCase(); + Schema CreateSchema() override { + return Schema( + {ColumnSchema("a", DataType::STRING, /* is_nullable = */ false), + ColumnSchema("b", DataType::INT64, false), + // Non-key columns + ColumnSchema("c", DataType::STRING, true), ColumnSchema("d", DataType::INT64, true), + ColumnSchema("e", DataType::STRING, true)}, + {10_ColId, 20_ColId, 30_ColId, 40_ColId, 50_ColId}, 2); + } }; const std::string kStrKey1 = "row1"; @@ -51,21 +58,6 @@ static const KeyBytes kEncodedDocKey1( static const KeyBytes kEncodedDocKey2( DocKey(KeyEntryValues(kStrKey2, kIntKey2)).Encode()); -static const Schema kSchemaForIteratorTests( - {ColumnSchema("a", DataType::STRING, /* is_nullable = */ false), - ColumnSchema("b", DataType::INT64, false), - // Non-key columns - ColumnSchema("c", DataType::STRING, true), ColumnSchema("d", DataType::INT64, true), - ColumnSchema("e", DataType::STRING, true)}, - {10_ColId, 20_ColId, 30_ColId, 40_ColId, 50_ColId}, 2); - -static Schema kProjectionForIteratorTests; - -void IntentIteratorTest::SetUpTestCase() { - ASSERT_OK(kSchemaForIteratorTests.CreateProjectionByNames( - {"c", "d", "e"}, &kProjectionForIteratorTests)); -} - void ValidateKeyAndValue( const IntentIterator& iter, const KeyBytes& expected_key_bytes, const KeyEntryValue& expected_key_entry_value, HybridTime expected_ht, @@ -147,7 +139,6 @@ SubDocKey(DocKey([], ["row1", 11111]), []) [kWeakRead, kWeakWrite] HT{ physical: )#"); const auto txn_context = TransactionOperationContext(*txn, &txn_status_manager); - auto doc_read_context = DocReadContext::TEST_Create(kSchemaForIteratorTests); { IntentIterator iter( @@ -233,7 +224,6 @@ SubDocKey(DocKey([], ["row1", 11111]), []) [kWeakRead, kWeakWrite] HT{ physical: )#"); const auto txn_context = TransactionOperationContext(*txn, &txn_status_manager); - auto doc_read_context = DocReadContext::TEST_Create(kSchemaForIteratorTests); { IntentIterator iter( @@ -381,10 +371,8 @@ TXN REV 30303030-3030-3030-3030-303030303032 HT{ physical: 4000 w: 3 } -> \ SubDocKey(DocKey([], ["row2", 22222]), []) [kWeakRead, kWeakWrite] HT{ physical: 4000 w: 3 } )#"); - const Schema& schema = kSchemaForIteratorTests; const auto txn_context = TransactionOperationContext(TransactionId::GenerateRandom(), &txn_status_manager); - auto doc_read_context = DocReadContext::TEST_Create(schema); // No committed intents as of HT 2000. { diff --git a/src/yb/docdb/packed_row-test.cc b/src/yb/docdb/packed_row-test.cc index db2766d4c860..bf89e375c675 100644 --- a/src/yb/docdb/packed_row-test.cc +++ b/src/yb/docdb/packed_row-test.cc @@ -44,7 +44,7 @@ QLValuePB RandomQLValue(DataType type) { void TestRowPacking(const Schema& schema, const std::vector& values) { ASSERT_EQ(schema.num_columns() - schema.num_key_columns(), values.size()); constexpr int kVersion = 1; - SchemaPacking schema_packing(schema); + SchemaPacking schema_packing(TableType::PGSQL_TABLE_TYPE, schema); RowPacker packer( kVersion, schema_packing, /* packed_size_limit= */ std::numeric_limits::max(), /* value_control_fields= */ Slice()); diff --git a/src/yb/docdb/pgsql_operation.cc b/src/yb/docdb/pgsql_operation.cc index 42f3ade2e2e8..63cd6f464277 100644 --- a/src/yb/docdb/pgsql_operation.cc +++ b/src/yb/docdb/pgsql_operation.cc @@ -391,7 +391,7 @@ Status PgsqlWriteOperation::Init(PgsqlResponsePB* response) { // Check if a duplicate value is inserted into a unique index. Result PgsqlWriteOperation::HasDuplicateUniqueIndexValue(const DocOperationApplyData& data) { VLOG(3) << "Looking for collisions in\n" << docdb::DocDBDebugDumpToStr( - data.doc_write_batch->doc_db(), SchemaPackingStorage()); + data.doc_write_batch->doc_db(), SchemaPackingStorage(TableType::PGSQL_TABLE_TYPE)); // We need to check backwards only for backfilled entries. bool ret = VERIFY_RESULT(HasDuplicateUniqueIndexValue(data, Direction::kForward)) || @@ -480,12 +480,12 @@ Result PgsqlWriteOperation::HasDuplicateUniqueIndexValue( << " vs New: " << yb::ToString(new_value) << "\nUsed read time as " << yb::ToString(data.read_time); DVLOG(3) << "DocDB is now:\n" << docdb::DocDBDebugDumpToStr( - data.doc_write_batch->doc_db(), SchemaPackingStorage()); + data.doc_write_batch->doc_db(), SchemaPackingStorage(TableType::PGSQL_TABLE_TYPE)); return true; } } - VLOG(2) << "No collision while checking at " << yb::ToString(read_time); + VLOG(2) << "No collision while checking at " << AsString(read_time); return false; } diff --git a/src/yb/docdb/randomized_docdb-test.cc b/src/yb/docdb/randomized_docdb-test.cc index d6474d48e9aa..8bf2b9e7ade5 100644 --- a/src/yb/docdb/randomized_docdb-test.cc +++ b/src/yb/docdb/randomized_docdb-test.cc @@ -82,6 +82,10 @@ class RandomizedDocDBTest : public DocDBTestBase, SeedRandom(); } + Schema CreateSchema() override { + return Schema(); + } + ~RandomizedDocDBTest() override {} void RunWorkloadWithSnaphots(bool enable_history_cleanup); diff --git a/src/yb/docdb/schema_packing.cc b/src/yb/docdb/schema_packing.cc index ea84c176e54d..c28a4cb45b48 100644 --- a/src/yb/docdb/schema_packing.cc +++ b/src/yb/docdb/schema_packing.cc @@ -32,8 +32,10 @@ namespace { // Used to mark column as skipped by packer. For instance in case of collection column. constexpr int64_t kSkippedColumnIdx = -1; -bool IsVarlenColumn(const ColumnSchema& column_schema) { - return column_schema.is_nullable() || column_schema.type_info()->var_length(); +bool IsVarlenColumn(TableType table_type, const ColumnSchema& column_schema) { + // CQL columns could have individual TTL. + return table_type == TableType::YQL_TABLE_TYPE || + column_schema.is_nullable() || column_schema.type_info()->var_length(); } size_t EncodedColumnSize(const ColumnSchema& column_schema) { @@ -69,7 +71,7 @@ std::string ColumnPackingData::ToString() const { id, num_varlen_columns_before, offset_after_prev_varlen_column, size, nullable); } -SchemaPacking::SchemaPacking(const Schema& schema) { +SchemaPacking::SchemaPacking(TableType table_type, const Schema& schema) { varlen_columns_count_ = 0; size_t offset_after_prev_varlen_column = 0; columns_.reserve(schema.num_columns() - schema.num_key_columns()); @@ -81,7 +83,7 @@ SchemaPacking::SchemaPacking(const Schema& schema) { continue; } column_to_idx_.emplace(column_id, columns_.size()); - bool varlen = IsVarlenColumn(column_schema); + bool varlen = IsVarlenColumn(table_type, column_schema); columns_.emplace_back(ColumnPackingData { .id = schema.column_id(i), .num_varlen_columns_before = varlen_columns_count_, @@ -169,10 +171,11 @@ bool SchemaPacking::CouldPack( return true; } -SchemaPackingStorage::SchemaPackingStorage() = default; +SchemaPackingStorage::SchemaPackingStorage(TableType table_type) : table_type_(table_type) {} SchemaPackingStorage::SchemaPackingStorage( - const SchemaPackingStorage& rhs, SchemaVersion min_schema_version) { + const SchemaPackingStorage& rhs, SchemaVersion min_schema_version) + : table_type_(rhs.table_type_) { for (const auto& [version, packing] : rhs.version_to_schema_packing_) { if (version < min_schema_version) { continue; @@ -200,7 +203,7 @@ void SchemaPackingStorage::AddSchema(SchemaVersion version, const Schema& schema auto inserted = version_to_schema_packing_.emplace( std::piecewise_construct, std::forward_as_tuple(version), - std::forward_as_tuple(schema)).second; + std::forward_as_tuple(table_type_, schema)).second; LOG_IF(DFATAL, !inserted) << "Duplicate schema version: " << version << ", " << AsString(version_to_schema_packing_); } diff --git a/src/yb/docdb/schema_packing.h b/src/yb/docdb/schema_packing.h index bcca8febf66a..e14a323dbbbd 100644 --- a/src/yb/docdb/schema_packing.h +++ b/src/yb/docdb/schema_packing.h @@ -20,6 +20,7 @@ #include #include "yb/common/common_fwd.h" +#include "yb/common/common_types.pb.h" #include "yb/common/column_id.h" #include "yb/docdb/docdb.fwd.h" @@ -62,7 +63,7 @@ struct ColumnPackingData { class SchemaPacking { public: - explicit SchemaPacking(const Schema& schema); + SchemaPacking(TableType table_type, const Schema& schema); explicit SchemaPacking(const SchemaPackingPB& pb); size_t columns() const { @@ -93,8 +94,8 @@ class SchemaPacking { bool operator==(const SchemaPacking&) const = default; - bool SchemaContainsPacking(const Schema& schema) const { - SchemaPacking packing(schema); + bool SchemaContainsPacking(TableType table_type, const Schema& schema) const { + SchemaPacking packing(table_type, schema); return packing == *this; } @@ -106,8 +107,8 @@ class SchemaPacking { class SchemaPackingStorage { public: - SchemaPackingStorage(); - explicit SchemaPackingStorage(const SchemaPackingStorage& rhs, SchemaVersion min_schema_version); + explicit SchemaPackingStorage(TableType table_type); + SchemaPackingStorage(const SchemaPackingStorage& rhs, SchemaVersion min_schema_version); Result GetPacking(SchemaVersion schema_version) const; Result GetPacking(Slice* packed_row) const; @@ -141,6 +142,7 @@ class SchemaPackingStorage { const google::protobuf::RepeatedPtrField& schemas, bool could_present, OverwriteSchemaPacking overwrite); + TableType table_type_; std::unordered_map version_to_schema_packing_; }; diff --git a/src/yb/master/sys_catalog.cc b/src/yb/master/sys_catalog.cc index d8b94d71a26b..19ae3ba24f9e 100644 --- a/src/yb/master/sys_catalog.cc +++ b/src/yb/master/sys_catalog.cc @@ -164,7 +164,7 @@ std::string SysCatalogTable::schema_column_metadata() { return kSysCatalogTableC SysCatalogTable::SysCatalogTable(Master* master, MetricRegistry* metrics, ElectedLeaderCallback leader_cb) : doc_read_context_(std::make_unique( - kLogPrefix, BuildTableSchema(), kSysCatalogSchemaVersion)), + kLogPrefix, TableType::YQL_TABLE_TYPE, BuildTableSchema(), kSysCatalogSchemaVersion)), metric_registry_(metrics), metric_entity_(METRIC_ENTITY_server.Instantiate(metric_registry_, "yb.master")), master_(master), diff --git a/src/yb/master/system_tablet.cc b/src/yb/master/system_tablet.cc index e589a3e079fd..00b97188500e 100644 --- a/src/yb/master/system_tablet.cc +++ b/src/yb/master/system_tablet.cc @@ -29,7 +29,7 @@ SystemTablet::SystemTablet(const Schema& schema, std::unique_ptr( - log_prefix_, schema, kSysCatalogSchemaVersion)), + log_prefix_, TableType::YQL_TABLE_TYPE, schema, kSysCatalogSchemaVersion)), yql_virtual_table_(std::move(yql_virtual_table)), tablet_id_(tablet_id) { } diff --git a/src/yb/tablet/tablet-split-test.cc b/src/yb/tablet/tablet-split-test.cc index 2ea2bcd5cd11..f23f9c5cb4ce 100644 --- a/src/yb/tablet/tablet-split-test.cc +++ b/src/yb/tablet/tablet-split-test.cc @@ -128,8 +128,9 @@ TEST_F(TabletSplitTest, SplitTablet) { } VLOG(1) << "Source tablet:" << std::endl - << docdb::DocDBDebugDumpToStr(tablet()->doc_db(), docdb::SchemaPackingStorage(), - docdb::IncludeBinary::kTrue); + << docdb::DocDBDebugDumpToStr( + tablet()->doc_db(), docdb::SchemaPackingStorage(tablet()->table_type()), + docdb::IncludeBinary::kTrue); const auto source_docdb_dump_str = tablet()->TEST_DocDBDumpStr(IncludeIntents::kTrue); std::unordered_set source_docdb_dump; tablet()->TEST_DocDBDumpToContainer(IncludeIntents::kTrue, &source_docdb_dump); diff --git a/src/yb/tablet/tablet.cc b/src/yb/tablet/tablet.cc index 8275e9786592..935ed86e2716 100644 --- a/src/yb/tablet/tablet.cc +++ b/src/yb/tablet/tablet.cc @@ -3687,7 +3687,7 @@ Status Tablet::ReadIntents(std::vector* intents) { auto intent_iter = std::unique_ptr( intents_db_->NewIterator(read_options)); intent_iter->SeekToFirst(); - docdb::SchemaPackingStorage schema_packing_storage; + docdb::SchemaPackingStorage schema_packing_storage(table_type()); for (; intent_iter->Valid(); intent_iter->Next()) { auto item = EntryToString(intent_iter->key(), intent_iter->value(), diff --git a/src/yb/tablet/tablet_metadata.cc b/src/yb/tablet/tablet_metadata.cc index 0ea3f0a565de..0ddde6ef8b87 100644 --- a/src/yb/tablet/tablet_metadata.cc +++ b/src/yb/tablet/tablet_metadata.cc @@ -111,9 +111,9 @@ const std::string kSnapshotsDirSuffix = ".snapshots"; // Raft group metadata // ============================================================================ -TableInfo::TableInfo(const std::string& log_prefix_, PrivateTag) +TableInfo::TableInfo(const std::string& log_prefix_, TableType table_type, PrivateTag) : log_prefix(log_prefix_), - doc_read_context(new docdb::DocReadContext(log_prefix)), + doc_read_context(new docdb::DocReadContext(log_prefix, table_type)), index_map(std::make_shared()) { } @@ -134,7 +134,8 @@ TableInfo::TableInfo(const std::string& tablet_log_prefix, table_type(table_type), cotable_id(CHECK_RESULT(ParseCotableId(primary, table_id))), log_prefix(MakeTableInfoLogPrefix(tablet_log_prefix, primary, table_id)), - doc_read_context(std::make_shared(log_prefix, schema, schema_version)), + doc_read_context(std::make_shared( + log_prefix, table_type, schema, schema_version)), index_map(std::make_shared(index_map)), index_info(index_info ? new IndexInfo(*index_info) : nullptr), schema_version(schema_version), @@ -186,7 +187,7 @@ Result TableInfo::LoadFromPB( const std::string& tablet_log_prefix, const TableId& primary_table_id, const TableInfoPB& pb) { Primary primary(primary_table_id == pb.table_id()); auto log_prefix = MakeTableInfoLogPrefix(tablet_log_prefix, primary, pb.table_id()); - auto result = std::make_shared(log_prefix, PrivateTag()); + auto result = std::make_shared(log_prefix, pb.table_type(), PrivateTag()); RETURN_NOT_OK(result->DoLoadFromPB(primary, pb)); return result; } @@ -238,7 +239,8 @@ Status TableInfo::MergeWithRestored( // the latest schema. const docdb::SchemaPacking& latest_packing = VERIFY_RESULT( doc_read_context->schema_packing_storage.GetPacking(schema_version)); - LOG_IF_WITH_PREFIX(DFATAL, !latest_packing.SchemaContainsPacking(doc_read_context->schema)) + LOG_IF_WITH_PREFIX(DFATAL, + !latest_packing.SchemaContainsPacking(table_type, doc_read_context->schema)) << "After merging schema packings during restore, latest schema does not" << " have the same packing as the corresponding latest packing for table " << table_id; diff --git a/src/yb/tablet/tablet_metadata.h b/src/yb/tablet/tablet_metadata.h index 203aeb3b2d58..331f3c4811e8 100644 --- a/src/yb/tablet/tablet_metadata.h +++ b/src/yb/tablet/tablet_metadata.h @@ -111,7 +111,7 @@ struct TableInfo { uint32_t wal_retention_secs = 0; // Public ctor with private argument to allow std::make_shared, but prevent public usage. - TableInfo(const std::string& log_prefix, PrivateTag); + TableInfo(const std::string& log_prefix, TableType table_type, PrivateTag); TableInfo(const std::string& tablet_log_prefix, Primary primary, std::string table_id, diff --git a/src/yb/tablet/tablet_snapshots.cc b/src/yb/tablet/tablet_snapshots.cc index 9b923855b34d..a962075abaa1 100644 --- a/src/yb/tablet/tablet_snapshots.cc +++ b/src/yb/tablet/tablet_snapshots.cc @@ -282,7 +282,8 @@ Status TabletSnapshots::Restore(SnapshotOperation* operation) { } Status TabletSnapshots::RestorePartialRows(SnapshotOperation* operation) { - docdb::DocWriteBatch write_batch(tablet().doc_db(), docdb::InitMarkerBehavior::kOptional); + docdb::DocWriteBatch write_batch( + tablet().doc_db(), docdb::InitMarkerBehavior::kOptional, nullptr); auto restore_patch = VERIFY_RESULT(GenerateRestoreWriteBatch( operation->request()->ToGoogleProtobuf(), &write_batch)); diff --git a/src/yb/tools/bulk_load_docdb_util.h b/src/yb/tools/bulk_load_docdb_util.h index e44fc7a5fd67..86fb16a356f2 100644 --- a/src/yb/tools/bulk_load_docdb_util.h +++ b/src/yb/tools/bulk_load_docdb_util.h @@ -27,6 +27,7 @@ class BulkLoadDocDBUtil : public docdb::DocDBRocksDBUtil { std::string tablet_id() override; size_t block_cache_size() const override { return 0; } const std::string& rocksdb_dir(); + Schema CreateSchema() override { return Schema(); } private: const std::string tablet_id_; diff --git a/src/yb/tools/data-patcher.cc b/src/yb/tools/data-patcher.cc index 4bfe8dd74530..4646e77aa147 100644 --- a/src/yb/tools/data-patcher.cc +++ b/src/yb/tools/data-patcher.cc @@ -373,7 +373,7 @@ Status AddDeltaToSstFile( auto builder = helper->NewTableBuilder(base_file_writer.get(), data_file_writer.get()); const auto add_kv = [&builder, debug, storage_db_type](const Slice& k, const Slice& v) { if (debug) { - static docdb::SchemaPackingStorage schema_packing_storage; + static docdb::SchemaPackingStorage schema_packing_storage(TableType::YQL_TABLE_TYPE); const Slice user_key(k.data(), k.size() - kKeySuffixLen); auto key_type = docdb::GetKeyType(user_key, storage_db_type); auto rocksdb_value_type = static_cast(*(k.end() - kKeySuffixLen)); @@ -505,7 +505,7 @@ Status AddDeltaToSstFile( << "value " << value.ToDebugHexString() << " (" << FormatSliceAsStr(value) << "), " << "decoded value " << DocDBValueToDebugStr( docdb::KeyType::kReverseTxnKey, iterator->key(), iterator->value(), - docdb::SchemaPackingStorage()); + docdb::SchemaPackingStorage(TableType::YQL_TABLE_TYPE)); return doc_ht_result.status(); } delta_data.AddEarlyTime(doc_ht_result->hybrid_time()); diff --git a/src/yb/tools/sst_dump.cc b/src/yb/tools/sst_dump.cc index 4c6faaa63fe1..78b5283b9b5a 100644 --- a/src/yb/tools/sst_dump.cc +++ b/src/yb/tools/sst_dump.cc @@ -33,7 +33,7 @@ class DocDBKVFormatterImpl : public rocksdb::DocDBKVFormatter { return docdb::EntryToString(rocksdb::ExtractUserKey(key), value, schema_packing_storage_, type); } - docdb::SchemaPackingStorage schema_packing_storage_; + docdb::SchemaPackingStorage schema_packing_storage_{TableType::YQL_TABLE_TYPE}; }; } // namespace diff --git a/src/yb/tools/yb-bulk_load.cc b/src/yb/tools/yb-bulk_load.cc index fc40f9efa855..7a0a9ebe14dc 100644 --- a/src/yb/tools/yb-bulk_load.cc +++ b/src/yb/tools/yb-bulk_load.cc @@ -329,7 +329,7 @@ Status BulkLoadTask::InsertRow(const string &row, // once we have secondary indexes we probably might need to ensure bulk load builds the indexes // as well. auto doc_read_context = std::make_shared( - "BULK LOAD: ", schema, schema_version); + "BULK LOAD: ", TableType::YQL_TABLE_TYPE, schema, schema_version); docdb::QLWriteOperation op( req, schema_version, doc_read_context, index_map, nullptr /* unique_index_key_schema */, TransactionOperationContext()); diff --git a/src/yb/util/tostring.h b/src/yb/util/tostring.h index 5bef146b5adb..003269feeb6f 100644 --- a/src/yb/util/tostring.h +++ b/src/yb/util/tostring.h @@ -168,6 +168,11 @@ typename std::enable_if::value, std::string>::type return PointerToString::Apply(value); } +template +auto ToString(std::reference_wrapper value) { + return ToString(value.get()); +} + inline const std::string& ToString(const std::string& str) { return str; } inline std::string ToString(const char* str) { return str; } From 2c22d3291b0850878942b2214f0cccb7a7d33212 Mon Sep 17 00:00:00 2001 From: Yamen Haddad Date: Thu, 23 Feb 2023 22:57:43 +0000 Subject: [PATCH 27/81] [#15211] DocDB: Improve tablet_split command output from yb-admin utility Summary: While manual tablet splitting using yb-admin, the response from command is not descriptive and is not providing any concrete info whether operation succeeded or failed.It only shows the message: ``` Response: ``` In this diff, we show the user of yb-admin the following descriptive message to check master logs to get the status of the tablet split: ``` split_tablet "4aadf157640f4fd7b6230af60222630d" was sent asynchronously. Check master logs for more details about the status of the task ``` The decision of showing this message is due to the fact that split_tablet is an asynchronous task (thus waiting for it to finish requires introducing a new RPC). Additionally, automatic tablet splitting will be on by default from 2.18. More details about this decision can be found in this [[ https://github.com/yugabyte/yugabyte-db/issues/15211#issuecomment-1442528147 | GH issue ]]. Test Plan: Manual testing by issuing: ``` ./build/latest/bin/yb-admin \ --master_addresses 127.0.0.1:7100,127.0.0.2:7100,127.0.0.3:7100 \ split_tablet [tablet_id] ``` After replacing the tablet_id with a valid tablet UUID. The response after using this command is: - If the split_tablet has been successfully sent: ``` split_tablet "4aadf157640f4fd7b6230af60222630d" was sent asynchronously. Check master logs for more details about the status of the task ``` - If there is an error, the error message wil be printed as follow: ` Response: AsString(resp)` Reviewers: asrivastava, rthallam, arybochkin Reviewed By: asrivastava, rthallam, arybochkin Subscribers: bogdan, ybase Differential Revision: https://phabricator.dev.yugabyte.com/D23127 --- src/yb/tools/yb-admin_client.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/yb/tools/yb-admin_client.cc b/src/yb/tools/yb-admin_client.cc index eef23432d70f..0f47a16aa493 100644 --- a/src/yb/tools/yb-admin_client.cc +++ b/src/yb/tools/yb-admin_client.cc @@ -2156,7 +2156,13 @@ Status ClusterAdminClient::SplitTablet(const std::string& tablet_id) { req.set_tablet_id(tablet_id); const auto resp = VERIFY_RESULT(InvokeRpc( &master::MasterAdminProxy::SplitTablet, *master_admin_proxy_, req)); - std::cout << "Response: " << AsString(resp) << std::endl; + if (resp.has_error()) { + std::cout << "Response: " << AsString(resp); + } else { + std::cout << "split_tablet \"" << tablet_id << "\" was sent asynchronously. " + "Check master logs for more details about the status of the task."; + } + std::cout << std::endl; return Status::OK(); } From 46bbde5c37b37eb8c2d0c93b8c774d9c7409839b Mon Sep 17 00:00:00 2001 From: Jethro Mak <88681329+Jethro-M@users.noreply.github.com> Date: Wed, 22 Feb 2023 19:14:36 -0500 Subject: [PATCH 28/81] [PLAT-7162] Allow setting up replication between two universes with different KMS configs Summary: **Context** With the backend changes from 0ae4d3ac5cdbb0a30a8bcbd8a6f0e98e6276e16d / D22535 , we now support setting up xCluster replication configs between two universes with different KMS config. **Change** -Remove the KMS configuration check during xCluster target universe validation Test Plan: - Setup an xCluster config between two universes with different KMS configs {F35014} {F35018} {F35019} Reviewers: rmadhavan Reviewed By: rmadhavan Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23108 --- .../xcluster/createConfig/CreateConfigModal.tsx | 9 --------- 1 file changed, 9 deletions(-) diff --git a/managed/ui/src/components/xcluster/createConfig/CreateConfigModal.tsx b/managed/ui/src/components/xcluster/createConfig/CreateConfigModal.tsx index 649c6c60946d..d94c0170a786 100644 --- a/managed/ui/src/components/xcluster/createConfig/CreateConfigModal.tsx +++ b/managed/ui/src/components/xcluster/createConfig/CreateConfigModal.tsx @@ -1,7 +1,6 @@ import React, { useRef, useState } from 'react'; import { useMutation, useQuery, useQueryClient } from 'react-query'; import { toast } from 'react-toastify'; -import _ from 'lodash'; import { FormikActions, FormikErrors, FormikProps } from 'formik'; import axios, { AxiosError } from 'axios'; @@ -474,14 +473,6 @@ const validateForm = async ( ) { errors.targetUniverse = 'The target universe must have the same Encryption in-Transit (TLS) configuration as the source universe. Edit the TLS configuration to proceed.'; - } else if ( - !_.isEqual( - values.targetUniverse?.value?.universeDetails?.encryptionAtRestConfig, - sourceUniveres?.universeDetails?.encryptionAtRestConfig - ) - ) { - errors.targetUniverse = - 'The target universe must have the same key management system (KMS) configuration as the source universe. Edit the KMS configuration to proceed.'; } throw errors; From f0c7607147a980e7894d42804aed05fddd3bacd7 Mon Sep 17 00:00:00 2001 From: Aman Date: Wed, 1 Mar 2023 20:42:36 +0530 Subject: [PATCH 29/81] [PLAT-6864] Adding missing properties to suggested kubernetes handler Summary: [PLAT-6864] Adding missing properties to suggested kubernetes handler This feature allows customers to quickly deploy a k8s provider without needing to first collect all the config on the home cluster. Test Plan: Manual Sample response from the call. Instance located on: http://10.150.2.92/config/cloud/k8s Still cannot deploy the universe because we run into permission issues. ``` { "code": "kubernetes", "config": { "KUBECONFIG_IMAGE_PULL_SECRET_NAME": "gcr-pull-secret", "KUBECONFIG_PULL_SECRET_NAME": "gcr-pull-secret", "KUBECONFIG_PULL_SECRET_CONTENT": "---\napiVersion: \"v1\"\nkind: \"Secret\"\nmetadata:\n managedFields:\n - apiVersion: \"v1\"\n fieldsType: \"FieldsV1\"\n fieldsV1:\n f:data:\n \".\": {}\n f:.dockerconfigjson: {}\n f:type: {}\n manager: \"kubectl-create\"\n operation: \"Update\"\n time: \"2023-02-28T16:59:23Z\"\n name: \"gcr-pull-secret\"\ndata:\n \".dockerconfigjson\": \"eyJhdXRocyI6eyJodHRwczovL2djci5pbyI6eyJ1c2VybmFtZSI6Il9qc29uX2tleSIsInBhc3N3b3JkIjoie1xuICBcInR5cGVcIjogXCJzZXJ2aWNlX2FjY291bnRcIixcbiAgXCJwcm9qZWN0X2lkXCI6IFwieXVnYWJ5dGVcIixcbiAgXCJwcml2YXRlX2tleV9pZFwiOiBcImM2Y2Q2NjEyZjQ5YzAxNTU5ZWI5MDc0ZGE3ZDZmMTJjNWNhNGI1ZjVcIixcbiAgXCJwcml2YXRlX2tleVwiOiBcIi0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLVxcbk1JSUV2QUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktZd2dnU2lBZ0VBQW9JQkFRREJyQXVKSytJaDNPMExcXG5ZVUZjUEdRR0VjSXgvUWgydWN1U0g4bkFRbWtIMHpoRUM0Yi9TZFY3K2kzSGFma3kvSnZNLzJIVCs3dkhsLzlpXFxuNDYrcEJYeFJSNHJLUDQxQzdCL2E2OW1XNmVOemx2bkNEZzE0VzV0M25TVEdNenZhRmgybGRhaG85OU5YeEg0YVxcbmVFYktJenF3a1NDVUpPeURYZzFPSmdiK3UrRTE0SWVYb3BGUCtnaGtJQURvMmczdVluWHgxK3FEdklFVEVIZWlcXG5vbW03SzRydzhuK0dYTnFMUzArWlZsdkNvSkZ6N1EyQzZZb0xFVEViaTA0MVdLVHBaN2d2SXlVSEk0Q0tOZjlnXFxuR1JzeHJQZ2VsdDFEWEpOd0g3K2J2c3FJS3h0VXFUWC9WdGZ0V1lDZXoyT0p4SzlFSDY5ek11WkkvODh3UGZpQ1xcbjZDMWlkQ21MQWdNQkFBRUNnZ0VBVXhGbGMwVFl2WmlDTFo3ZHgxOTVzV2E5aWU1VFNvVWZUUUlJYVVleXYvSDNcXG5FS3gzRHhsT2tWbjVkNjQ2L2dUK0dGdXpUSmFNOEt0L2xSMCtjenNRbXRZYUwrdzVQU3FFOFVPMnRMV0Y0c2F2XFxuUW0zL2xnK0lxN0hCbTZkTzlSZ0dTRGtpYmo5WnJTdnd3STYxSWk4aEZhbHh0VWp2RHR0d3pvT2QvWFY5UDd0WlxcbkVTbGNuTzdmL0lEUm9UVm9lL3lqdENEYURYS0JKLzZIMnloV0IxQ2VkektCYUNSU3h3OXdSaFVSeDd0RkR5ODlcXG5uZkMvUUJ2QmJtK1FmSHNyRGR0QWdUVmFOdVl0aTJyWHRRS3pLNTNZS2pWdWx1cjEyWXBZZmR2ckQ0ZGZJMUMrXFxuSUZnUXlTRkxQSkVmeGdCaGpiUXVuYXFHaXF2RXo0cmJVNFNvNFE2a3hRS0JnUUQrQ0d2UUpFeUJGY1NIZWZKc1xcbmdNQkIrK21GT0RGQWJHZUtHWlM4UkxJakFPUGt3dEF4M0dlVStDRmJ5RHNPUHZ5R0NlZmRqT3N5a1RyWHhOZ3JcXG40b0MydGJmMzh3YmxoTE5QTGhkZ3pIdzA4Wm4rcVU3L1pIM21WbWFVVzRlQ2d6ZjExdGI2SzliRWNqOHh5eXN5XFxuVElOSnlRMlNuSWlkRHczRjA0NHdUQ2JoZFFLQmdRRERLL2ZuS3ZzelRDVmlub3lLOFlUY3ZQbGF4eTd2b09KWlxcbnMwcURJOXRIYXQwcER3Rmg4c0ppWmdpK21WcVhIWnFsS1IrazNMb2d0WHVRT2VnMmlPYWhDdW1tS0RRTEppVzNcXG5RbVE2K2oraHlKYXlmZ0hBQjJMdmczRndmUStvUzAvZ3JvQnVBUE4wNDI3eCtxSVNPL0MrMkJoWTNob1BqMjRPXFxuT2FyNXgzcCsvd0tCZ0I1d2llSzJCc2pNLy9DdjV5R2pSK1pnTmpvMVlvbHlzL0Z6WVVReUF4cDRwOVlvbGVQelxcbm9pSFNuY1N1dUl6YmRVem1jUGMyUkpTWm5IQndjT3JKU0YwY1p6LzV5aERFbHV2Yy9RWW5XcGJKZ0lkb2FFUG9cXG5HY0g1Qm0rdWpBMVNoZWthWmtZeDdKazdpMkQvRkYwQm1CWEdWcThEdk1iOUg4eGRya01SVFBObEFvR0FDUVlCXFxuWWlpMGtIeVd4ZVo2clBuWm9MTUQvRlIxTVJNSkpBaU1DMkFmQlVCRmxWNWlPL3NUVGFWV0Jpbms3Rk9zRUxMRVxcbkNjVkN2YnhtU0JiRlM3QXNxWWx0b0FiLzBQd2toa0tGaTh5Q0lLSzdoT2thZjV4WDBLY05HWnhPbXdOUVlCenhcXG5SeVM0SE5GZkVjVDZ0VWtjS2xhbldZUjdRL2pucDdTMjJQR0dFbFVDZ1lCc2lIVU5RVjd4c01FdDJqYkFCVVVmXFxuVVJnL1NPa2IxY1p3Nnl4ZXFabi9BeUY0Z2xDU3BnNSs1MUVLM0tDWWJQTDFteTIwaVQwNGtJU2ZiT2VaVEllZ1xcblJjVDhYeGE3cEZiaEZjaFV2a09iMU9VTTVWZTVrRjB1RTN3eThueENoNDAvaFRBZTAyVWNCTVhDUXJuT2gxeWNcXG41dnFIaHVKelU2a1ZXdHJSYXRmcnZ3PT1cXG4tLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tXFxuXCIsXG4gIFwiY2xpZW50X2VtYWlsXCI6IFwieWItYW5pamhhd2FuLXN2Y0B5dWdhYnl0ZS5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbVwiLFxuICBcImNsaWVudF9pZFwiOiBcIjEwODA3NDExOTEzNTg4NjQwNjgxN1wiLFxuICBcImF1dGhfdXJpXCI6IFwiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tL28vb2F1dGgyL2F1dGhcIixcbiAgXCJ0b2tlbl91cmlcIjogXCJodHRwczovL29hdXRoMi5nb29nbGVhcGlzLmNvbS90b2tlblwiLFxuICBcImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybFwiOiBcImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL29hdXRoMi92MS9jZXJ0c1wiLFxuICBcImNsaWVudF94NTA5X2NlcnRfdXJsXCI6IFwiaHR0cHM6Ly93d3cuZ29vZ2xlYXBpcy5jb20vcm9ib3QvdjEvbWV0YWRhdGEveDUwOS95Yi1hbmlqaGF3YW4tc3ZjJTQweXVnYWJ5dGUuaWFtLmdzZXJ2aWNlYWNjb3VudC5jb21cIlxufSIsImVtYWlsIjoiYW55QHZhbGlkLmVtYWlsIiwiYXV0aCI6IlgycHpiMjVmYTJWNU9uc0tJQ0FpZEhsd1pTSTZJQ0p6WlhKMmFXTmxYMkZqWTI5MWJuUWlMQW9nSUNKd2NtOXFaV04wWDJsa0lqb2dJbmwxWjJGaWVYUmxJaXdLSUNBaWNISnBkbUYwWlY5clpYbGZhV1FpT2lBaVl6WmpaRFkyTVRKbU5EbGpNREUxTlRsbFlqa3dOelJrWVRka05tWXhNbU0xWTJFMFlqVm1OU0lzQ2lBZ0luQnlhWFpoZEdWZmEyVjVJam9nSWkwdExTMHRRa1ZIU1U0Z1VGSkpWa0ZVUlNCTFJWa3RMUzB0TFZ4dVRVbEpSWFpCU1VKQlJFRk9RbWRyY1docmFVYzVkekJDUVZGRlJrRkJVME5DUzFsM1oyZFRhVUZuUlVGQmIwbENRVkZFUW5KQmRVcExLMGxvTTA4d1RGeHVXVlZHWTFCSFVVZEZZMGw0TDFGb01uVmpkVk5JT0c1QlVXMXJTREI2YUVWRE5HSXZVMlJXTnl0cE0waGhabXQ1TDBwMlRTOHlTRlFyTjNaSWJDODVhVnh1TkRZcmNFSlllRkpTTkhKTFVEUXhRemRDTDJFMk9XMVhObVZPZW14MmJrTkVaekUwVnpWME0yNVRWRWROZW5aaFJtZ3liR1JoYUc4NU9VNVllRWcwWVZ4dVpVVmlTMGw2Y1hkclUwTlZTazk1UkZobk1VOUtaMklyZFN0Rk1UUkpaVmh2Y0VaUUsyZG9hMGxCUkc4eVp6TjFXVzVZZURFcmNVUjJTVVZVUlVobGFWeHViMjF0TjBzMGNuYzRiaXRIV0U1eFRGTXdLMXBXYkhaRGIwcEdlamRSTWtNMldXOU1SVlJGWW1rd05ERlhTMVJ3V2pkbmRrbDVWVWhKTkVOTFRtWTVaMXh1UjFKemVISlFaMlZzZERGRVdFcE9kMGczSzJKMmMzRkpTM2gwVlhGVVdDOVdkR1owVjFsRFpYb3lUMHA0U3psRlNEWTVlazExV2trdk9EaDNVR1pwUTF4dU5rTXhhV1JEYlV4QlowMUNRVUZGUTJkblJVRlZlRVpzWXpCVVdYWmFhVU5NV2pka2VERTVOWE5YWVRscFpUVlVVMjlWWmxSUlNVbGhWV1Y1ZGk5SU0xeHVSVXQ0TTBSNGJFOXJWbTQxWkRZME5pOW5WQ3RIUm5WNlZFcGhUVGhMZEM5c1VqQXJZM3B6VVcxMFdXRk1LM2MxVUZOeFJUaFZUekowVEZkR05ITmhkbHh1VVcwekwyeG5LMGx4TjBoQ2JUWmtUemxTWjBkVFJHdHBZbW81V25KVGRuZDNTVFl4U1drNGFFWmhiSGgwVldwMlJIUjBkM3B2VDJRdldGWTVVRGQwV2x4dVJWTnNZMjVQTjJZdlNVUlNiMVJXYjJVdmVXcDBRMFJoUkZoTFFrb3ZOa2d5ZVdoWFFqRkRaV1I2UzBKaFExSlRlSGM1ZDFKb1ZWSjROM1JHUkhrNE9WeHVibVpETDFGQ2RrSmliU3RSWmtoemNrUmtkRUZuVkZaaFRuVlpkR2t5Y2xoMFVVdDZTelV6V1V0cVZuVnNkWEl4TWxsd1dXWmtkbkpFTkdSbVNURkRLMXh1U1VablVYbFRSa3hRU2tWbWVHZENhR3BpVVhWdVlYRkhhWEYyUlhvMGNtSlZORk52TkZFMmEzaFJTMEpuVVVRclEwZDJVVXBGZVVKR1kxTklaV1pLYzF4dVowMUNRaXNyYlVaUFJFWkJZa2RsUzBkYVV6aFNURWxxUVU5UWEzZDBRWGd6UjJWVkswTkdZbmxFYzA5UWRubEhRMlZtWkdwUGMzbHJWSEpZZUU1bmNseHVORzlETW5SaVpqTTRkMkpzYUV4T1VFeG9aR2Q2U0hjd09GcHVLM0ZWTnk5YVNETnRWbTFoVlZjMFpVTm5lbVl4TVhSaU5rczVZa1ZqYWpoNGVYbHplVnh1VkVsT1NubFJNbE51U1dsa1JIY3pSakEwTkhkVVEySm9aRkZMUW1kUlJFUkxMMlp1UzNaemVsUkRWbWx1YjNsTE9GbFVZM1pRYkdGNGVUZDJiMDlLV2x4dWN6QnhSRWs1ZEVoaGREQndSSGRHYURoelNtbGFaMmtyYlZaeFdFaGFjV3hMVWl0ck0weHZaM1JZZFZGUFpXY3lhVTloYUVOMWJXMUxSRkZNU21sWE0xeHVVVzFSTml0cUsyaDVTbUY1Wm1kSVFVSXlUSFpuTTBaM1psRXJiMU13TDJkeWIwSjFRVkJPTURReU4zZ3JjVWxUVHk5REt6SkNhRmt6YUc5UWFqSTBUMXh1VDJGeU5YZ3pjQ3N2ZDB0Q1owSTFkMmxsU3pKQ2MycE5MeTlEZGpWNVIycFNLMXBuVG1wdk1WbHZiSGx6TDBaNldWVlJlVUY0Y0RSd09WbHZiR1ZRZWx4dWIybElVMjVqVTNWMVNYcGlaRlY2YldOUVl6SlNTbE5hYmtoQ2QyTlBja3BUUmpCaldub3ZOWGxvUkVWc2RYWmpMMUZaYmxkd1lrcG5TV1J2WVVWUWIxeHVSMk5JTlVKdEszVnFRVEZUYUdWcllWcHJXWGczU21zM2FUSkVMMFpHTUVKdFFsaEhWbkU0UkhaTllqbElPSGhrY210TlVsUlFUbXhCYjBkQlExRlpRbHh1V1dscE1HdEllVmQ0WlZvMmNsQnVXbTlNVFVRdlJsSXhUVkpOU2twQmFVMURNa0ZtUWxWQ1JteFdOV2xQTDNOVVZHRldWMEpwYm1zM1JrOXpSVXhNUlZ4dVEyTldRM1ppZUcxVFFtSkdVemRCYzNGWmJIUnZRV0l2TUZCM2EyaHJTMFpwT0hsRFNVdExOMmhQYTJGbU5YaFlNRXRqVGtkYWVFOXRkMDVSV1VKNmVGeHVVbmxUTkVoT1JtWkZZMVEyZEZWclkwdHNZVzVYV1ZJM1VTOXFibkEzVXpJeVVFZEhSV3hWUTJkWlFuTnBTRlZPVVZZM2VITk5SWFF5YW1KQlFsVlZabHh1VlZKbkwxTlBhMkl4WTFwM05ubDRaWEZhYmk5QmVVWTBaMnhEVTNCbk5TczFNVVZMTTB0RFdXSlFUREZ0ZVRJd2FWUXdOR3RKVTJaaVQyVmFWRWxsWjF4dVVtTlVPRmg0WVRkd1JtSm9SbU5vVlhaclQySXhUMVZOTlZabE5XdEdNSFZGTTNkNU9HNTRRMmcwTUM5b1ZFRmxNREpWWTBKTldFTlJjbTVQYURGNVkxeHVOWFp4U0doMVNucFZObXRXVjNSeVVtRjBabkoyZHowOVhHNHRMUzB0TFVWT1JDQlFVa2xXUVZSRklFdEZXUzB0TFMwdFhHNGlMQW9nSUNKamJHbGxiblJmWlcxaGFXd2lPaUFpZVdJdFlXNXBhbWhoZDJGdUxYTjJZMEI1ZFdkaFlubDBaUzVwWVcwdVozTmxjblpwWTJWaFkyTnZkVzUwTG1OdmJTSXNDaUFnSW1Oc2FXVnVkRjlwWkNJNklDSXhNRGd3TnpReE1Ua3hNelU0T0RZME1EWTRNVGNpTEFvZ0lDSmhkWFJvWDNWeWFTSTZJQ0pvZEhSd2N6b3ZMMkZqWTI5MWJuUnpMbWR2YjJkc1pTNWpiMjB2Ynk5dllYVjBhREl2WVhWMGFDSXNDaUFnSW5SdmEyVnVYM1Z5YVNJNklDSm9kSFJ3Y3pvdkwyOWhkWFJvTWk1bmIyOW5iR1ZoY0dsekxtTnZiUzkwYjJ0bGJpSXNDaUFnSW1GMWRHaGZjSEp2ZG1sa1pYSmZlRFV3T1Y5alpYSjBYM1Z5YkNJNklDSm9kSFJ3Y3pvdkwzZDNkeTVuYjI5bmJHVmhjR2x6TG1OdmJTOXZZWFYwYURJdmRqRXZZMlZ5ZEhNaUxBb2dJQ0pqYkdsbGJuUmZlRFV3T1Y5alpYSjBYM1Z5YkNJNklDSm9kSFJ3Y3pvdkwzZDNkeTVuYjI5bmJHVmhjR2x6TG1OdmJTOXliMkp2ZEM5Mk1TOXRaWFJoWkdGMFlTOTROVEE1TDNsaUxXRnVhV3BvWVhkaGJpMXpkbU1sTkRCNWRXZGhZbmwwWlM1cFlXMHVaM05sY25acFkyVmhZMk52ZFc1MExtTnZiU0lLZlE9PSJ9fX0=\"\ntype: \"kubernetes.io/dockerconfigjson\"\n", "KUBECONFIG_IMAGE_REGISTRY": "gcr.io/yugabyte/yugabyte" }, "regionList": [ { "code": "us-west1", "name": "us-west1", "latitude": 0.0, "longitude": 0.0, "zoneList": [ { "code": "us-west1-b", "name": "us-west1-b", "config": { "STORAGE_CLASS": "yb-standard" } }, { "code": "us-west1-c", "name": "us-west1-c", "config": { "STORAGE_CLASS": "yb-standard" } }, { "code": "us-west1-a", "name": "us-west1-a", "config": { "STORAGE_CLASS": "yb-standard" } } ] } ] } ``` Reviewers: kkannan, sanketh, bgandhi, sneelakantan Reviewed By: sneelakantan Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D22882 --- .../yw/common/ShellKubernetesManager.java | 9 +- .../handlers/CloudProviderHandler.java | 107 ++- managed/src/main/resources/k8s_regions.json | 34 + .../CloudProviderControllerTest.java | 32 +- managed/src/test/resources/testYugaware.json | 697 ++++++++++++++++++ 5 files changed, 857 insertions(+), 22 deletions(-) create mode 100644 managed/src/main/resources/k8s_regions.json create mode 100644 managed/src/test/resources/testYugaware.json diff --git a/managed/src/main/java/com/yugabyte/yw/common/ShellKubernetesManager.java b/managed/src/main/java/com/yugabyte/yw/common/ShellKubernetesManager.java index 961184bbfb34..573ae0547a0a 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/ShellKubernetesManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/ShellKubernetesManager.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonParser.Feature; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; @@ -91,7 +92,10 @@ private ShellResponse execCommand( private T deserialize(String json, Class type) { try { - return new ObjectMapper().configure(Feature.ALLOW_SINGLE_QUOTES, true).readValue(json, type); + return new ObjectMapper() + .configure(Feature.ALLOW_SINGLE_QUOTES, true) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .readValue(json, type); } catch (Exception e) { throw new RuntimeException("Error deserializing response from kubectl command: ", e); } @@ -161,6 +165,9 @@ public List getServices( @Override public Pod getPodObject(Map config, String namespace, String podName) { + if (namespace == null) { + namespace = ""; + } List commandList = ImmutableList.of("kubectl", "get", "pod", "--namespace", namespace, "-o", "json", podName); ShellResponse response = diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java index e5f3d484cfc1..79099f1b1394 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/handlers/CloudProviderHandler.java @@ -70,6 +70,10 @@ import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Secret; import java.util.ArrayList; +import io.fabric8.kubernetes.client.utils.Serialization; +import java.io.IOException; +import java.io.File; +import java.io.InputStream; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -86,11 +90,20 @@ import play.Configuration; import play.Environment; import play.libs.Json; +import scala.reflect.internal.tpe.TypeMaps.ContainsCollector; +import java.io.BufferedReader; +import java.io.FileReader; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodSpec; +import io.fabric8.kubernetes.api.model.Container; +import com.yugabyte.yw.common.KubernetesManagerFactory; + +import play.Application; +import play.Play; public class CloudProviderHandler { public static final String YB_FIREWALL_TAGS = "YB_FIREWALL_TAGS"; public static final String SKIP_KEYPAIR_VALIDATION_KEY = "yb.provider.skip_keypair_validation"; - private static final Logger LOG = LoggerFactory.getLogger(CloudProviderHandler.class); private static final JsonNode KUBERNETES_CLOUD_INSTANCE_TYPE = Json.parse("{\"instanceTypeCode\": \"cloud\", \"numCores\": 0.5, \"memSizeGB\": 1.5}"); @@ -504,22 +517,32 @@ public KubernetesProviderFormData suggestedKubernetesConfigs() { throw new PlatformServiceException( INTERNAL_SERVER_ERROR, "Required configuration is missing."); } - String pullSecretContent = getKubernetesPullSecretContent(pullSecretName); + Secret pullSecret = getKubernetesPullSecret(pullSecretName); KubernetesProviderFormData formData = new KubernetesProviderFormData(); formData.code = kubernetes; - if (pullSecretContent != null) { + if (pullSecret != null) { formData.config = ImmutableMap.of( - "KUBECONFIG_IMAGE_PULL_SECRET_NAME", pullSecretName, - "KUBECONFIG_PULL_SECRET_NAME", pullSecretName, // filename - "KUBECONFIG_PULL_SECRET_CONTENT", pullSecretContent); + "KUBECONFIG_IMAGE_PULL_SECRET_NAME", + pullSecretName, + "KUBECONFIG_PULL_SECRET_NAME", + pullSecretName, // filename + "KUBECONFIG_PULL_SECRET_CONTENT", + Serialization.asYaml(pullSecret), // Yaml formatted + "KUBECONFIG_IMAGE_REGISTRY", + getKubernetesImageRepository()); // Location of the registry } for (String region : regionToAZ.keySet()) { KubernetesProviderFormData.RegionData regionData = new KubernetesProviderFormData.RegionData(); regionData.code = region; + String regName = getRegionNameFromCode(region); + if (regName == null) { + regName = region; + } + regionData.name = regName; for (String az : regionToAZ.get(region)) { KubernetesProviderFormData.RegionData.ZoneData zoneData = new KubernetesProviderFormData.RegionData.ZoneData(); @@ -533,7 +556,15 @@ public KubernetesProviderFormData suggestedKubernetesConfigs() { return formData; } catch (RuntimeException e) { - throw new PlatformServiceException(INTERNAL_SERVER_ERROR, e.getMessage()); + LOG.error( + e.getClass() + + ": " + + e.getMessage() + + ": " + + e.getCause() + + "\n" + + e.getStackTrace().toString()); + throw e; // new PlatformServiceException(INTERNAL_SERVER_ERROR, e.getMessage()); } } // Performs region and zone discovery based on @@ -567,9 +598,9 @@ private Multimap computeKubernetesRegionToZoneInfo() { return regionToAZ; } // Fetches the secret secretName from current namespace, removes - // extra metadata and returns the secret as JSON string. Returns - // null if the secret is not present. - private String getKubernetesPullSecretContent(String secretName) { + // Extra metadata and returns the secret object. + // Returns null if the secret is not present. + private Secret getKubernetesPullSecret(String secretName) { Secret pullSecret; try { pullSecret = kubernetesManagerFactory.getManager().getSecret(null, secretName, null); @@ -594,11 +625,65 @@ private String getKubernetesPullSecretContent(String secretName) { metadata.setSelfLink(null); metadata.setCreationTimestamp(null); metadata.setResourceVersion(null); + metadata.setManagedFields(null); if (metadata.getAnnotations() != null) { metadata.getAnnotations().remove("kubectl.kubernetes.io/last-applied-configuration"); } - return pullSecret.toString(); + return pullSecret; + } + + public String getKubernetesImageRepository() { + String podName = System.getenv("HOSTNAME"); + if (podName == null) { + podName = "yugaware-0"; + } + + String containerName = System.getenv("container"); + if (containerName == null) { + containerName = "yugaware"; + } + // Container Name can change between yugaware and yugaware-docker. + containerName = containerName.split("-")[0]; + + Pod podObject = kubernetesManagerFactory.getManager().getPodObject(null, null, podName); + if (podObject == null) { + throw new PlatformServiceException( + INTERNAL_SERVER_ERROR, "Error while fetching pod details for yugaware"); + } + String imageName = null; + List containers = podObject.getSpec().getContainers(); + for (Container c : containers) { + if (containerName.equals(c.getName())) { + imageName = c.getImage(); + } + } + if (imageName == null) { + throw new PlatformServiceException( + INTERNAL_SERVER_ERROR, "Error while fetching image details for yugaware"); + } + String[] parsed = imageName.split("/"); + /* Algorithm Used + Take last element, split it with ":", take the 0th element of that list. + in that element replace string yugaware with yugabyte. + gcr.io/yugabyte/dev-ci-yugaware:2.17.2.0-b9999480 -> gcr.io/yugabyte/dev-ci-yugabyte */ + parsed[parsed.length - 1] = parsed[parsed.length - 1].split(":")[0]; + parsed[parsed.length - 1] = parsed[parsed.length - 1].replace("yugaware", "yugabyte"); + return String.join("/", parsed); + } + + public String getRegionNameFromCode(String code) { + LOG.info("Code is:", code); + String regionFile = "k8s_regions.json"; + Application app = play.Play.application(); + InputStream inputStream = app.resourceAsStream(regionFile); + JsonNode jsonNode = Json.parse(inputStream); + JsonNode nameNode = jsonNode.get(code); + if (nameNode == null || nameNode.isMissingNode()) { + LOG.info("Could not find code in file, sending it back as name"); + return code; + } + return jsonNode.get(code).asText(); } public Provider setupNewDockerProvider(Customer customer) { diff --git a/managed/src/main/resources/k8s_regions.json b/managed/src/main/resources/k8s_regions.json new file mode 100644 index 000000000000..93666be9132b --- /dev/null +++ b/managed/src/main/resources/k8s_regions.json @@ -0,0 +1,34 @@ +{ + "us-west-1": "US West (N. California)", + "us-west-2": "US West (Oregon)", + "us-east-1": "US East (N. Virginia)", + "us-east-2": "US East (Ohio)", + "us-south": "US South", + "us-north": "US North", + "south-asia": "South Asia", + "south-east-asia": "SE Asia", + "new-zealand": "New Zealand", + "japan": "Japan", + "eu-west-1": "Europe (Ireland)", + "eu-west-2": "Europe (London)", + "eu-west-3": "Europe (Paris)", + "eu-west-4": "Europe (Amsterdam)", + "eu-central-1": "Europe (Frankfurt)", + "ap-southeast-1": "Asia Pacific (Singapore)", + "ap-southeast-2": "Asia Pacific (Sydney)", + "ap-northeast-1": "Asia Pacific (Tokyo)", + "ap-northeast-2": "Asia Pacific (Seoul)", + "ap-northeast-3": "Asia Pacific (Osaka)", + "sa-east-1": "South America (Sao Paulo)", + "ap-south-1": "Asia Pacific (Mumbai)", + "ca-central-1": "Montreal (Canada)", + "me-south-1": "Middle East (Bahrain)", + "ap-east-1": "Asia Pacific (Hong Kong)", + "af-south-1": "Africa (Cape Town)", + "eu-south-1": "Europe (Milan)", + "eu-north-1": "Europe (Stockholm)", + "eu-east": "EU East", + "china": "China", + "brazil": "Brazil", + "australia": "Australia" +} diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderControllerTest.java index ac18371b5014..7cfe9b48cced 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/CloudProviderControllerTest.java @@ -64,9 +64,17 @@ import com.yugabyte.yw.models.helpers.provider.GCPCloudInfo; import com.yugabyte.yw.models.helpers.TaskType; import io.fabric8.kubernetes.api.model.Node; +import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.PodList; +import io.fabric8.kubernetes.api.model.PodStatus; import io.fabric8.kubernetes.api.model.NodeList; import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.utils.Serialization; import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import org.yaml.snakeyaml.Yaml; + import java.io.IOException; import java.net.URLEncoder; import java.nio.file.Files; @@ -489,6 +497,15 @@ public void testGetK8sSuggestedConfigWithoutPullSecret() { } private void testGetK8sSuggestedConfigBase(boolean noPullSecret) { + Pod testPod = null; + try { + File jsonFile = new File("src/test/resources/testYugaware.json"); + InputStream jsonStream = new FileInputStream(jsonFile); + + testPod = Serialization.unmarshal(jsonStream, Pod.class); + when(mockKubernetesManager.getPodObject(any(), any(), any())).thenReturn(testPod); + } catch (Exception e) { + } String pullSecretName = "pull-sec"; String storageClassName = "ssd-class"; // Was not able to get this working after trying various @@ -541,18 +558,13 @@ private void testGetK8sSuggestedConfigBase(boolean noPullSecret) { if (noPullSecret) { assertTrue(Json.fromJson(json.path("config"), Map.class).isEmpty()); } else { - String parsedSecretString = - "{\"metadata\":{" - + "\"annotations\":{}," - + "\"name\":\"" - + pullSecretName - + "\"}," - + "\"data\":{\".dockerconfigjson\":\"sec-key\"}}"; - Secret parsedSecret = TestUtils.deserialize(parsedSecretString, Secret.class); - assertValueAtPath(json, "/config/KUBECONFIG_IMAGE_PULL_SECRET_NAME", pullSecretName); assertValueAtPath(json, "/config/KUBECONFIG_PULL_SECRET_NAME", pullSecretName); - assertValueAtPath(json, "/config/KUBECONFIG_PULL_SECRET_CONTENT", parsedSecret.toString()); + Yaml ya = new Yaml(); + String one = ya.dump(ya.load(json.at("/config/KUBECONFIG_PULL_SECRET_CONTENT").toString())); + assertTrue(one.trim().endsWith("\".dockerconfigjson\": \"sec-key\"")); + String registryPath = "quay.io/yugabyte/yugabyte-itest"; + assertValueAtPath(json, "/config/KUBECONFIG_IMAGE_REGISTRY", registryPath); } assertValues( diff --git a/managed/src/test/resources/testYugaware.json b/managed/src/test/resources/testYugaware.json new file mode 100644 index 000000000000..502778ba2db7 --- /dev/null +++ b/managed/src/test/resources/testYugaware.json @@ -0,0 +1,697 @@ +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "annotations": { + "checksum/config": "be6ed74cd25768edf63927c11bb47f7403e9b46e0dd5543ac9687db3a40d568d" + }, + "creationTimestamp": "2023-02-14T21:20:16Z", + "generateName": "yw-ns-jenk-i3082-20230214-211928-yugaware-", + "labels": { + "app": "yw-ns-jenk-i3082-20230214-211928-yugaware", + "controller-revision-hash": "yw-ns-jenk-i3082-20230214-211928-yugaware-6f4bb8f44f", + "statefulset.kubernetes.io/pod-name": "yw-ns-jenk-i3082-20230214-211928-yugaware-0" + }, + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-0", + "namespace": "yw-ns-jenk-i3082-20230214-211928", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "StatefulSet", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware", + "uid": "e6f6a6b6-0b57-4f27-9108-2844129ce720" + } + ], + "resourceVersion": "137716233", + "uid": "79d31f87-d210-4246-a718-25c420f01554" + }, + "spec": { + "containers": [ + { + "args": [ + "-c", + "huge_pages=off" + ], + "env": [ + { + "name": "POSTGRES_USER", + "valueFrom": { + "secretKeyRef": { + "key": "postgres_user", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + }, + { + "name": "POSTGRES_PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "postgres_password", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + }, + { + "name": "POSTGRES_DB", + "valueFrom": { + "secretKeyRef": { + "key": "postgres_db", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + }, + { + "name": "PGDATA", + "value": "/var/lib/postgresql/data/pgdata" + } + ], + "image": "postgres:14.6", + "imagePullPolicy": "Always", + "name": "postgres", + "ports": [ + { + "containerPort": 5432, + "name": "postgres", + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": "500m", + "memory": "1Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/lib/postgresql/data", + "name": "yugaware-storage", + "subPath": "postgres_data_14" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-w2r5r", + "readOnly": true + } + ] + }, + { + "args": [ + "--config.file=/prometheus_configs/prometheus.yml", + "--storage.tsdb.path=/prometheus/", + "--web.enable-admin-api", + "--web.enable-lifecycle", + "--storage.tsdb.retention.time=15d", + "--query.max-concurrency=20", + "--query.max-samples=5e+06", + "--query.timeout=30s" + ], + "image": "prom/prometheus:v2.41.0", + "imagePullPolicy": "Always", + "name": "prometheus", + "ports": [ + { + "containerPort": 9090, + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": "500m", + "memory": "4Gi" + } + }, + "securityContext": { + "runAsUser": 0 + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/prometheus_configs", + "name": "yugaware-storage", + "subPath": "prometheus.yml" + }, + { + "mountPath": "/prometheus/", + "name": "yugaware-storage" + }, + { + "mountPath": "/opt/yugabyte/prometheus/targets", + "name": "yugaware-storage", + "subPath": "swamper_targets" + }, + { + "mountPath": "/opt/yugabyte/prometheus/rules", + "name": "yugaware-storage", + "subPath": "swamper_rules" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-w2r5r", + "readOnly": true + } + ] + }, + { + "args": [ + "bin/yugaware", + "-Dconfig.file=/data/application.docker.conf" + ], + "command": [ + "/sbin/tini", + "--" + ], + "env": [ + { + "name": "POSTGRES_USER", + "valueFrom": { + "secretKeyRef": { + "key": "postgres_user", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + }, + { + "name": "POSTGRES_PASSWORD", + "valueFrom": { + "secretKeyRef": { + "key": "postgres_password", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + }, + { + "name": "POSTGRES_DB", + "valueFrom": { + "secretKeyRef": { + "key": "postgres_db", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + }, + { + "name": "APP_SECRET", + "valueFrom": { + "secretKeyRef": { + "key": "app_secret", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + } + ], + "image": "quay.io/yugabyte/yugaware-itest:2.16.2.0-b25", + "imagePullPolicy": "Always", + "name": "yugaware", + "ports": [ + { + "containerPort": 9000, + "name": "yugaware", + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": "5", + "memory": "8Gi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/data", + "name": "yugaware-config" + }, + { + "mountPath": "/opt/yugabyte/yugaware/data/", + "name": "yugaware-storage", + "subPath": "data" + }, + { + "mountPath": "/opt/yugaware_data/", + "name": "yugaware-storage", + "subPath": "data" + }, + { + "mountPath": "/opt/yugabyte/releases/", + "name": "yugaware-storage", + "subPath": "releases" + }, + { + "mountPath": "/opt/yugabyte/ybc/releases/", + "name": "yugaware-storage", + "subPath": "ybc_releases" + }, + { + "mountPath": "/opt/releases/", + "name": "yugaware-storage", + "subPath": "releases" + }, + { + "mountPath": "/opt/yugabyte/prometheus/targets", + "name": "yugaware-storage", + "subPath": "swamper_targets" + }, + { + "mountPath": "/opt/yugabyte/prometheus/rules", + "name": "yugaware-storage", + "subPath": "swamper_rules" + }, + { + "mountPath": "/prometheus_configs", + "name": "yugaware-storage", + "subPath": "prometheus.yml" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-w2r5r", + "readOnly": true + } + ] + }, + { + "image": "nginxinc/nginx-unprivileged:1.23.3", + "imagePullPolicy": "Always", + "name": "nginx", + "ports": [ + { + "containerPort": 8080, + "protocol": "TCP" + } + ], + "resources": { + "requests": { + "cpu": "250m", + "memory": "300Mi" + } + }, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/etc/nginx/conf.d/", + "name": "nginx-config" + }, + { + "mountPath": "/etc/nginx/nginx.conf", + "name": "nginx-main-config", + "subPath": "nginx.conf" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-w2r5r", + "readOnly": true + } + ] + } + ], + "dnsPolicy": "ClusterFirst", + "enableServiceLinks": true, + "hostname": "yw-ns-jenk-i3082-20230214-211928-yugaware-0", + "imagePullSecrets": [ + { + "name": "yugabyte-k8s-pull-secret" + } + ], + "initContainers": [ + { + "command": [ + "cp", + "/default_prometheus_config/prometheus.yml", + "/prometheus_configs/prometheus.yml" + ], + "image": "quay.io/yugabyte/yugaware-itest:2.16.2.0-b25", + "imagePullPolicy": "Always", + "name": "prometheus-configuration", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/default_prometheus_config", + "name": "prometheus-config" + }, + { + "mountPath": "/prometheus_configs", + "name": "yugaware-storage", + "subPath": "prometheus.yml" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-w2r5r", + "readOnly": true + } + ] + }, + { + "command": [ + "bash", + "-c", + "/bin/bash /pg_upgrade_11_to_14/pg-upgrade-11-to-14.sh;" + ], + "env": [ + { + "name": "PGDATANEW", + "value": "/var/lib/postgresql/14/pgdata" + }, + { + "name": "PGDATAOLD", + "value": "/var/lib/postgresql/11/pgdata" + }, + { + "name": "PGUSER", + "valueFrom": { + "secretKeyRef": { + "key": "postgres_user", + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-global-config" + } + } + }, + { + "name": "POSTGRES_INITDB_ARGS", + "value": "-U $PGUSER" + } + ], + "image": "tianon/postgres-upgrade:11-to-14", + "imagePullPolicy": "Always", + "name": "postgres-upgrade", + "resources": {}, + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "volumeMounts": [ + { + "mountPath": "/var/lib/postgresql/11/", + "name": "yugaware-storage", + "subPath": "postgres_data" + }, + { + "mountPath": "/var/lib/postgresql/14/", + "name": "yugaware-storage", + "subPath": "postgres_data_14" + }, + { + "mountPath": "/pg_upgrade_11_to_14", + "name": "pg-upgrade-11-to-14" + }, + { + "mountPath": "/pg_upgrade_logs", + "name": "yugaware-storage", + "subPath": "postgres_data_14" + }, + { + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", + "name": "kube-api-access-w2r5r", + "readOnly": true + } + ] + } + ], + "nodeName": "gke-itest-release-default-pool-36021bc1-9978", + "preemptionPolicy": "PreemptLowerPriority", + "priority": 0, + "restartPolicy": "Always", + "schedulerName": "default-scheduler", + "securityContext": {}, + "serviceAccount": "yw-ns-jenk-i3082-20230214-211928", + "serviceAccountName": "yw-ns-jenk-i3082-20230214-211928", + "subdomain": "yw-ns-jenk-i3082-20230214-211928-yugaware", + "terminationGracePeriodSeconds": 30, + "tolerations": [ + { + "effect": "NoExecute", + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "tolerationSeconds": 300 + }, + { + "effect": "NoExecute", + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "tolerationSeconds": 300 + } + ], + "volumes": [ + { + "name": "yugaware-storage", + "persistentVolumeClaim": { + "claimName": "yw-ns-jenk-i3082-20230214-211928-yugaware-storage" + } + }, + { + "emptyDir": {}, + "name": "yugaware-ui" + }, + { + "name": "yugaware-config", + "projected": { + "defaultMode": 420, + "sources": [ + { + "configMap": { + "items": [ + { + "key": "application.docker.conf", + "path": "application.docker.conf" + } + ], + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-app-config" + } + } + ] + } + }, + { + "configMap": { + "defaultMode": 420, + "items": [ + { + "key": "default.conf", + "path": "default.conf" + } + ], + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-nginx-config" + }, + "name": "nginx-config" + }, + { + "configMap": { + "defaultMode": 420, + "items": [ + { + "key": "nginx.conf", + "path": "nginx.conf" + } + ], + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-nginx-main-config" + }, + "name": "nginx-main-config" + }, + { + "configMap": { + "defaultMode": 420, + "items": [ + { + "key": "prometheus.yml", + "path": "prometheus.yml" + } + ], + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-prometheus-config" + }, + "name": "prometheus-config" + }, + { + "configMap": { + "defaultMode": 420, + "items": [ + { + "key": "pg-upgrade-11-to-14.sh", + "path": "pg-upgrade-11-to-14.sh" + } + ], + "name": "yw-ns-jenk-i3082-20230214-211928-yugaware-pg-upgrade" + }, + "name": "pg-upgrade-11-to-14" + }, + { + "name": "kube-api-access-w2r5r", + "projected": { + "defaultMode": 420, + "sources": [ + { + "serviceAccountToken": { + "expirationSeconds": 3607, + "path": "token" + } + }, + { + "configMap": { + "items": [ + { + "key": "ca.crt", + "path": "ca.crt" + } + ], + "name": "kube-root-ca.crt" + } + }, + { + "downwardAPI": { + "items": [ + { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + }, + "path": "namespace" + } + ] + } + } + ] + } + } + ] + }, + "status": { + "conditions": [ + { + "lastProbeTime": null, + "lastTransitionTime": "2023-02-14T21:21:13Z", + "status": "True", + "type": "Initialized" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2023-02-14T21:21:16Z", + "status": "True", + "type": "Ready" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2023-02-14T21:21:16Z", + "status": "True", + "type": "ContainersReady" + }, + { + "lastProbeTime": null, + "lastTransitionTime": "2023-02-14T21:20:20Z", + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "containerd://8f91af66b9d1cf2a19243e571a4e57ad", + "image": "docker.io/nginxinc/nginx-unprivileged:1.23.3", + "imageID": "docker.io/nginxinc/nginx-unprivileged@sha256:097313923c0f29fd09", + "lastState": {}, + "name": "nginx", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2023-02-14T21:21:16Z" + } + } + }, + { + "containerID": "containerd://ea2e767a15935ebe6e08dfdab912df62030", + "image": "docker.io/library/postgres:14.6", + "imageID": "docker.io/library/postgres@sha256:f565573d74aedc9b218e1d191b04", + "lastState": {}, + "name": "postgres", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2023-02-14T21:21:13Z" + } + } + }, + { + "containerID": "containerd://3acd37fc33e8b81ed08617c80236d604122275", + "image": "docker.io/prom/prometheus:v2.41.0", + "imageID": "docker.io/prom/prometheus@sha256:1a3e9a", + "lastState": {}, + "name": "prometheus", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2023-02-14T21:21:14Z" + } + } + }, + { + "containerID": "containerd://d5ac6ec891bd71d2e6397a3970a00a41b463d6ff0291857", + "image": "quay.io/yugabyte/yugaware-itest:2.16.2.0-b25", + "imageID": "quay.io/yugabyte/yugaware-itest@sha256:3bc8039cbf", + "lastState": {}, + "name": "yugaware", + "ready": true, + "restartCount": 0, + "started": true, + "state": { + "running": { + "startedAt": "2023-02-14T21:21:15Z" + } + } + } + ], + "hostIP": "10.150.0.237", + "initContainerStatuses": [ + { + "containerID": "containerd://10e056ff9596b5fb1158332f4", + "image": "quay.io/yugabyte/yugaware-itest:2.16.2.0-b25", + "imageID": "quay.io/yugabyte/yugaware-itest@sha256:3bc8039cbfc99c94de95", + "lastState": {}, + "name": "prometheus-configuration", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "containerd://10e056ff9596b5fb1158332f40c1d5", + "exitCode": 0, + "finishedAt": "2023-02-14T21:21:02Z", + "reason": "Completed", + "startedAt": "2023-02-14T21:21:02Z" + } + } + }, + { + "containerID": "containerd://d5ca7ce756ff9f883c6609d21e66cef8a238100281f", + "image": "docker.io/tianon/postgres-upgrade:11-to-14", + "imageID": "docker.io/tianon/postgres-upgrade@sha256:35853b5b4968", + "lastState": {}, + "name": "postgres-upgrade", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "containerd://d5ca7ce756ff9f883c6609d21e", + "exitCode": 0, + "finishedAt": "2023-02-14T21:21:12Z", + "reason": "Completed", + "startedAt": "2023-02-14T21:21:12Z" + } + } + } + ], + "phase": "Running", + "podIP": "10.180.14.79", + "podIPs": [ + { + "ip": "10.180.14.79" + } + ], + "qosClass": "Burstable", + "startTime": "2023-02-14T21:20:20Z" + } +} From e686690c5acdc6b779e6b60eb72e6aecc7497d3f Mon Sep 17 00:00:00 2001 From: Adithya Bharadwaj Date: Wed, 1 Mar 2023 22:08:46 +0530 Subject: [PATCH 30/81] [#13163] CDCSDK: Making CDC checkpoint logs on tablet peer periodic Summary: Every time we set the 2DC, CDC checkpoint and CDC safetime in the tablet peer, we print an INFO log. This is done by the 'UpdatePeersAndMetrics' thread, which runs once every 15 seconds. Now we are making the logs a VLOG, so that they can only be turned on when required. Test Plan: NA Reviewers: skumar, hsunder Reviewed By: hsunder Subscribers: hsunder, ycdcxcluster Differential Revision: https://phabricator.dev.yugabyte.com/D23245 --- src/yb/tablet/tablet_peer.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/yb/tablet/tablet_peer.cc b/src/yb/tablet/tablet_peer.cc index 66f28cc86152..ef3441abee86 100644 --- a/src/yb/tablet/tablet_peer.cc +++ b/src/yb/tablet/tablet_peer.cc @@ -1074,7 +1074,7 @@ yb::OpId TabletPeer::GetLatestLogEntryOpId() const { } Status TabletPeer::set_cdc_min_replicated_index_unlocked(int64_t cdc_min_replicated_index) { - LOG_WITH_PREFIX(INFO) << "Setting cdc min replicated index to " << cdc_min_replicated_index; + VLOG(1) << "Setting cdc min replicated index to " << cdc_min_replicated_index; RETURN_NOT_OK(meta_->set_cdc_min_replicated_index(cdc_min_replicated_index)); Log* log = log_atomic_.load(std::memory_order_acquire); if (log) { @@ -1107,14 +1107,13 @@ int64_t TabletPeer::get_cdc_min_replicated_index() { } Status TabletPeer::set_cdc_sdk_min_checkpoint_op_id(const OpId& cdc_sdk_min_checkpoint_op_id) { - LOG_WITH_PREFIX(INFO) << "Setting CDCSDK min checkpoint opId to " - << cdc_sdk_min_checkpoint_op_id.ToString(); + VLOG(1) << "Setting CDCSDK min checkpoint opId to " << cdc_sdk_min_checkpoint_op_id.ToString(); RETURN_NOT_OK(meta_->set_cdc_sdk_min_checkpoint_op_id(cdc_sdk_min_checkpoint_op_id)); return Status::OK(); } Status TabletPeer::set_cdc_sdk_safe_time(const HybridTime& cdc_sdk_safe_time) { - LOG_WITH_PREFIX(INFO) << "Setting CDCSDK safe time to " << cdc_sdk_safe_time; + VLOG(1) << "Setting CDCSDK safe time to " << cdc_sdk_safe_time; RETURN_NOT_OK(meta_->set_cdc_sdk_safe_time(cdc_sdk_safe_time)); return Status::OK(); } From bfc509f813b8bfee0ed6be38992cc8d8b2b62ad7 Mon Sep 17 00:00:00 2001 From: Divyansh Chaturvedi Date: Thu, 2 Mar 2023 11:41:49 +0530 Subject: [PATCH 31/81] [#14209] YSQL : Create system view for yb_terminated_queries Summary: With this diff, we registered `yb_terminated_queries` as a system view by creating a migration for the same. `yb_terminated_queries` is a view that helps users in faster debugging in cases where Postgres terminates a query intentionally. It reports the following : - The Database Name - The Query Text - The Termination Reason - The Backend PID - The Query Start Time - The Query Termination Time More information on `yb_terminated_queries` can be found in * bb650487e5260159692a545713c2547f270fc2de Test Plan: `yb_build.sh --java-test TestPgYbStat` `yb_build.sh --java-test TestPgRegressYbStat` Reviewers: hbhanawat, telgersma, rvenkatesh Reviewed By: rvenkatesh Subscribers: yql, smishra Differential Revision: https://phabricator.dev.yugabyte.com/D22640 --- .../test/java/org/yb/pgsql/TestPgYbStat.java | 22 ------------- .../src/backend/catalog/yb_system_views.sql | 11 +++++++ .../src/include/catalog/pg_yb_migration.dat | 4 +-- .../src/test/regress/expected/yb_pg_rules.out | 8 +++++ .../src/test/regress/expected/yb_stat.out | 32 +----------------- src/postgres/src/test/regress/sql/yb_stat.sql | 33 ------------------- .../V33__14209__yb_terminated_queries.sql | 12 +++++++ 7 files changed, 34 insertions(+), 88 deletions(-) create mode 100644 src/yb/yql/pgwrapper/ysql_migrations/V33__14209__yb_terminated_queries.sql diff --git a/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgYbStat.java b/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgYbStat.java index a1d5ba77f321..54bcbcf4909d 100644 --- a/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgYbStat.java +++ b/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgYbStat.java @@ -49,22 +49,6 @@ private void executeQueryAndExpectNoResults(final String query, } } - // TODO: Will remove this when blocking upgrade issue lands that prohibits a system - // view from being properly added moving between different versions. - private void createQueryTerminationView(final Connection inputConnection) throws Exception { - String createViewQuery = "CREATE VIEW yb_terminated_queries AS " + - "SELECT " + - "D.datname AS databasename," + - "S.backend_pid AS backend_pid," + - "S.query_text AS query_text," + - "S.termination_reason AS termination_reason," + - "S.query_start AS query_start_time," + - "S.query_end AS query_end_time " + - "FROM yb_pg_stat_get_queries(NULL) AS S " + - "LEFT JOIN pg_database AS D ON (S.db_oid = D.oid);"; - executeQueryAndExpectNoResults(createViewQuery, inputConnection); - } - private void executeQueryAndExpectTempFileLimitExceeded(final String query, final Connection inputConnection) throws Exception { try (Statement statement = inputConnection.createStatement()) { @@ -184,8 +168,6 @@ public void testYbTerminatedQueriesMultipleCauses() throws Exception { executeQueryAndExpectTempFileLimitExceeded(oversized_query, connection); try (Statement statement = connection.createStatement()) { - createQueryTerminationView(connection); - // By current implementation, we expect that the queries will overflow from the end // of the array and start overwiting the oldest entries stored at the beginning of // the array. Consider this a circular buffer. @@ -238,8 +220,6 @@ public void testYbTerminatedQueriesOverflow() throws Exception { executeQueryAndExpectNoResults("SET work_mem TO 2048", connection); - createQueryTerminationView(connection); - // By current implementation, we expect that the queries will overflow from the end // of the array and start overwiting the oldest entries stored at the beginning of // the array. Consider this a circular buffer. @@ -283,8 +263,6 @@ public void testYBMultipleConnections() throws Exception { executeQueryAndExpectTempFileLimitExceeded(statement1, connection1); executeQueryAndExpectTempFileLimitExceeded(statement2, connection2); - createQueryTerminationView(connection1); - assertTrue(waitUntilConditionSatisfiedOrTimeout( "SELECT backend_pid, query_text FROM yb_terminated_queries", connection1, (ResultSet result) -> { diff --git a/src/postgres/src/backend/catalog/yb_system_views.sql b/src/postgres/src/backend/catalog/yb_system_views.sql index 1f245a92e62e..92498dbd7ac7 100644 --- a/src/postgres/src/backend/catalog/yb_system_views.sql +++ b/src/postgres/src/backend/catalog/yb_system_views.sql @@ -14,6 +14,17 @@ * string literal (including a function body!) or a multiline comment. */ +CREATE VIEW yb_terminated_queries AS +SELECT + D.datname AS databasename, + S.backend_pid AS backend_pid, + S.query_text AS query_text, + S.termination_reason AS termination_reason, + S.query_start AS query_start_time, + S.query_end AS query_end_time +FROM yb_pg_stat_get_queries(NULL) AS S +LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); + CREATE VIEW pg_roles AS SELECT rolname, diff --git a/src/postgres/src/include/catalog/pg_yb_migration.dat b/src/postgres/src/include/catalog/pg_yb_migration.dat index ec71f2e9f924..e6b399309524 100644 --- a/src/postgres/src/include/catalog/pg_yb_migration.dat +++ b/src/postgres/src/include/catalog/pg_yb_migration.dat @@ -12,7 +12,7 @@ [ # For better version control conflict detection, list latest migration filename -# here: V32__7376__pg_stat_activity_catalog_version.sql -{ major => '32', minor => '0', name => '', time_applied => '_null_' } +# here: V33__14209__yb_terminated_queries.sql +{ major => '33', minor => '0', name => '', time_applied => '_null_' } ] diff --git a/src/postgres/src/test/regress/expected/yb_pg_rules.out b/src/postgres/src/test/regress/expected/yb_pg_rules.out index cd3f86b17162..097ceca5fc4a 100644 --- a/src/postgres/src/test/regress/expected/yb_pg_rules.out +++ b/src/postgres/src/test/regress/expected/yb_pg_rules.out @@ -2399,6 +2399,14 @@ shoelace_obsolete| SELECT shoelace.sl_name, WHERE (NOT (EXISTS ( SELECT shoe.shoename FROM shoe WHERE (shoe.slcolor = shoelace.sl_color)))); +yb_terminated_queries| SELECT d.datname AS databasename, + s.backend_pid, + s.query_text, + s.termination_reason, + s.query_start AS query_start_time, + s.query_end AS query_end_time + FROM (yb_pg_stat_get_queries(NULL::oid) s(db_oid, backend_pid, query_text, termination_reason, query_start, query_end) + LEFT JOIN pg_database d ON ((s.db_oid = d.oid))); SELECT tablename, rulename, definition FROM pg_rules ORDER BY tablename, rulename; pg_settings|pg_settings_n|CREATE RULE pg_settings_n AS diff --git a/src/postgres/src/test/regress/expected/yb_stat.out b/src/postgres/src/test/regress/expected/yb_stat.out index 0708ed3ed42f..ef7e8f4c19c0 100644 --- a/src/postgres/src/test/regress/expected/yb_stat.out +++ b/src/postgres/src/test/regress/expected/yb_stat.out @@ -19,18 +19,8 @@ SELECT pg_sleep(1); (1 row) -CREATE view yb_terminated_queries AS - SELECT - D.datname AS databasename, - S.backend_pid AS backend_pid, - S.query_text AS query_text, - S.termination_reason AS termination_reason, - S.query_start AS query_start_time, - S.query_end AS query_end_time - FROM yb_pg_stat_get_queries(null) AS S - LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); \d yb_terminated_queries - View "public.yb_terminated_queries" + View "pg_catalog.yb_terminated_queries" Column | Type | Collation | Nullable | Default --------------------+--------------------------+-----------+----------+--------- databasename | name | | | @@ -56,16 +46,6 @@ SELECT databasename, termination_reason, query_text FROM yb_terminated_queries W CREATE DATABASE db2; \c db2 -CREATE view yb_terminated_queries AS - SELECT - D.datname AS databasename, - S.backend_pid AS backend_pid, - S.query_text AS query_text, - S.termination_reason AS termination_reason, - S.query_start AS query_start_time, - S.query_end AS query_end_time - FROM yb_pg_stat_get_queries(null) AS S - LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); SELECT databasename, termination_reason, query_text FROM yb_terminated_queries; databasename | termination_reason | query_text --------------+---------------------------------------------------+------------------------------------------------ @@ -219,16 +199,6 @@ ALTER ROLE test_user WITH createdb; \c yugabyte test_user CREATE DATABASE test_user_database; \c test_user_database test_user -CREATE view yb_terminated_queries AS - SELECT - D.datname AS databasename, - S.backend_pid AS backend_pid, - S.query_text AS query_text, - S.termination_reason AS termination_reason, - S.query_start AS query_start_time, - S.query_end AS query_end_time - FROM yb_pg_stat_get_queries(null) AS S - LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); SET work_mem TO 128; -- Some shenanigans with temp_file_limit means you need to be superuser -- to change the value of this config. diff --git a/src/postgres/src/test/regress/sql/yb_stat.sql b/src/postgres/src/test/regress/sql/yb_stat.sql index 5e09d267b65f..e4501e88e070 100644 --- a/src/postgres/src/test/regress/sql/yb_stat.sql +++ b/src/postgres/src/test/regress/sql/yb_stat.sql @@ -15,17 +15,6 @@ SELECT 'bob' FROM generate_series(0, 1000000); -- statistics. SELECT pg_sleep(1); -CREATE view yb_terminated_queries AS - SELECT - D.datname AS databasename, - S.backend_pid AS backend_pid, - S.query_text AS query_text, - S.termination_reason AS termination_reason, - S.query_start AS query_start_time, - S.query_end AS query_end_time - FROM yb_pg_stat_get_queries(null) AS S - LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); - \d yb_terminated_queries SELECT databasename, termination_reason, query_text FROM yb_terminated_queries; SELECT databasename, termination_reason, query_text FROM yb_terminated_queries WHERE databasename = 'yugabyte'; @@ -33,17 +22,6 @@ SELECT databasename, termination_reason, query_text FROM yb_terminated_queries W CREATE DATABASE db2; \c db2 -CREATE view yb_terminated_queries AS - SELECT - D.datname AS databasename, - S.backend_pid AS backend_pid, - S.query_text AS query_text, - S.termination_reason AS termination_reason, - S.query_start AS query_start_time, - S.query_end AS query_end_time - FROM yb_pg_stat_get_queries(null) AS S - LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); - SELECT databasename, termination_reason, query_text FROM yb_terminated_queries; SET work_mem TO 64; @@ -106,17 +84,6 @@ ALTER ROLE test_user WITH createdb; CREATE DATABASE test_user_database; \c test_user_database test_user -CREATE view yb_terminated_queries AS - SELECT - D.datname AS databasename, - S.backend_pid AS backend_pid, - S.query_text AS query_text, - S.termination_reason AS termination_reason, - S.query_start AS query_start_time, - S.query_end AS query_end_time - FROM yb_pg_stat_get_queries(null) AS S - LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); - SET work_mem TO 128; -- Some shenanigans with temp_file_limit means you need to be superuser diff --git a/src/yb/yql/pgwrapper/ysql_migrations/V33__14209__yb_terminated_queries.sql b/src/yb/yql/pgwrapper/ysql_migrations/V33__14209__yb_terminated_queries.sql new file mode 100644 index 000000000000..5788b53f7c92 --- /dev/null +++ b/src/yb/yql/pgwrapper/ysql_migrations/V33__14209__yb_terminated_queries.sql @@ -0,0 +1,12 @@ +SET LOCAL yb_non_ddl_txn_for_sys_tables_allowed TO true; + +CREATE OR REPLACE VIEW pg_catalog.yb_terminated_queries WITH (use_initdb_acl = true) AS +SELECT + D.datname AS databasename, + S.backend_pid AS backend_pid, + S.query_text AS query_text, + S.termination_reason AS termination_reason, + S.query_start AS query_start_time, + S.query_end AS query_end_time +FROM yb_pg_stat_get_queries(NULL) AS S +LEFT JOIN pg_database AS D ON (S.db_oid = D.oid); From 3b00dab134081957c54f674db16ccb7a5d624d61 Mon Sep 17 00:00:00 2001 From: Charles Wang Date: Mon, 27 Feb 2023 12:17:33 -0800 Subject: [PATCH 32/81] Revert "Revert "[PLAT-5734] Support EAR enabled/TLS universes for attach/detach"" Summary: This reverts commit 118798e7ecdce95f5be687079d0d041e79e80970. Test Plan: run all itests Reviewers: sanketh Reviewed By: sanketh Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23196 --- managed/devops/bin/yb_attach_detach.py | 28 +- managed/devops/replicated.yml | 1 - .../yugabyte/yw/common/ReleaseManager.java | 17 +- .../yw/common/certmgmt/CertificateHelper.java | 4 + .../common/kms/util/EncryptionAtRestUtil.java | 4 + .../controllers/AttachDetachController.java | 103 +++- .../yw/forms/DetachUniverseFormData.java | 7 + .../yugabyte/yw/models/CertificateInfo.java | 43 ++ .../com/yugabyte/yw/models/KmsHistory.java | 2 + .../java/com/yugabyte/yw/models/Universe.java | 5 +- .../com/yugabyte/yw/models/UniverseSpec.java | 485 ++++++++++++++++-- .../yw/models/helpers/CloudInfoInterface.java | 1 - .../src/main/resources/application.test.conf | 1 - managed/src/main/resources/reference.conf | 10 +- managed/src/main/resources/v1.routes | 2 +- .../tasks/CreateSupportBundleTest.java | 3 - .../AttachDetachControllerTest.java | 15 +- .../yugabyte/yw/models/UniverseSpecTest.java | 34 ++ 18 files changed, 674 insertions(+), 91 deletions(-) create mode 100644 managed/src/main/java/com/yugabyte/yw/forms/DetachUniverseFormData.java create mode 100644 managed/src/test/java/com/yugabyte/yw/models/UniverseSpecTest.java diff --git a/managed/devops/bin/yb_attach_detach.py b/managed/devops/bin/yb_attach_detach.py index c25d93024a94..685cf448a1cb 100644 --- a/managed/devops/bin/yb_attach_detach.py +++ b/managed/devops/bin/yb_attach_detach.py @@ -9,8 +9,10 @@ # https://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt import argparse import io +import json import logging import mimetypes +import os.path import re import shutil import urllib.request as urllib_request @@ -38,13 +40,15 @@ class YBAttachDetach: /Users/cwang/Downloads/output.tar.gz -p http://localhost:9000 -c f33e3c9b-75ab-4c30-80ad-cba85646ea39 -t ce5dd0f1-3e3f-4334-aa64-28f1c87cb3f2 """ - def __init__(self, action, universe_uuid, file, customer_uuid, api_token, platform_host): + def __init__(self, action, universe_uuid, file, customer_uuid, + api_token, platform_host, skip_releases): self.action = action self.universe_uuid = universe_uuid self.file = file self.customer_uuid = customer_uuid self.api_token = api_token self.platform_host = platform_host + self.skip_releases = skip_releases self.set_url_request_variables() def run(self): @@ -66,8 +70,12 @@ def run(self): logging.info("Completed attaching universe.") elif self.action == DETACH_ACTION: logging.info("Detaching universe...") + data = { + "skipReleases": self.skip_releases + } req = urllib_request.Request( - url=self.detach_url, method="POST", headers=self.default_headers) + url=self.detach_url, method="POST", headers=self.default_headers, + data=json.dumps(data).encode()) with urllib_request.urlopen(req) as response, open(self.file, "wb") as file: shutil.copyfileobj(response, file) logging.info("Completed detaching universe.") @@ -88,6 +96,9 @@ def validate_arguments(self): f"Invalid action passed in. Got {self.action}. " f"Expected one of: {VALID_ACTIONS}") + if (self.action == DETACH_ACTION and os.path.isfile(self.file)): + raise ValueError(f"File {self.file} already exists") + def set_url_request_variables(self): """ Use arguments passed in to generate required urls/headers @@ -98,7 +109,10 @@ def set_url_request_variables(self): f"{self.customer_uuid}/universes/{self.universe_uuid}/import") self.detach_url = (f"{self.base_url}/customers/" f"{self.customer_uuid}/universes/{self.universe_uuid}/export") - self.default_headers = {"X-AUTH-YW-API-TOKEN": self.api_token} + self.default_headers = { + "X-AUTH-YW-API-TOKEN": self.api_token, + "Content-Type": "application/json" + } logging.debug("Base url: %s", self.base_url) logging.debug("Detach url: %s", self.detach_url) logging.debug("Attach url: %s", self.attach_url) @@ -139,6 +153,10 @@ def parse_arguments(): parser.add_argument( "-p", "--platform_host", required=True, help="Base endpoint platform requests are sent to") + parser.add_argument( + "-s", "--skip_releases", action="store_true", + required=False, default=False, + help="For detach, whether or not do include software and ybc releases") args = parser.parse_args() logging.info("\n") @@ -149,6 +167,8 @@ def parse_arguments(): logging.info("Customer: %s", args.customer) logging.info("Api token: %s", args.api_token) logging.info("Platform host: %s", args.platform_host) + if (args.action == "detach_universe"): + logging.info("Skip releases: %s", args.skip_releases) logging.info("------------------------------------------------------") logging.info("\n") @@ -233,5 +253,5 @@ def __bytes__(self): cmd_args = parse_arguments() yb_attach_detach = YBAttachDetach( cmd_args.action, cmd_args.univ_uuid, cmd_args.file, - cmd_args.customer, cmd_args.api_token, cmd_args.platform_host) + cmd_args.customer, cmd_args.api_token, cmd_args.platform_host, cmd_args.skip_releases) yb_attach_detach.run() diff --git a/managed/devops/replicated.yml b/managed/devops/replicated.yml index 78eeb2f8185b..3208d76ce6f6 100644 --- a/managed/devops/replicated.yml +++ b/managed/devops/replicated.yml @@ -538,7 +538,6 @@ components: swamper.targetPath = /opt/yugabyte/prometheus/targets swamper.rulesPath = /opt/yugabyte/prometheus/rules multiTenant = false - releases.path = "/opt/yugabyte/releases" docker.release = "/opt/yugabyte/release" thirdparty.packagePath = /opt/third-party helm.packagePath = "/opt/yugabyte/helm" diff --git a/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java b/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java index 074feba370b1..921ae0215822 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/ReleaseManager.java @@ -4,6 +4,7 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; @@ -33,7 +34,6 @@ import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.Paths; -import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -41,7 +41,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Set; @@ -70,7 +69,7 @@ public class ReleaseManager { private static final String YB_PACKAGE_REGEX = "yugabyte-(?:ee-)?(.*)-(alma|centos|linux|el8|darwin)(.*).tar.gz"; - private final ConfigHelper configHelper; + public final ConfigHelper configHelper; private final Configuration appConfig; private final GFlagsValidation gFlagsValidation; private final Commissioner commissioner; @@ -293,6 +292,12 @@ public Boolean matchesRegion(Region region) { List matched = matchPackages(arch); return matched.size() > 0; } + + @ApiModelProperty(value = "local release", hidden = true) + @JsonIgnore + public boolean isLocalRelease() { + return !(s3 != null || gcs != null || http != null); + } } private Predicate getPackageFilter(String pathMatchGlob) { @@ -885,7 +890,7 @@ public void addGFlagsMetadataFiles(String version, ReleaseMetadata releaseMetada List missingGFlagsFilesList = gFlagsValidation.getMissingGFlagFileList(version); if (missingGFlagsFilesList.size() != 0) { String releasesPath = appConfig.getString(Util.YB_RELEASES_PATH); - if (isLocalRelease(releaseMetadata)) { + if (releaseMetadata.isLocalRelease()) { try (InputStream inputStream = getTarGZipDBPackageInputStream(version, releaseMetadata)) { gFlagsValidation.fetchGFlagFilesFromTarGZipInputStream( inputStream, version, missingGFlagsFilesList, releasesPath); @@ -1019,8 +1024,4 @@ public Map getReleases() { public boolean getInUse(String version) { return Universe.existsRelease(version); } - - private boolean isLocalRelease(ReleaseMetadata rm) { - return !(rm.s3 != null || rm.gcs != null || rm.http != null); - } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/certmgmt/CertificateHelper.java b/managed/src/main/java/com/yugabyte/yw/common/certmgmt/CertificateHelper.java index b68f3cf318aa..11d56f1c30ac 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/certmgmt/CertificateHelper.java +++ b/managed/src/main/java/com/yugabyte/yw/common/certmgmt/CertificateHelper.java @@ -139,6 +139,10 @@ public static String getCAKeyPath(String storagePath, UUID customerUUID, UUID ca CERT_PATH + "/ca.key.pem", storagePath, customerUUID.toString(), caCertUUID.toString()); } + public static String getCADirPath(String storagePath, UUID customerUUID, UUID caCertUUID) { + return String.format(CERT_PATH, storagePath, customerUUID.toString(), caCertUUID.toString()); + } + private static String generateUniqueRootCALabel(String nodePrefix, CertConfigType certType) { // Default the cert label with node prefix. diff --git a/managed/src/main/java/com/yugabyte/yw/common/kms/util/EncryptionAtRestUtil.java b/managed/src/main/java/com/yugabyte/yw/common/kms/util/EncryptionAtRestUtil.java index e963f17deb78..619bffa7c27b 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/kms/util/EncryptionAtRestUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/kms/util/EncryptionAtRestUtil.java @@ -169,6 +169,10 @@ public static boolean keyRefExists(UUID universeUUID, byte[] keyRef) { KmsHistoryId.TargetType.UNIVERSE_KEY); } + public static boolean keyRefExists(UUID universeUUID, String keyRef) { + return KmsHistory.entryExists(universeUUID, keyRef, KmsHistoryId.TargetType.UNIVERSE_KEY); + } + @Deprecated public static KmsHistory getActiveKey(UUID universeUUID) { KmsHistory activeHistory = null; diff --git a/managed/src/main/java/com/yugabyte/yw/controllers/AttachDetachController.java b/managed/src/main/java/com/yugabyte/yw/controllers/AttachDetachController.java index a395a70a0e72..4ffdcd3129ec 100644 --- a/managed/src/main/java/com/yugabyte/yw/controllers/AttachDetachController.java +++ b/managed/src/main/java/com/yugabyte/yw/controllers/AttachDetachController.java @@ -13,30 +13,42 @@ package com.yugabyte.yw.controllers; +import com.fasterxml.jackson.databind.JsonNode; import com.google.inject.Inject; import com.typesafe.config.Config; import com.yugabyte.yw.common.ConfigHelper; import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.common.ReleaseManager; +import com.yugabyte.yw.common.ReleaseManager.ReleaseMetadata; +import com.yugabyte.yw.common.SwamperHelper; import com.yugabyte.yw.common.config.ProviderConfKeys; import com.yugabyte.yw.common.config.RuntimeConfGetter; -import com.yugabyte.yw.forms.PlatformResults; -import com.yugabyte.yw.models.AccessKey; +import com.yugabyte.yw.common.kms.util.EncryptionAtRestUtil; +import com.yugabyte.yw.forms.DetachUniverseFormData; +import com.yugabyte.yw.models.CertificateInfo; import com.yugabyte.yw.models.Customer; import com.yugabyte.yw.models.InstanceType; +import com.yugabyte.yw.models.KmsConfig; +import com.yugabyte.yw.models.KmsHistory; import com.yugabyte.yw.models.PriceComponent; import com.yugabyte.yw.models.Provider; import com.yugabyte.yw.models.Universe; import com.yugabyte.yw.models.UniverseSpec; +import com.yugabyte.yw.models.UniverseSpec.PlatformPaths; import com.yugabyte.yw.models.XClusterConfig; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import play.api.libs.Files.TemporaryFile; import play.mvc.Http; import play.mvc.Result; +@Slf4j public class AttachDetachController extends AbstractPlatformController { @Inject private Config config; @@ -45,12 +57,28 @@ public class AttachDetachController extends AbstractPlatformController { @Inject private RuntimeConfGetter confGetter; + @Inject private ReleaseManager releaseManager; + + @Inject private SwamperHelper swamperHelper; + + private static final String STORAGE_PATH = "yb.storage.path"; + private static final String RELEASES_PATH = "yb.releases.path"; + private static final String YBC_RELEASE_PATH = "ybc.docker.release"; + private static final String YBC_RELEASES_PATH = "ybc.releases.path"; + public Result exportUniverse(UUID customerUUID, UUID universeUUID) throws IOException { + JsonNode requestBody = request().body().asJson(); + + DetachUniverseFormData detachUniverseFormData = + formFactory.getFormDataOrBadRequest(requestBody, DetachUniverseFormData.class); + log.debug("Universe spec will include releases: {}", !detachUniverseFormData.skipReleases); + Customer customer = Customer.getOrBadRequest(customerUUID); Universe universe = Universe.getOrBadRequest(universeUUID); Provider provider = Provider.getOrBadRequest( UUID.fromString(universe.getUniverseDetails().getPrimaryCluster().userIntent.provider)); + List instanceTypes = InstanceType.findByProvider( provider, @@ -66,11 +94,42 @@ public Result exportUniverse(UUID customerUUID, UUID universeUUID) throws IOExce "Detach universe currently does not support universes with xcluster replication set up."); } - List accessKeys = AccessKey.getAll(provider.uuid); - List priceComponents = PriceComponent.findByProvider(provider); - String storagePath = confGetter.getStaticConf().getString("yb.storage.path"); + List certificateInfoList = CertificateInfo.getCertificateInfoList(universe); + + List kmsHistoryList = + EncryptionAtRestUtil.getAllUniverseKeys(universe.universeUUID); + kmsHistoryList.sort((h1, h2) -> h1.timestamp.compareTo(h2.timestamp)); + List kmsConfigs = + kmsHistoryList + .stream() + .map(kmsHistory -> kmsHistory.configUuid) + .distinct() + .map(c -> KmsConfig.get(c)) + .collect(Collectors.toList()); + + // Non-local releases will no be populated by importLocalReleases, so we need to add it + // ourselves. + ReleaseMetadata ybReleaseMetadata = + releaseManager.getReleaseByVersion( + universe.getUniverseDetails().getPrimaryCluster().userIntent.ybSoftwareVersion); + if (ybReleaseMetadata != null && ybReleaseMetadata.isLocalRelease()) { + ybReleaseMetadata = null; + } + + String storagePath = confGetter.getStaticConf().getString(STORAGE_PATH); + String releasesPath = confGetter.getStaticConf().getString(RELEASES_PATH); + String ybcReleasePath = confGetter.getStaticConf().getString(YBC_RELEASE_PATH); + String ybcReleasesPath = confGetter.getStaticConf().getString(YBC_RELEASES_PATH); + + PlatformPaths platformPaths = + PlatformPaths.builder() + .storagePath(storagePath) + .releasesPath(releasesPath) + .ybcReleasePath(ybcReleasePath) + .ybcReleasesPath(ybcReleasesPath) + .build(); UniverseSpec universeSpec = UniverseSpec.builder() @@ -78,9 +137,13 @@ public Result exportUniverse(UUID customerUUID, UUID universeUUID) throws IOExce .universeConfig(universe.getConfig()) .provider(provider) .instanceTypes(instanceTypes) - .accessKeys(accessKeys) .priceComponents(priceComponents) - .oldStoragePath(storagePath) + .certificateInfoList(certificateInfoList) + .kmsHistoryList(kmsHistoryList) + .kmsConfigs(kmsConfigs) + .ybReleaseMetadata(ybReleaseMetadata) + .oldPlatformPaths(platformPaths) + .skipReleases(detachUniverseFormData.skipReleases) .build(); InputStream is = universeSpec.exportSpec(); @@ -94,14 +157,32 @@ public Result importUniverse(UUID customerUUID, UUID universeUUID) throws IOExce Http.MultipartFormData body = request().body().asMultipartFormData(); Http.MultipartFormData.FilePart tempSpecFile = body.getFile("spec"); + if (Universe.maybeGet(universeUUID).isPresent()) { + throw new PlatformServiceException( + BAD_REQUEST, + String.format("Universe with uuid %s already exists", universeUUID.toString())); + } + if (tempSpecFile == null) { throw new PlatformServiceException(BAD_REQUEST, "Failed to get uploaded spec file"); } - String storagePath = confGetter.getStaticConf().getString("yb.storage.path"); + String storagePath = confGetter.getStaticConf().getString(STORAGE_PATH); + String releasesPath = confGetter.getStaticConf().getString(RELEASES_PATH); + String ybcReleasePath = confGetter.getStaticConf().getString(YBC_RELEASE_PATH); + String ybcReleasesPath = confGetter.getStaticConf().getString(YBC_RELEASES_PATH); + + PlatformPaths platformPaths = + PlatformPaths.builder() + .storagePath(storagePath) + .releasesPath(releasesPath) + .ybcReleasePath(ybcReleasePath) + .ybcReleasesPath(ybcReleasesPath) + .build(); + File tempFile = (File) tempSpecFile.getFile(); - UniverseSpec universeSpec = UniverseSpec.importSpec(tempFile, storagePath, customer); - universeSpec.save(storagePath); - return PlatformResults.withData(universeSpec); + UniverseSpec universeSpec = UniverseSpec.importSpec(tempFile, platformPaths, customer); + universeSpec.save(platformPaths, releaseManager, swamperHelper); + return ok(); } } diff --git a/managed/src/main/java/com/yugabyte/yw/forms/DetachUniverseFormData.java b/managed/src/main/java/com/yugabyte/yw/forms/DetachUniverseFormData.java new file mode 100644 index 000000000000..5d170999f757 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/forms/DetachUniverseFormData.java @@ -0,0 +1,7 @@ +// Copyright (c) Yugabyte, Inc. + +package com.yugabyte.yw.forms; + +public class DetachUniverseFormData { + public boolean skipReleases = false; +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/CertificateInfo.java b/managed/src/main/java/com/yugabyte/yw/models/CertificateInfo.java index deeffafa17b5..394d02b5b0b5 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/CertificateInfo.java +++ b/managed/src/main/java/com/yugabyte/yw/models/CertificateInfo.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.yugabyte.yw.common.PlatformServiceException; @@ -19,6 +20,7 @@ import com.yugabyte.yw.common.kms.util.hashicorpvault.HashicorpVaultConfigParams; import com.yugabyte.yw.common.utils.FileUtils; import com.yugabyte.yw.forms.CertificateParams; +import com.yugabyte.yw.forms.UniverseDefinitionTaskParams; import com.yugabyte.yw.models.helpers.CommonUtils; import io.ebean.Finder; import io.ebean.Model; @@ -33,6 +35,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -349,6 +352,16 @@ public static CertificateInfo get(UUID certUUID) { return find.byId(certUUID); } + public static Optional maybeGet(UUID certUUID) { + // Find the CertificateInfo. + CertificateInfo certificateInfo = find.byId(certUUID); + if (certificateInfo == null) { + LOG.trace("Cannot find certificateInfo {}", certUUID); + return Optional.empty(); + } + return Optional.of(certificateInfo); + } + public static CertificateInfo getOrBadRequest(UUID certUUID, UUID customerUUID) { CertificateInfo certificateInfo = get(certUUID); if (certificateInfo == null) { @@ -451,6 +464,7 @@ public void setInUse(boolean inUse) { @ApiModelProperty( value = "Associated universe details for the certificate", accessMode = READ_ONLY) + @JsonProperty public List getUniverseDetails() { if (universeDetailSubsets == null) { Set universes = Universe.universeDetailsIfCertsExists(this.uuid, this.customerUUID); @@ -460,6 +474,7 @@ public List getUniverseDetails() { } } + @JsonIgnore public void setUniverseDetails(List universeDetailSubsets) { this.universeDetailSubsets = universeDetailSubsets; } @@ -530,4 +545,32 @@ public void checkEditable(UUID certUUID, UUID customerUUID) { BAD_REQUEST, "Cannot edit pre-customized cert. Create a new one."); } } + + public static List getCertificateInfoList(Universe universe) { + List certificateInfoList = new ArrayList(); + UUID rootCA = null; + UUID clientRootCA = null; + UniverseDefinitionTaskParams universeDetails = universe.getUniverseDetails(); + if (EncryptionInTransitUtil.isRootCARequired(universeDetails)) { + rootCA = universeDetails.rootCA; + if (rootCA == null) { + throw new RuntimeException("No valid RootCA found for " + universeDetails.universeUUID); + } + certificateInfoList.add(CertificateInfo.get(rootCA)); + } + + if (EncryptionInTransitUtil.isClientRootCARequired(universeDetails)) { + clientRootCA = universeDetails.getClientRootCA(); + if (clientRootCA == null) { + throw new RuntimeException( + "No valid clientRootCA found for " + universeDetails.universeUUID); + } + + // check against the root to see if need to export + if (!clientRootCA.equals(rootCA)) { + certificateInfoList.add(CertificateInfo.get(clientRootCA)); + } + } + return certificateInfoList; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java b/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java index 2f5782783365..d847048b0356 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java +++ b/managed/src/main/java/com/yugabyte/yw/models/KmsHistory.java @@ -12,6 +12,7 @@ import static io.swagger.annotations.ApiModelProperty.AccessMode.READ_ONLY; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.ebean.Ebean; import io.ebean.Finder; import io.ebean.Model; @@ -81,6 +82,7 @@ public static KmsHistory createKmsHistory( return keyHistory; } + @JsonIgnore public static void setKeyRefStatus( UUID targetUUID, UUID confidUUID, diff --git a/managed/src/main/java/com/yugabyte/yw/models/Universe.java b/managed/src/main/java/com/yugabyte/yw/models/Universe.java index e16ae2983928..74fa74b90a43 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/Universe.java +++ b/managed/src/main/java/com/yugabyte/yw/models/Universe.java @@ -739,7 +739,7 @@ public String getCertificateClientToNode() { UniverseDefinitionTaskParams details = this.getUniverseDetails(); if (details.getPrimaryCluster().userIntent.enableClientToNodeEncrypt) { // This means there must be a root CA associated with it. - if (details.rootAndClientRootCASame) { + if (details.rootAndClientRootCASame && details.rootCA != null) { return CertificateInfo.get(details.rootCA).certificate; } return CertificateInfo.get(details.getClientRootCA()).certificate; @@ -929,6 +929,7 @@ public static Cluster getCluster(Universe universe, String nodeName) { * @return the host (private_ip) and port of the current master leader in the universe or null if * not found */ + @JsonIgnore public HostAndPort getMasterLeader() { final String masterAddresses = getMasterAddresses(); final String cert = getCertificateNodetoNode(); @@ -948,6 +949,7 @@ public HostAndPort getMasterLeader() { * * @return NodeDetails of the master leader */ + @JsonIgnore public NodeDetails getMasterLeaderNode() { return getNodeByPrivateIP(getMasterLeaderHostText()); } @@ -958,6 +960,7 @@ public NodeDetails getMasterLeaderNode() { * @return a String of the private_ip of the current master leader in the universe or an empty * string if not found */ + @JsonIgnore public String getMasterLeaderHostText() { final HostAndPort masterLeader = getMasterLeader(); if (masterLeader == null) return ""; diff --git a/managed/src/main/java/com/yugabyte/yw/models/UniverseSpec.java b/managed/src/main/java/com/yugabyte/yw/models/UniverseSpec.java index 1c771e8c7495..cbc604dea803 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/UniverseSpec.java +++ b/managed/src/main/java/com/yugabyte/yw/models/UniverseSpec.java @@ -13,14 +13,23 @@ package com.yugabyte.yw.models; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import com.yugabyte.yw.commissioner.Common; +import com.yugabyte.yw.common.ReleaseManager; +import com.yugabyte.yw.common.ReleaseManager.ReleaseMetadata; +import com.yugabyte.yw.common.SwamperHelper; import com.yugabyte.yw.common.Util; +import com.yugabyte.yw.common.certmgmt.CertificateHelper; +import com.yugabyte.yw.common.kms.util.EncryptionAtRestUtil; import com.yugabyte.yw.common.utils.FileUtils; -import com.yugabyte.yw.models.helpers.CloudInfoInterface; - +import com.yugabyte.yw.models.helpers.provider.GCPCloudInfo; +import com.yugabyte.yw.models.helpers.provider.KubernetesInfo; +import com.yugabyte.yw.models.helpers.provider.region.KubernetesRegionInfo; import io.ebean.annotation.Transactional; import java.io.BufferedOutputStream; import java.io.File; @@ -36,6 +45,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.zip.GZIPOutputStream; import lombok.Builder; import lombok.Data; @@ -43,6 +55,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; import play.libs.Json; @Slf4j @@ -57,21 +70,29 @@ public class UniverseSpec { public Universe universe; - // Already consists of regions and regions contain availability zones + // Contains Region, AvailabilityZone, and AccessKey entities. public Provider provider; - public List accessKeys; - public List instanceTypes; public List priceComponents; + public List certificateInfoList; + + public List kmsConfigs; + + public List kmsHistoryList; + public Map universeConfig; - private String oldStoragePath; + public PlatformPaths oldPlatformPaths; + + private ReleaseMetadata ybReleaseMetadata; + + private boolean skipReleases; public InputStream exportSpec() throws IOException { - String specBasePath = this.oldStoragePath + "/universe-specs/export"; + String specBasePath = this.oldPlatformPaths.storagePath + "/universe-specs/export"; String specName = UniverseSpec.generateSpecName(true); String specJsonPath = specBasePath + "/" + specName + ".json"; String specTarGZPath = specBasePath + "/" + specName + ".tar.gz"; @@ -88,10 +109,24 @@ public InputStream exportSpec() throws IOException { // Save universe spec. Util.copyFileToTarGZ(jsonSpecFile, "universe-spec.json", tarOS); - // Save access keys. - for (AccessKey accessKey : accessKeys) { + // Save access key files. + for (AccessKey accessKey : this.provider.allAccessKeys) { exportAccessKey(accessKey, tarOS); } + if (this.provider.code.equals(Common.CloudType.kubernetes.toString())) { + exportKubernetesAccessKey(tarOS); + } + + // Save certificate files. + for (CertificateInfo certificateInfo : certificateInfoList) { + exportCertificateInfo(certificateInfo, tarOS); + } + + // Save ybc and software release files. + if (!this.skipReleases) { + exportYBSoftwareReleases(tarOS); + exportYbcReleases(tarOS); + } } Files.deleteIfExists(jsonSpecFile.toPath()); @@ -103,6 +138,7 @@ public InputStream exportSpec() throws IOException { private void exportUniverseSpecObj(File jsonSpecFile) throws IOException { ObjectMapper mapper = Json.mapper(); ObjectNode universeSpecObj = (ObjectNode) Json.toJson(this); + log.debug("Finished serializing universeSpec object."); universeSpecObj = setIgnoredJsonProperties(universeSpecObj); mapper.writeValue(jsonSpecFile, universeSpecObj); } @@ -113,36 +149,92 @@ private void exportAccessKey(AccessKey accessKey, TarArchiveOutputStream tarArch File pubKeyFile = new File(pubKeyPath); File accessKeyFolder = pubKeyFile.getParentFile(); Util.addFilesToTarGZ(accessKeyFolder.getAbsolutePath(), "keys/", tarArchive); + log.debug("Added accessKey {} to tar gz file.", accessKey.getKeyCode()); + } + + private void exportKubernetesAccessKey(TarArchiveOutputStream tarArchive) throws IOException { + KubernetesInfo kubernetesInfo = + this.provider.getProviderDetails().getCloudInfo().getKubernetes(); + String kubernetesPullSecretPath = kubernetesInfo.getKubernetesPullSecret(); + File kubernetesPullSecretFile = new File(kubernetesPullSecretPath); + File accessKeyFolder = kubernetesPullSecretFile.getParentFile(); + Util.addFilesToTarGZ(accessKeyFolder.getAbsolutePath(), "keys/", tarArchive); + log.debug("Added kubernetes accessKey {} to tar gz file.", kubernetesPullSecretFile.getName()); + } + + // Certs are stored under {yb.storage.path}/certs/{customer_uuid}/{certificate_info_uuid}/ + // We will store each certificate folder in tar gz under certs/ without customer_uuid for + // simplicity. + // Legacy yugabytedb.crt/yugabytedb.pem files will be included as they are stored under + // certificateFolder (usually in n2n rootCA folder). + private void exportCertificateInfo( + CertificateInfo certificateInfo, TarArchiveOutputStream tarArchive) throws IOException { + String certificatePath = certificateInfo.certificate; + File certificateFile = new File(certificatePath); + File certificateFolder = certificateFile.getParentFile(); + Util.addFilesToTarGZ(certificateFolder.getAbsolutePath(), "certs/", tarArchive); + log.debug("Added certificate {} to tar gz file.", certificateInfo.label); + } + + private void exportYBSoftwareReleases(TarArchiveOutputStream tarArchive) throws IOException { + String universeVersion = + this.universe.getUniverseDetails().getPrimaryCluster().userIntent.ybSoftwareVersion; + File releaseFolder = + new File(String.format("%s/%s", this.oldPlatformPaths.releasesPath, universeVersion)); + Util.addFilesToTarGZ(releaseFolder.getAbsolutePath(), "releases/", tarArchive); + log.debug("Added software release {} to tar gz file.", universeVersion); + } + + private void exportYbcReleases(TarArchiveOutputStream tarArchive) throws IOException { + File ybcReleaseFolder = new File(this.oldPlatformPaths.ybcReleasePath); + String universeYbcVersion = this.universe.getUniverseDetails().ybcSoftwareVersion; + if (this.universe.getUniverseDetails().ybcInstalled && ybcReleaseFolder.isDirectory()) { + File[] ybcTarFiles = ybcReleaseFolder.listFiles(); + Pattern ybcVersionPattern = + Pattern.compile(String.format("%s-", Pattern.quote(universeYbcVersion))); + if (ybcTarFiles != null) { + // Need to add folder to tar gz file, otherwise, there are issues with untar. + tarArchive.putArchiveEntry(tarArchive.createArchiveEntry(ybcReleaseFolder, "ybcRelease/")); + tarArchive.closeArchiveEntry(); + + for (File ybcTarFile : ybcTarFiles) { + Matcher matcher = ybcVersionPattern.matcher(ybcTarFile.getName()); + boolean matchFound = matcher.find(); + if (matchFound) { + Util.addFilesToTarGZ(ybcTarFile.getAbsolutePath(), "ybcRelease/", tarArchive); + } + } + } + log.debug("Added ybc release {} to tar gz file.", universeYbcVersion); + } } - // Add any member variables that have the @jsonIgnored annotation. + // Retrieve unmasked details from provider, region, and availability zone. public ObjectNode setIgnoredJsonProperties(ObjectNode universeSpecObj) { - Map envVars = CloudInfoInterface.fetchEnvVars(provider); - JsonNode providerUnmaskedConfig = Json.toJson(envVars); + ProviderDetails unmaskedProviderDetails = this.provider.getProviderDetails(); + JsonNode unmaskedProviderDetailsJson = Json.toJson(unmaskedProviderDetails); ObjectNode providerObj = (ObjectNode) universeSpecObj.get("provider"); - providerObj.set("config", providerUnmaskedConfig); + providerObj.set("details", unmaskedProviderDetailsJson); List regions = this.provider.regions; ArrayNode regionsObj = (ArrayNode) providerObj.get("regions"); - if (regionsObj.isArray()) { for (int i = 0; i < regions.size(); i++) { ObjectNode regionObj = (ObjectNode) regionsObj.get(i); Region region = regions.get(i); - envVars = CloudInfoInterface.fetchEnvVars(region); - JsonNode regionUnmaskedConfig = Json.toJson(envVars); - regionObj.set("config", regionUnmaskedConfig); + RegionDetails unmaskedRegionDetails = region.getRegionDetails(); + JsonNode unmaskedRegionDetailsJson = Json.toJson(unmaskedRegionDetails); + regionObj.set("details", unmaskedRegionDetailsJson); List zones = region.zones; ArrayNode zonesObj = (ArrayNode) regionObj.get("zones"); - if (zonesObj.isArray()) { for (int j = 0; j < zones.size(); j++) { ObjectNode zoneObj = (ObjectNode) zonesObj.get(j); AvailabilityZone zone = zones.get(j); - envVars = CloudInfoInterface.fetchEnvVars(zone); - JsonNode zoneUnmaskedConfig = Json.toJson(envVars); - zoneObj.set("config", zoneUnmaskedConfig); + AvailabilityZoneDetails unmaskedAZDetails = zone.getAvailabilityZoneDetails(); + JsonNode unmaskedAZDetailsJson = Json.toJson(unmaskedAZDetails); + zoneObj.set("details", unmaskedAZDetailsJson); } } } @@ -150,8 +242,14 @@ public ObjectNode setIgnoredJsonProperties(ObjectNode universeSpecObj) { return universeSpecObj; } - public static UniverseSpec importSpec(File tarFile, String storagePath, Customer customer) - throws IOException { + public static UniverseSpec importSpec( + File tarFile, PlatformPaths platformPaths, Customer customer) throws IOException { + + String storagePath = platformPaths.storagePath; + String releasesPath = platformPaths.releasesPath; + String ybcReleasePath = platformPaths.ybcReleasePath; + String ybcReleasesPath = platformPaths.ybcReleasesPath; + String specBasePath = storagePath + "/universe-specs/import"; String specName = UniverseSpec.generateSpecName(false); String specFolderPath = specBasePath + "/" + specName; @@ -162,12 +260,25 @@ public static UniverseSpec importSpec(File tarFile, String storagePath, Customer // Retrieve universe spec. UniverseSpec universeSpec = UniverseSpec.importUniverseSpec(specFolderPath); - // Update spec with new customer information. - universeSpec.updateUniverseCustomerDetails(customer); - // Copy access keys to correct location if existing provider does not exist. universeSpec.importAccessKeys(specFolderPath, storagePath); + // Copy certificate files to correct location. + universeSpec.importCertificateInfoList(specFolderPath, storagePath, customer.getUuid()); + + if (!universeSpec.skipReleases) { + // Import the ybsoftwareversions, etc if exists. + universeSpec.importSoftwareReleases(specFolderPath, releasesPath); + + // Import the ybcsoftware version, etc if exists. + universeSpec.importYbcSoftwareReleases(specFolderPath, ybcReleasePath); + universeSpec.createYbcReleasesFolder(ybcReleasesPath); + } + + // Update universe related metadata due to platform switch. + // Placed after copying files due to dependencies upon files existing in correct location. + universeSpec.updateUniverseMetadata(storagePath, customer); + return universeSpec; } @@ -175,6 +286,7 @@ private static UniverseSpec importUniverseSpec(String specFolderPath) throws IOE File jsonDir = new File(specFolderPath + "/universe-spec.json"); ObjectMapper mapper = Json.mapper(); UniverseSpec universeSpec = mapper.readValue(jsonDir, UniverseSpec.class); + log.debug("Finished deserializing universe spec."); return universeSpec; } @@ -187,36 +299,228 @@ private void importAccessKeys(String specFolderPath, String storagePath) throws File[] keyFiles = keyDir.listFiles(); if (keyFiles != null) { for (File keyFile : keyFiles) { - String extension = FilenameUtils.getExtension(keyFile.getName()); - Set permissions = - PosixFilePermissions.fromString(DEFAULT_PERMISSIONS); - if (extension.equals("pem")) { - permissions = PosixFilePermissions.fromString(PEM_PERMISSIONS); - } else if (extension.equals("pub")) { - permissions = PosixFilePermissions.fromString(PUB_PERMISSIONS); + // K8s universes have kubeconfig accessKey on per az level, which will be a directory. + if (!keyFile.isDirectory()) { + String extension = FilenameUtils.getExtension(keyFile.getName()); + Set permissions = + PosixFilePermissions.fromString(DEFAULT_PERMISSIONS); + if (extension.equals("pem")) { + permissions = PosixFilePermissions.fromString(PEM_PERMISSIONS); + } else if (extension.equals("pub")) { + permissions = PosixFilePermissions.fromString(PUB_PERMISSIONS); + } + Files.setPosixFilePermissions(keyFile.toPath(), permissions); } - Files.setPosixFilePermissions(keyFile.toPath(), permissions); } } + log.debug("Saved access key folder {}.", keyDir.getName()); } } File destKeyMasterDir = new File(storagePath + "/keys"); org.apache.commons.io.FileUtils.copyDirectory(srcKeyMasterDir, destKeyMasterDir); - log.debug("Saved access keys to {}", destKeyMasterDir.getPath()); + log.debug("Saved access key file to {}", destKeyMasterDir.getPath()); + } + } + + private void importCertificateInfoList( + String specFolderPath, String storagePath, UUID customerUUID) throws IOException { + File srcCertificateInfoMasterDir = new File(specFolderPath + "/certs"); + File[] certificateInfoDirs = srcCertificateInfoMasterDir.listFiles(); + + if (certificateInfoDirs != null) { + for (File certificateInfoDir : certificateInfoDirs) { + String certificateInfoUUID = certificateInfoDir.getName(); + String certificateInfoBasePath = + CertificateHelper.getCADirPath( + storagePath, customerUUID, UUID.fromString(certificateInfoUUID)); + log.debug("Current certificate directory {}", certificateInfoBasePath); + Files.createDirectories(Paths.get(certificateInfoBasePath)); + + File[] certificateInfoFiles = certificateInfoDir.listFiles(); + if (certificateInfoFiles != null) { + for (File certificateInfoFile : certificateInfoFiles) { + Set permissions = + PosixFilePermissions.fromString(DEFAULT_PERMISSIONS); + Files.setPosixFilePermissions(certificateInfoFile.toPath(), permissions); + } + } + + File destCertificateInfoDir = new File(certificateInfoBasePath); + org.apache.commons.io.FileUtils.copyDirectory(certificateInfoDir, destCertificateInfoDir); + log.debug("Save certificate info folder {}.", certificateInfoUUID); + } + } + } + + private void importSoftwareReleases(String specFolderPath, String releasesPath) + throws IOException { + String universeVersion = + this.universe.getUniverseDetails().getPrimaryCluster().userIntent.ybSoftwareVersion; + File srcReleasesDir = + new File(String.format("%s/releases/%s", specFolderPath, universeVersion)); + + if (srcReleasesDir.isDirectory()) { + File destReleasesDir = new File(String.format("%s/%s", releasesPath, universeVersion)); + Files.createDirectories(Paths.get(releasesPath)); + org.apache.commons.io.FileUtils.copyDirectory(srcReleasesDir, destReleasesDir); + log.debug("Finished importing software release {}.", universeVersion); + } + } + + private void importYbcSoftwareReleases(String specFolderPath, String ybcReleasePath) + throws IOException { + File srcYbcReleaseDir = new File(String.format("%s/%s", specFolderPath, "ybcRelease")); + File destYbcReleaseDir = new File(ybcReleasePath); + if (srcYbcReleaseDir.isDirectory()) { + org.apache.commons.io.FileUtils.copyDirectory(srcYbcReleaseDir, destYbcReleaseDir); + log.debug("Finished importing ybc software release."); } } - public void updateUniverseCustomerDetails(Customer customer) { + private void createYbcReleasesFolder(String ybcReleasesPath) throws IOException { + if (!(new File(ybcReleasesPath)).isDirectory()) { + Files.createDirectories(Paths.get(ybcReleasesPath)); + log.debug("Created ybc releases folder as it was not found."); + } + } + + private void updateUniverseDetails(Customer customer) { Long customerId = customer.getCustomerId(); - universe.customerId = customerId; + this.universe.customerId = customerId; + + this.universe.setConfig(this.universeConfig); + } + + private void updateProviderDetails(String storagePath, Customer customer) { + // Use new customer. + provider.customerUUID = customer.uuid; + + if (this.provider.code.equals(Common.CloudType.kubernetes.toString())) { + // Update abs. path for kubernetesPullSecret to use new yb.storage.path. + KubernetesInfo kubernetesInfo = + this.provider.getProviderDetails().getCloudInfo().getKubernetes(); + kubernetesInfo.setKubernetesPullSecret( + UniverseSpec.replaceBeginningPath( + kubernetesInfo.getKubernetesPullSecret(), + this.oldPlatformPaths.storagePath, + storagePath)); + + // Update abs. path for kubeConfig to use new yb.storage.path. + for (Region region : Region.getByProvider(this.provider.uuid)) { + for (AvailabilityZone az : AvailabilityZone.getAZsForRegion(region.uuid)) { + KubernetesRegionInfo kubernetesRegionInfo = + az.getAvailabilityZoneDetails().getCloudInfo().getKubernetes(); + kubernetesRegionInfo.setKubeConfig( + UniverseSpec.replaceBeginningPath( + kubernetesRegionInfo.getKubeConfig(), + this.oldPlatformPaths.storagePath, + storagePath)); + } + } + } else if (this.provider.code.equals(Common.CloudType.gcp.toString())) { + // Update abs. path for credentials.json to use new yb.storage.path. + GCPCloudInfo gcpCloudInfo = this.provider.getProviderDetails().getCloudInfo().getGcp(); + gcpCloudInfo.setGceApplicationCredentialsPath( + UniverseSpec.replaceBeginningPath( + gcpCloudInfo.getGceApplicationCredentialsPath(), + this.oldPlatformPaths.storagePath, + storagePath)); + } + } + + private void updateCertificateInfoDetails(String storagePath, Customer customer) { + UUID customerUUID = customer.getUuid(); + for (CertificateInfo certificateInfo : this.certificateInfoList) { + UUID oldCustomerUUID = certificateInfo.customerUUID; + certificateInfo.customerUUID = customerUUID; + + // HashicorpVault does not have privateKey. + if (!StringUtils.isEmpty(certificateInfo.privateKey)) { + certificateInfo.privateKey = + UniverseSpec.replaceBeginningPath( + certificateInfo.privateKey, this.oldPlatformPaths.storagePath, storagePath); + certificateInfo.privateKey = + certificateInfo.privateKey.replaceFirst( + oldCustomerUUID.toString(), customerUUID.toString()); + } + + certificateInfo.certificate = + UniverseSpec.replaceBeginningPath( + certificateInfo.certificate, this.oldPlatformPaths.storagePath, storagePath); + certificateInfo.certificate = + certificateInfo.certificate.replaceFirst( + oldCustomerUUID.toString(), customerUUID.toString()); + } + } + + // Update absolute path to access keys due to yb.storage.path changes. + private void updateAccessKeyDetails(String storagePath) { + for (AccessKey key : this.provider.allAccessKeys) { + key.getKeyInfo().publicKey = + UniverseSpec.replaceBeginningPath( + key.getKeyInfo().publicKey, this.oldPlatformPaths.storagePath, storagePath); + key.getKeyInfo().privateKey = + UniverseSpec.replaceBeginningPath( + key.getKeyInfo().privateKey, this.oldPlatformPaths.storagePath, storagePath); + key.getKeyInfo().vaultPasswordFile = + UniverseSpec.replaceBeginningPath( + key.getKeyInfo().vaultPasswordFile, this.oldPlatformPaths.storagePath, storagePath); + key.getKeyInfo().vaultFile = + UniverseSpec.replaceBeginningPath( + key.getKeyInfo().vaultFile, this.oldPlatformPaths.storagePath, storagePath); + } + } + + private void updateKmsConfigDetails(Customer customer) { + for (KmsConfig kmsConfig : this.kmsConfigs) { + kmsConfig.customerUUID = customer.uuid; + } + } + + private void updateUniverseMetadata(String storagePath, Customer customer) { + + // Update universe information with new customer information and universe config. + updateUniverseDetails(customer); + + // Update provider information with new customer information and k8s specific file paths. + updateProviderDetails(storagePath, customer); + + // Update certificate file paths. + updateCertificateInfoDetails(storagePath, customer); + + // Update access key file paths. + updateAccessKeyDetails(storagePath); + + updateKmsConfigDetails(customer); } @Transactional - public void save(String storagePath) { - // Check if provider exists, if not, save all entities related to provider. + public void save( + PlatformPaths platformPaths, ReleaseManager releaseManager, SwamperHelper swamperHelper) { + + // Check if provider exists, if not, save all entities related to provider (Region, + // AvailabilityZone, AccessKey). if (!Provider.maybeGet(this.provider.uuid).isPresent()) { this.provider.save(); + if (this.provider.code.equals(Common.CloudType.kubernetes.toString())) { + // Kubernetes provider contains kubernetesPullSecret file. + FileData.writeFileToDB( + this.provider + .getProviderDetails() + .getCloudInfo() + .getKubernetes() + .getKubernetesPullSecret()); + + // Each az contains its own kubeConfig. + for (Region region : Region.getByProvider(this.provider.uuid)) { + for (AvailabilityZone az : AvailabilityZone.getAZsForRegion(region.uuid)) { + FileData.writeFileToDB( + az.getAvailabilityZoneDetails().getCloudInfo().getKubernetes().getKubeConfig()); + } + } + } + for (InstanceType instanceType : this.instanceTypes) { instanceType.save(); } @@ -225,24 +529,77 @@ public void save(String storagePath) { priceComponent.save(); } - for (AccessKey key : this.getAccessKeys()) { - key.getKeyInfo().publicKey = - key.getKeyInfo().publicKey.replace(this.oldStoragePath, storagePath); - key.getKeyInfo().privateKey = - key.getKeyInfo().privateKey.replace(this.oldStoragePath, storagePath); - key.getKeyInfo().vaultPasswordFile = - key.getKeyInfo().vaultPasswordFile.replace(this.oldStoragePath, storagePath); - key.getKeyInfo().vaultFile = - key.getKeyInfo().vaultFile.replace(this.oldStoragePath, storagePath); + // Sync access keys to db. + for (AccessKey key : this.provider.allAccessKeys) { + AccessKey.KeyInfo keyInfo = key.getKeyInfo(); + FileData.writeFileToDB(keyInfo.vaultFile); + FileData.writeFileToDB(keyInfo.vaultPasswordFile); + if (keyInfo.privateKey != null) { + FileData.writeFileToDB(keyInfo.privateKey); + } + if (keyInfo.publicKey != null) { + FileData.writeFileToDB(keyInfo.publicKey); + } + + File accessKeyDir = new File(keyInfo.vaultFile).getParentFile(); + // GCP provider contains credentials.json file. + if (this.provider.code.equals(Common.CloudType.gcp.toString())) { + FileData.writeFileToDB( + Paths.get(accessKeyDir.getAbsolutePath(), "credentials.json").toString()); + } + } + } + + for (CertificateInfo certificateInfo : certificateInfoList) { + if (!CertificateInfo.maybeGet(certificateInfo.uuid).isPresent()) { + certificateInfo.save(); + + File certificateInfoBaseDir = + new File( + CertificateHelper.getCADirPath( + platformPaths.storagePath, certificateInfo.customerUUID, certificateInfo.uuid)); + File[] certificateInfoFiles = certificateInfoBaseDir.listFiles(); + if (certificateInfoFiles != null) { + for (File certificateInfoFile : certificateInfoFiles) { + FileData.writeFileToDB(certificateInfoFile.getAbsolutePath()); + } + } } + } - for (AccessKey key : this.accessKeys) { - key.save(); + for (KmsConfig kmsConfig : kmsConfigs) { + if (KmsConfig.get(kmsConfig.configUUID) == null) { + kmsConfig.save(); } } - this.universe.setConfig(this.universeConfig); + for (KmsHistory kmsHistory : kmsHistoryList) { + if (!EncryptionAtRestUtil.keyRefExists(this.universe.universeUUID, kmsHistory.uuid.keyRef)) { + kmsHistory.save(); + } + } + + if (!this.skipReleases) { + // Update and save software releases and ybc software releases. + if (ybReleaseMetadata != null) { + String universeVersion = + this.universe.getUniverseDetails().getPrimaryCluster().userIntent.ybSoftwareVersion; + if (releaseManager.getReleaseByVersion(universeVersion) == null) { + releaseManager.addReleaseWithMetadata(universeVersion, ybReleaseMetadata); + } else { + releaseManager.updateReleaseMetadata(universeVersion, ybReleaseMetadata); + } + } + + // Imports local ybc and software releases. + releaseManager.importLocalReleases(); + releaseManager.updateCurrentReleases(); + } + this.universe.save(); + + // Update prometheus files. + swamperHelper.writeUniverseTargetJson(this.universe); } private static String generateSpecName(boolean isExport) { @@ -251,4 +608,30 @@ private static String generateSpecName(boolean isExport) { String specName = "yb-universe-spec-" + type + "-" + datePrefix; return specName; } + + // Only replace path at the beginning. + @VisibleForTesting + public static String replaceBeginningPath( + String pathToModify, String initialRoot, String finalRoot) { + String regex = "^" + Pattern.quote(initialRoot); + return pathToModify.replaceAll(regex, finalRoot); + } + + @Builder + @Jacksonized + @Data + public static class PlatformPaths { + + @JsonProperty("storagePath") + public String storagePath; + + @JsonProperty("releasesPath") + public String releasesPath; + + @JsonProperty("ybcReleasePath") + public String ybcReleasePath; + + @JsonProperty("ybcReleasesPath") + public String ybcReleasesPath; + } } diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/CloudInfoInterface.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/CloudInfoInterface.java index fefbcfa773fd..493a93464a30 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/helpers/CloudInfoInterface.java +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/CloudInfoInterface.java @@ -30,7 +30,6 @@ import com.yugabyte.yw.models.helpers.provider.region.GCPRegionCloudInfo; import com.yugabyte.yw.models.helpers.provider.region.azs.DefaultAZCloudInfo; import com.yugabyte.yw.models.helpers.provider.region.KubernetesRegionInfo; - import play.libs.Json; public interface CloudInfoInterface { diff --git a/managed/src/main/resources/application.test.conf b/managed/src/main/resources/application.test.conf index d62cb3907cab..15c548fc7389 100644 --- a/managed/src/main/resources/application.test.conf +++ b/managed/src/main/resources/application.test.conf @@ -12,7 +12,6 @@ db { yb { storage.path="/tmp" - # Reduced number of threads in unit test health.max_num_parallel_checks = 1 health.max_num_parallel_node_checks = 1 diff --git a/managed/src/main/resources/reference.conf b/managed/src/main/resources/reference.conf index 2daf69f733dc..85c98bf812a6 100644 --- a/managed/src/main/resources/reference.conf +++ b/managed/src/main/resources/reference.conf @@ -378,9 +378,10 @@ yb { wait_for_leaders_on_preferred=true } releases { - num_releases_to_keep_default = 3 - num_releases_to_keep_cloud = 2 - download_helm_chart_http_timeout = 60s + path = "/opt/yugabyte/releases" + num_releases_to_keep_default = 3 + num_releases_to_keep_cloud = 2 + download_helm_chart_http_timeout = 60s } ha { replication_schedule_enabled = false @@ -861,7 +862,8 @@ runtime_config { "yb.query_stats.thread_ttl", "yb.perf_advisor.universe_batch_size", "yb.perf_advisor.scheduler_interval_mins", - "yb.perf_advisor.cleanup.gc_check_interval" + "yb.perf_advisor.cleanup.gc_check_interval". + "yb.releases.path" ] scope_strictness.enabled = true data_validation.enabled = true diff --git a/managed/src/main/resources/v1.routes b/managed/src/main/resources/v1.routes index b9be447ea274..c36ebea08e4c 100644 --- a/managed/src/main/resources/v1.routes +++ b/managed/src/main/resources/v1.routes @@ -293,7 +293,7 @@ GET /customers/:cUUID/universes/:uniUUID/range_hash c # Import a universe POST /customers/:cUUID/universes/import com.yugabyte.yw.controllers.ImportController.importUniverse(cUUID: java.util.UUID) -# Link/Unlink universe +# Attach/Detach universe POST /customers/:cUUID/universes/:uniUUID/export com.yugabyte.yw.controllers.AttachDetachController.exportUniverse(cUUID: java.util.UUID, uniUUID: java.util.UUID) POST /customers/:cUUID/universes/:uniUUID/import com.yugabyte.yw.controllers.AttachDetachController.importUniverse(cUUID: java.util.UUID, uniUUID: java.util.UUID) diff --git a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundleTest.java b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundleTest.java index 7ddd12913330..0756566b1c88 100644 --- a/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundleTest.java +++ b/managed/src/test/java/com/yugabyte/yw/commissioner/tasks/CreateSupportBundleTest.java @@ -38,8 +38,6 @@ @RunWith(MockitoJUnitRunner.class) public class CreateSupportBundleTest extends CommissionerBaseTest { - @Mock private YBClient mockYBClientActual; - private Universe universe; private Customer customer; protected RuntimeConfigFactory runtimeConfigFactory; @@ -51,7 +49,6 @@ public void setUp() { this.customer = ModelFactory.testCustomer(); this.universe = ModelFactory.createUniverse(customer.getCustomerId()); this.runtimeConfigFactory = mockBaseTaskDependencies.getRuntimeConfigFactory(); - when(mockYBClient.getClient(any(), any())).thenReturn(mockYBClientActual); } @After diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/AttachDetachControllerTest.java b/managed/src/test/java/com/yugabyte/yw/controllers/AttachDetachControllerTest.java index 8fa9dd8339f3..c4a6750a266f 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/AttachDetachControllerTest.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/AttachDetachControllerTest.java @@ -14,6 +14,7 @@ import static org.mockito.Mockito.when; import static play.mvc.Http.Status.METHOD_NOT_ALLOWED; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.protobuf.ByteString; @@ -150,16 +151,20 @@ public void testInvalidXClusterDetach() { e.printStackTrace(); } + ObjectNode bodyJson = Json.newObject(); + bodyJson.put("skipReleases", true); + Result result = FakeApiHelper.doRequestWithAuthTokenAndBody( "POST", xClusterApiEndpoint, user.createAuthToken(), createXClusterRequestParams); assertOk(result); - result = - assertPlatformException( - () -> - FakeApiHelper.doRequestWithAuthTokenAndBody( - "POST", detachEndpoint, user.createAuthToken(), null)); + result = assertPlatformException(() -> detachUniverse(bodyJson)); assertEquals(METHOD_NOT_ALLOWED, result.status()); } + + private Result detachUniverse(JsonNode bodyJson) { + return FakeApiHelper.doRequestWithAuthTokenAndBody( + "POST", detachEndpoint, user.createAuthToken(), bodyJson); + } } diff --git a/managed/src/test/java/com/yugabyte/yw/models/UniverseSpecTest.java b/managed/src/test/java/com/yugabyte/yw/models/UniverseSpecTest.java new file mode 100644 index 000000000000..5bac735988d3 --- /dev/null +++ b/managed/src/test/java/com/yugabyte/yw/models/UniverseSpecTest.java @@ -0,0 +1,34 @@ +// Copyright (c) YugaByte, Inc. + +package com.yugabyte.yw.models; + +import static org.junit.Assert.assertEquals; + +import com.yugabyte.yw.common.FakeDBApplication; +import org.junit.Test; + +public class UniverseSpecTest extends FakeDBApplication { + + @Test + public void testReplaceBeginningPathChanged() { + String pathToModify = "/opt/yugaware/path/to/opt/yugaware/file"; + String initialRootPath = "/opt/yugaware"; + String finalRootPath = "/root/data"; + String expectedModifiedPath = "/root/data/path/to/opt/yugaware/file"; + + String modifiedPath = + UniverseSpec.replaceBeginningPath(pathToModify, initialRootPath, finalRootPath); + assertEquals(expectedModifiedPath, modifiedPath); + } + + @Test + public void testReplaceBeginningPathUnChanged() { + String pathToModify = "/opt/yugaware/path/to/opt/path/file"; + String initialRootPath = "/opt/yugaware"; + String finalRootPath = "/opt/yugaware"; + + String modifiedPath = + UniverseSpec.replaceBeginningPath(pathToModify, initialRootPath, finalRootPath); + assertEquals(pathToModify, modifiedPath); + } +} From e2e04eb1b986e121f6464dbe076fd2b4d4d63c60 Mon Sep 17 00:00:00 2001 From: Athul Sathyan <44346129+athulsn@users.noreply.github.com> Date: Thu, 2 Mar 2023 09:56:17 +0530 Subject: [PATCH 33/81] [PLAT-7600][UI revamp] Alignment/Padding issue in the input fields. Summary: Fixed autocomplete component input contents alignment issue Test Plan: Tested manually {F35022} {F35023} Reviewers: kkannan, lsangappa Reviewed By: lsangappa Subscribers: jenkins-bot, ui, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23268 --- managed/ui/src/redesign/theme/mainTheme.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/managed/ui/src/redesign/theme/mainTheme.ts b/managed/ui/src/redesign/theme/mainTheme.ts index d574ea63abc5..f5f0c3eb97c1 100644 --- a/managed/ui/src/redesign/theme/mainTheme.ts +++ b/managed/ui/src/redesign/theme/mainTheme.ts @@ -282,7 +282,7 @@ export const mainTheme = createTheme({ }, inputRoot: { minHeight: variables.inputHeight, - height: 'auto', + height: 'auto !important', padding: 4 }, tag: { From 9dc34bd96501adf138f7833c3ee0573b11a3596b Mon Sep 17 00:00:00 2001 From: Yury Shchetinin Date: Wed, 1 Mar 2023 14:53:03 +0300 Subject: [PATCH 34/81] [PLAT-5028] Fix swagger Summary: fixed swagger Test Plan: - Reviewers: sb-yb Reviewed By: sb-yb Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23243 --- .../yw/common/gflags/SpecificGFlags.java | 9 ++++- .../src/main/resources/swagger-strict.json | 38 +++++++++++++++++++ managed/src/main/resources/swagger.json | 6 ++- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java b/managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java index c0dbaa3ad185..5708fea1f15d 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java +++ b/managed/src/main/java/com/yugabyte/yw/common/gflags/SpecificGFlags.java @@ -9,6 +9,8 @@ import com.google.common.collect.ImmutableMap; import com.yugabyte.yw.commissioner.tasks.UniverseTaskBase; import com.yugabyte.yw.models.helpers.NodeDetails; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -21,6 +23,7 @@ import lombok.EqualsAndHashCode; @Data +@ApiModel(description = "GFlags for current cluster") public class SpecificGFlags { @EqualsAndHashCode @@ -28,8 +31,12 @@ public static class PerProcessFlags { public Map> value = new HashMap<>(); } - private boolean inheritFromPrimary; + @ApiModelProperty private boolean inheritFromPrimary; + + @ApiModelProperty(value = "Gflags grouped by procces") private PerProcessFlags perProcessFlags; + + @ApiModelProperty(value = "Overrides for gflags per availability zone") private Map perAZ = new HashMap<>(); @JsonIgnore diff --git a/managed/src/main/resources/swagger-strict.json b/managed/src/main/resources/swagger-strict.json index d0a15a1f50df..8ae1d02909d7 100644 --- a/managed/src/main/resources/swagger-strict.json +++ b/managed/src/main/resources/swagger-strict.json @@ -6233,6 +6233,21 @@ "required" : [ "buildNumber" ], "type" : "object" }, + "PerProcessFlags" : { + "properties" : { + "value" : { + "additionalProperties" : { + "additionalProperties" : { + "type" : "string" + }, + "type" : "object" + }, + "type" : "object" + } + }, + "required" : [ "value" ], + "type" : "object" + }, "PerfAdvisorSettingsFormData" : { "properties" : { "connection_skew_interval_mins" : { @@ -8594,6 +8609,26 @@ "required" : [ "clusters", "creatingUser", "kubernetesUpgradeSupported", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "upgradeOption", "upgradeSystemCatalog", "ybSoftwareVersion" ], "type" : "object" }, + "SpecificGFlags" : { + "description" : "GFlags for current cluster", + "properties" : { + "inheritFromPrimary" : { + "type" : "boolean" + }, + "perAZ" : { + "additionalProperties" : { + "$ref" : "#/definitions/PerProcessFlags" + }, + "description" : "Overrides for gflags per availability zone", + "type" : "object" + }, + "perProcessFlags" : { + "$ref" : "#/definitions/PerProcessFlags", + "description" : "Gflags grouped by procces" + } + }, + "type" : "object" + }, "StateChangeAuditInfo" : { "properties" : { "customerId" : { @@ -10838,6 +10873,9 @@ "format" : "int32", "type" : "integer" }, + "specificGFlags" : { + "$ref" : "#/definitions/SpecificGFlags" + }, "tserverGFlags" : { "additionalProperties" : { "type" : "string" diff --git a/managed/src/main/resources/swagger.json b/managed/src/main/resources/swagger.json index d3755d685228..48eeda1cd97c 100644 --- a/managed/src/main/resources/swagger.json +++ b/managed/src/main/resources/swagger.json @@ -8651,6 +8651,7 @@ "type" : "object" }, "SpecificGFlags" : { + "description" : "GFlags for current cluster", "properties" : { "inheritFromPrimary" : { "type" : "boolean" @@ -8659,13 +8660,14 @@ "additionalProperties" : { "$ref" : "#/definitions/PerProcessFlags" }, + "description" : "Overrides for gflags per availability zone", "type" : "object" }, "perProcessFlags" : { - "$ref" : "#/definitions/PerProcessFlags" + "$ref" : "#/definitions/PerProcessFlags", + "description" : "Gflags grouped by procces" } }, - "required" : [ "inheritFromPrimary", "perAZ", "perProcessFlags" ], "type" : "object" }, "StateChangeAuditInfo" : { From cb0cd3b8993d28364024eb7d0968c729a78fb08d Mon Sep 17 00:00:00 2001 From: vkumar Date: Thu, 2 Mar 2023 06:05:39 +0000 Subject: [PATCH 35/81] [PLAT-7303] OCP backup failure due to permission issue when copying over xxhash binary Summary: - Backups on Openshift failed when attempting to copy over xxhash binary from platform to DB pods - Workaround is to copy over to write-able `/tmp` directory for K8s universes. - Made changes to `yb_backup.py` script to look for `xxhash` in `/tmp` directory for K8s based deployments. - When required, the `tmp` can be made configurable. Test Plan: Jenkins: all tests - Tested backup on OCP. - Itests verify K8s and VM based deployments Reviewers: sneelakantan, svarshney, oleg Reviewed By: oleg Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23182 --- managed/devops/bin/yb_backup.py | 31 ++++++++----------- .../InstallThirdPartySoftwareK8s.java | 6 ++-- .../yw/common/NodeUniverseManager.java | 9 ++++++ 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/managed/devops/bin/yb_backup.py b/managed/devops/bin/yb_backup.py index 575574700dcc..0fc67e706d7c 100755 --- a/managed/devops/bin/yb_backup.py +++ b/managed/devops/bin/yb_backup.py @@ -110,6 +110,7 @@ YB_ADMIN_HELP_RE = re.compile(r'^ \d+\. (\w+).*') XXH64HASH_TOOL_PATH = os.path.join(YB_HOME_DIR, 'bin/xxhash') +XXH64HASH_TOOL_PATH_K8S = '/tmp/xxhash' XXH64_FILE_EXT = 'xxh64' XXH64_X86_BIN = 'xxhsum_x86' XXH64_AARCH_BIN = 'xxhsum_aarch' @@ -962,7 +963,7 @@ def init(self, snapshot_bucket, pg_based_backup): properties['check-sums'] = not self.backup.args.disable_checksums if not self.backup.args.disable_checksums: properties['hash-algorithm'] = XXH64_FILE_EXT \ - if self.backup.use_xxhash_checksum else SHA_FILE_EXT + if self.backup.xxhash_checksum_path else SHA_FILE_EXT def init_locations(self, tablet_leaders, snapshot_bucket): locations = self.body['locations'] @@ -1050,7 +1051,7 @@ def __init__(self): self.ip_to_ssh_key_map = {} self.secondary_to_primary_ip_map = {} self.region_to_location = {} - self.use_xxhash_checksum = None + self.xxhash_checksum_path = '' self.database_version = YBVersion("unknown") self.manifest = YBManifest(self) self.parse_arguments() @@ -1462,10 +1463,10 @@ def post_process_arguments(self): if live_tservers: # Need to check the architecture for only first node, rest # will be same in the cluster. - global XXH64HASH_TOOL_PATH + xxhash_tool_path = XXH64HASH_TOOL_PATH_K8S if self.is_k8s() else XXH64HASH_TOOL_PATH tserver = live_tservers[0] try: - self.run_ssh_cmd("[ -d '{}' ]".format(XXH64HASH_TOOL_PATH), + self.run_ssh_cmd("[ -d '{}' ]".format(xxhash_tool_path), tserver, upload_cloud_cfg=False).strip() node_machine_arch = self.run_ssh_cmd(['uname', '-m'], tserver, upload_cloud_cfg=False).strip() @@ -1473,16 +1474,12 @@ def post_process_arguments(self): xxh64_bin = XXH64_AARCH_BIN else: xxh64_bin = XXH64_X86_BIN - XXH64HASH_TOOL_PATH = os.path.join(XXH64HASH_TOOL_PATH, xxh64_bin) - self.use_xxhash_checksum = True + self.xxhash_checksum_path = os.path.join(xxhash_tool_path, xxh64_bin) except Exception: logging.warn("[app] xxhsum tool missing on the host, continuing with sha256") - self.use_xxhash_checksum = False else: raise BackupException("No Live TServer exists. " "Check the TServer nodes status & try again.") - else: - self.use_xxhash_checksum = False if self.args.mac: # As this arg is used only for the purpose of tests & we use hardcoded paths only @@ -2483,8 +2480,7 @@ def rearrange_snapshot_dirs( return tserver_ip_to_tablet_id_to_snapshot_dirs def create_checksum_cmd_not_quoted(self, file_path, checksum_file_path): - assert self.use_xxhash_checksum is not None - tool_path = XXH64HASH_TOOL_PATH if self.use_xxhash_checksum else SHA_TOOL_PATH + tool_path = self.xxhash_checksum_path if self.xxhash_checksum_path else SHA_TOOL_PATH prefix = pipes.quote(tool_path) return "{} {} > {}".format(prefix, file_path, checksum_file_path) @@ -2493,8 +2489,7 @@ def create_checksum_cmd(self, file_path, checksum_file_path): pipes.quote(file_path), pipes.quote(checksum_file_path)) def checksum_path(self, file_path): - assert self.use_xxhash_checksum is not None - ext = XXH64_FILE_EXT if self.use_xxhash_checksum else SHA_FILE_EXT + ext = XXH64_FILE_EXT if self.xxhash_checksum_path else SHA_FILE_EXT return file_path + '.' + ext def checksum_path_downloaded(self, file_path): @@ -3161,11 +3156,11 @@ def load_or_create_manifest(self): try: self.download_file(src_manifest_path, manifest_path) except subprocess.CalledProcessError as ex: - if self.use_xxhash_checksum: + if self.xxhash_checksum_path: # Possibly old checksum tool was used for the backup. # Try again with the old tool. logging.warning("Try to use " + SHA_TOOL_PATH + " for Manifest") - self.use_xxhash_checksum = False + self.xxhash_checksum_path = '' self.download_file(src_manifest_path, manifest_path) else: raise ex @@ -3182,11 +3177,11 @@ def load_or_create_manifest(self): if self.manifest.is_loaded(): if not self.args.disable_checksums and \ self.manifest.get_hash_algorithm() == XXH64_FILE_EXT \ - and not self.use_xxhash_checksum: - raise BackupException("Manifest references unavailable tool " + XXH64HASH_TOOL_PATH) + and not self.xxhash_checksum_path: + raise BackupException("Manifest references unavailable tool: xxhash") else: self.manifest.create_by_default(self.args.backup_location) - self.use_xxhash_checksum = False + self.xxhash_checksum_path = '' if self.args.verbose: logging.info("{} manifest: {}".format( diff --git a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallThirdPartySoftwareK8s.java b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallThirdPartySoftwareK8s.java index f1f00502a127..ed4d9f818556 100644 --- a/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallThirdPartySoftwareK8s.java +++ b/managed/src/main/java/com/yugabyte/yw/commissioner/tasks/subtasks/InstallThirdPartySoftwareK8s.java @@ -91,14 +91,16 @@ public void installThirdPartyPackagesTaskK8s(Universe universe) { .timeoutSecs(UPLOAD_XXSUM_PACKAGE_TIMEOUT_SEC) .build(); for (NodeDetails node : universe.getNodes()) { - String xxhsumTargetPath = nodeUniverseManager.getYbHomeDir(node, universe) + "/bin"; + String xxhsumTargetPathParent = nodeUniverseManager.getYbTmpDir(); + String xxhsumTargetPath = xxhsumTargetPathParent + "/xxhash.tar.gz"; nodeUniverseManager .uploadFileToNode( node, universe, xxhsumPackagePath, xxhsumTargetPath, PACKAGE_PERMISSIONS, context) .processErrors(); String extractxxhSumBinaryCmd = - String.format("cd %s && tar -xvf xxhash.tar.gz && rm xxhash.tar.gz", xxhsumTargetPath); + String.format( + "cd %s && tar -xvf xxhash.tar.gz && rm xxhash.tar.gz", xxhsumTargetPathParent); List unTarxxhSumBinariesCmd = Arrays.asList("bash", "-c", extractxxhSumBinaryCmd); nodeUniverseManager .runCommand(node, universe, unTarxxhSumBinariesCmd, context) diff --git a/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java b/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java index aac6968bcf07..843738cf60b7 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java +++ b/managed/src/main/java/com/yugabyte/yw/common/NodeUniverseManager.java @@ -262,6 +262,15 @@ public String getYbHomeDir(NodeDetails node, Universe universe) { return provider.getYbHome(); } + /** + * Placeholder method to get tmp directory for node + * + * @return tmp directory + */ + public String getYbTmpDir() { + return "/tmp"; + } + private void addConnectionParams( Universe universe, NodeDetails node, ShellProcessContext context, List commandArgs) { UniverseDefinitionTaskParams.Cluster cluster = From 1d98b7ea79d92ef893ddce4ea9b03bdc896f60e2 Mon Sep 17 00:00:00 2001 From: Aleksandr Malyshev Date: Wed, 1 Mar 2023 18:47:00 +0300 Subject: [PATCH 36/81] [PLAT-7561] Retry read write check in case of failure Summary: Previously we used to retry ysql read write check up to 2 times in case of failure. This is needed because the request we're doing may fail because of transaction conflict. So, we need to retry the request in this case. Now, we'll mark result as failure so that the check is performed once again - this way we should stop getting read write failure alerts. Test Plan: Create universe and leave it created for 1 day. Make sure read write test alert is not getting fired periodically. Reviewers: vbansal Reviewed By: vbansal Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23250 --- managed/src/main/java/com/yugabyte/yw/models/Users.java | 2 +- managed/src/main/resources/health/node_health.py.template | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/managed/src/main/java/com/yugabyte/yw/models/Users.java b/managed/src/main/java/com/yugabyte/yw/models/Users.java index 24abf7868b73..ff33de9ec201 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/Users.java +++ b/managed/src/main/java/com/yugabyte/yw/models/Users.java @@ -344,7 +344,7 @@ public String createAuthToken() { return authToken; } - public void setAuthToken(String authToken) { + public void updateAuthToken(String authToken) { this.authToken = authToken; save(); } diff --git a/managed/src/main/resources/health/node_health.py.template b/managed/src/main/resources/health/node_health.py.template index d5a0be6f8e18..e70ccbd344b7 100755 --- a/managed/src/main/resources/health/node_health.py.template +++ b/managed/src/main/resources/health/node_health.py.template @@ -1273,9 +1273,7 @@ class NodeChecker(): self.kill_spawned_postgres_workers() metric.add_value(0) - # It's metric-only check, hence we do set has_error=False to avoid retries - - # - as we retried in the check itself - return e.fill_and_return_entry([output], has_error=False, metrics=[metric]) + return e.fill_and_return_entry([output], has_error=True, metrics=[metric]) def check_clock_skew(self): logging.info("Checking clock synchronization on node {}".format(self.node)) From 42b1639f9e7a2a0a6098b2e3a9b2c4215e8ff4a3 Mon Sep 17 00:00:00 2001 From: Aleksandr Malyshev Date: Thu, 2 Mar 2023 15:23:28 +0300 Subject: [PATCH 37/81] [PLAT-7390] WebHook alert channel authentication Summary: Need to implement Webhook HTTP auth (Basic, at least) to support Moogsoft AIops integration Test Plan: Unit tested. Test with MoogSoft once trial details are received. Reviewers: sb-yb Reviewed By: sb-yb Subscribers: jenkins-bot, sanketh, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23035 --- .../alerts/AlertChannelWebHookParams.java | 4 ++ .../alerts/impl/AlertChannelWebBase.java | 33 ++++++++++- .../alerts/impl/AlertChannelWebHook.java | 3 +- .../yw/common/audit/AuditService.java | 3 +- .../com/yugabyte/yw/models/AlertChannel.java | 2 + .../yw/models/helpers/auth/BasicAuth.java | 14 +++++ .../yw/models/helpers/auth/HttpAuth.java | 22 +++++++ .../yw/models/helpers/auth/HttpAuthType.java | 7 +++ .../yw/models/helpers/auth/NoneAuth.java | 14 +++++ .../yw/models/helpers/auth/TokenAuth.java | 22 +++++++ .../helpers/auth/UsernamePasswordAuth.java | 17 ++++++ .../h2/V243__Alter_Alert_Channel_Params.sql | 1 + .../V243__Alter_Alert_Channel_Params.sql | 3 + .../src/main/resources/swagger-strict.json | 57 +++++++++++++++++++ managed/src/main/resources/swagger.json | 57 +++++++++++++++++++ .../alerts/impl/AlertChannelWebhookTest.java | 53 +++++++++++++++++ 16 files changed, 307 insertions(+), 5 deletions(-) create mode 100644 managed/src/main/java/com/yugabyte/yw/models/helpers/auth/BasicAuth.java create mode 100644 managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuth.java create mode 100644 managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuthType.java create mode 100644 managed/src/main/java/com/yugabyte/yw/models/helpers/auth/NoneAuth.java create mode 100644 managed/src/main/java/com/yugabyte/yw/models/helpers/auth/TokenAuth.java create mode 100644 managed/src/main/java/com/yugabyte/yw/models/helpers/auth/UsernamePasswordAuth.java create mode 100644 managed/src/main/resources/db/migration/default/h2/V243__Alter_Alert_Channel_Params.sql create mode 100644 managed/src/main/resources/db/migration/default/postgres/V243__Alter_Alert_Channel_Params.sql diff --git a/managed/src/main/java/com/yugabyte/yw/common/alerts/AlertChannelWebHookParams.java b/managed/src/main/java/com/yugabyte/yw/common/alerts/AlertChannelWebHookParams.java index 4209c08fc94b..7674ca04401c 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/alerts/AlertChannelWebHookParams.java +++ b/managed/src/main/java/com/yugabyte/yw/common/alerts/AlertChannelWebHookParams.java @@ -5,6 +5,7 @@ import static io.swagger.annotations.ApiModelProperty.AccessMode.READ_WRITE; import com.fasterxml.jackson.annotation.JsonTypeName; +import com.yugabyte.yw.models.helpers.auth.HttpAuth; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import javax.validation.constraints.NotNull; @@ -21,4 +22,7 @@ public class AlertChannelWebHookParams extends AlertChannelParams { @NotNull @URL private String webhookUrl; + + @ApiModelProperty(value = "Authentication Details", accessMode = READ_WRITE) + private HttpAuth httpAuth; } diff --git a/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebBase.java b/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebBase.java index 0c23eab01b1e..4c3418378c53 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebBase.java +++ b/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebBase.java @@ -4,9 +4,14 @@ import com.yugabyte.yw.common.WSClientRefresher; import com.yugabyte.yw.common.alerts.AlertTemplateVariableService; +import com.yugabyte.yw.models.helpers.auth.HttpAuth; +import com.yugabyte.yw.models.helpers.auth.TokenAuth; +import com.yugabyte.yw.models.helpers.auth.UsernamePasswordAuth; import java.util.concurrent.CompletionStage; import play.libs.Json; +import play.libs.ws.WSAuthScheme; import play.libs.ws.WSClient; +import play.libs.ws.WSRequest; import play.libs.ws.WSResponse; abstract class AlertChannelWebBase extends AlertChannelBase { @@ -21,13 +26,35 @@ public AlertChannelWebBase( } protected WSResponse sendRequest(String clientKey, String url, Object body) throws Exception { + return sendRequest(clientKey, url, body, null); + } + + protected WSResponse sendRequest(String clientKey, String url, Object body, HttpAuth httpAuth) + throws Exception { WSClient wsClient = wsClientRefresher.getClient(clientKey); - CompletionStage promise = + WSRequest request = wsClient .url(url) .addHeader("Content-Type", "application/json") - .addHeader("Accept", "application/json") - .post(Json.toJson(body)); + .addHeader("Accept", "application/json"); + if (httpAuth != null) { + switch (httpAuth.getType()) { + case BASIC: + UsernamePasswordAuth usernamePasswordAuth = (UsernamePasswordAuth) httpAuth; + request.setAuth( + usernamePasswordAuth.getUsername(), + usernamePasswordAuth.getPassword(), + WSAuthScheme.valueOf(usernamePasswordAuth.getType().name())); + break; + case TOKEN: + TokenAuth tokenAuth = (TokenAuth) httpAuth; + request.addHeader(tokenAuth.getTokenHeader(), tokenAuth.getTokenValue()); + break; + default: + // NONE + } + } + CompletionStage promise = request.post(Json.toJson(body)); return promise.toCompletableFuture().get(); } } diff --git a/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebHook.java b/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebHook.java index 11c5b78fa3af..ad130c02d6bc 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebHook.java +++ b/managed/src/main/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebHook.java @@ -48,7 +48,8 @@ public void sendNotification( JsonNode body = Json.parse(text); try { - WSResponse response = sendRequest(WEBHOOK_WS_KEY, params.getWebhookUrl(), body); + WSResponse response = + sendRequest(WEBHOOK_WS_KEY, params.getWebhookUrl(), body, params.getHttpAuth()); if (response.getStatus() != HttpStatus.SC_OK) { throw new PlatformNotificationException( diff --git a/managed/src/main/java/com/yugabyte/yw/common/audit/AuditService.java b/managed/src/main/java/com/yugabyte/yw/common/audit/AuditService.java index 44553e0bb7c5..4ee538ee5af4 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/audit/AuditService.java +++ b/managed/src/main/java/com/yugabyte/yw/common/audit/AuditService.java @@ -81,7 +81,8 @@ public class AuditService { "$..smtpPassword", // Hashicorp token "$..HC_VAULT_TOKEN", - "$..vaultToken"); + "$..vaultToken", + "$..token"); public static final List SECRET_JSON_PATHS = SECRET_PATHS.stream().map(JsonPath::compile).collect(Collectors.toList()); diff --git a/managed/src/main/java/com/yugabyte/yw/models/AlertChannel.java b/managed/src/main/java/com/yugabyte/yw/models/AlertChannel.java index 94aedc597dcf..e80b4b866028 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/AlertChannel.java +++ b/managed/src/main/java/com/yugabyte/yw/models/AlertChannel.java @@ -19,6 +19,7 @@ import io.ebean.Finder; import io.ebean.Model; import io.ebean.annotation.DbJson; +import io.ebean.annotation.Encrypted; import io.ebean.annotation.EnumValue; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -95,6 +96,7 @@ public enum ChannelType { @JsonSubTypes.Type(value = AlertChannelWebHookParams.class, name = "WebHook") }) @ApiModelProperty(value = "Channel params", accessMode = READ_WRITE) + @Encrypted private AlertChannelParams params; @JsonIgnore diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/BasicAuth.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/BasicAuth.java new file mode 100644 index 000000000000..c48ff245248d --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/BasicAuth.java @@ -0,0 +1,14 @@ +package com.yugabyte.yw.models.helpers.auth; + +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +@ApiModel("Basic Auth information") +public class BasicAuth extends UsernamePasswordAuth { + public BasicAuth() { + setType(HttpAuthType.BASIC); + } +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuth.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuth.java new file mode 100644 index 000000000000..c80e82cc9ca4 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuth.java @@ -0,0 +1,22 @@ +package com.yugabyte.yw.models.helpers.auth; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiModelProperty.AccessMode; +import lombok.Data; + +@Data +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(value = NoneAuth.class, name = "NONE"), + @Type(value = BasicAuth.class, name = "BASIC"), + @Type(value = TokenAuth.class, name = "TOKEN") +}) +@ApiModel("HTTP Auth information") +public class HttpAuth { + @ApiModelProperty(value = "HTTP Auth type", accessMode = AccessMode.READ_WRITE) + private HttpAuthType type; +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuthType.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuthType.java new file mode 100644 index 000000000000..88689976b64d --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/HttpAuthType.java @@ -0,0 +1,7 @@ +package com.yugabyte.yw.models.helpers.auth; + +public enum HttpAuthType { + NONE, + BASIC, + TOKEN +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/NoneAuth.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/NoneAuth.java new file mode 100644 index 000000000000..f9e3a8e1ee92 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/NoneAuth.java @@ -0,0 +1,14 @@ +package com.yugabyte.yw.models.helpers.auth; + +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +@ApiModel("No Auth") +public class NoneAuth extends HttpAuth { + public NoneAuth() { + setType(HttpAuthType.NONE); + } +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/TokenAuth.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/TokenAuth.java new file mode 100644 index 000000000000..08baaa52793f --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/TokenAuth.java @@ -0,0 +1,22 @@ +package com.yugabyte.yw.models.helpers.auth; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiModelProperty.AccessMode; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +@ApiModel("Token Auth information") +public class TokenAuth extends HttpAuth { + public TokenAuth() { + setType(HttpAuthType.TOKEN); + } + + @ApiModelProperty(value = "Header name", accessMode = AccessMode.READ_WRITE) + private String tokenHeader; + + @ApiModelProperty(value = "Token value", accessMode = AccessMode.READ_WRITE) + private String tokenValue; +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/UsernamePasswordAuth.java b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/UsernamePasswordAuth.java new file mode 100644 index 000000000000..a7212dbcf414 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/helpers/auth/UsernamePasswordAuth.java @@ -0,0 +1,17 @@ +package com.yugabyte.yw.models.helpers.auth; + +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiModelProperty.AccessMode; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class UsernamePasswordAuth extends HttpAuth { + + @ApiModelProperty(value = "Username", accessMode = AccessMode.READ_WRITE) + private String username; + + @ApiModelProperty(value = "Password", accessMode = AccessMode.READ_WRITE) + private String password; +} diff --git a/managed/src/main/resources/db/migration/default/h2/V243__Alter_Alert_Channel_Params.sql b/managed/src/main/resources/db/migration/default/h2/V243__Alter_Alert_Channel_Params.sql new file mode 100644 index 000000000000..d9633be7166b --- /dev/null +++ b/managed/src/main/resources/db/migration/default/h2/V243__Alter_Alert_Channel_Params.sql @@ -0,0 +1 @@ +ALTER TABLE alert_channel ALTER COLUMN params TYPE binary varying; diff --git a/managed/src/main/resources/db/migration/default/postgres/V243__Alter_Alert_Channel_Params.sql b/managed/src/main/resources/db/migration/default/postgres/V243__Alter_Alert_Channel_Params.sql new file mode 100644 index 000000000000..68b885521bba --- /dev/null +++ b/managed/src/main/resources/db/migration/default/postgres/V243__Alter_Alert_Channel_Params.sql @@ -0,0 +1,3 @@ +-- Copyright (c) YugaByte, Inc. + +ALTER TABLE alert_channel ALTER COLUMN params TYPE bytea USING pgp_sym_encrypt(params::text, 'alert_channel::params'); diff --git a/managed/src/main/resources/swagger-strict.json b/managed/src/main/resources/swagger-strict.json index 8ae1d02909d7..d15c743539ba 100644 --- a/managed/src/main/resources/swagger-strict.json +++ b/managed/src/main/resources/swagger-strict.json @@ -508,6 +508,10 @@ "$ref" : "#/definitions/AlertChannelParams" }, { "properties" : { + "httpAuth" : { + "$ref" : "#/definitions/HTTP Auth information", + "description" : "Authentication Details" + }, "webhookUrl" : { "description" : "Webhook URL", "type" : "string" @@ -2371,6 +2375,23 @@ "required" : [ "creatingUser", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "storageConfigUUID", "timeTakenPartial" ], "type" : "object" }, + "Basic Auth information" : { + "allOf" : [ { + "$ref" : "#/definitions/HTTP Auth information" + }, { + "properties" : { + "password" : { + "description" : "Password", + "type" : "string" + }, + "username" : { + "description" : "Username", + "type" : "string" + } + }, + "type" : "object" + } ] + }, "BootstarpBackupParams" : { "description" : "Backup parameters for bootstrapping", "properties" : { @@ -4371,6 +4392,17 @@ "required" : [ "clusters", "creatingUser", "kubernetesUpgradeSupported", "masterGFlags", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "tserverGFlags", "upgradeOption" ], "type" : "object" }, + "HTTP Auth information" : { + "discriminator" : "type", + "properties" : { + "type" : { + "description" : "HTTP Auth type", + "enum" : [ "NONE", "BASIC", "TOKEN" ], + "type" : "string" + } + }, + "type" : "object" + }, "HashedTimestampColumnFinderResponse" : { "properties" : { "current_database" : { @@ -5679,6 +5711,14 @@ }, "type" : "object" }, + "No Auth" : { + "allOf" : [ { + "$ref" : "#/definitions/HTTP Auth information" + }, { + "properties" : { }, + "type" : "object" + } ] + }, "NodeActionFormData" : { "properties" : { "nodeAction" : { @@ -9712,6 +9752,23 @@ "required" : [ "allowInsecure", "clusters", "creatingUser", "enableClientToNodeEncrypt", "enableNodeToNodeEncrypt", "kubernetesUpgradeSupported", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "upgradeOption" ], "type" : "object" }, + "Token Auth information" : { + "allOf" : [ { + "$ref" : "#/definitions/HTTP Auth information" + }, { + "properties" : { + "tokenHeader" : { + "description" : "Header name", + "type" : "string" + }, + "tokenValue" : { + "description" : "Token value", + "type" : "string" + } + }, + "type" : "object" + } ] + }, "TriggerHealthCheckResult" : { "description" : "The response type for triggering a health check. Contains the timestamp of when the health check was triggered.", "properties" : { diff --git a/managed/src/main/resources/swagger.json b/managed/src/main/resources/swagger.json index 48eeda1cd97c..fd486d6b28b2 100644 --- a/managed/src/main/resources/swagger.json +++ b/managed/src/main/resources/swagger.json @@ -508,6 +508,10 @@ "$ref" : "#/definitions/AlertChannelParams" }, { "properties" : { + "httpAuth" : { + "$ref" : "#/definitions/HTTP Auth information", + "description" : "Authentication Details" + }, "webhookUrl" : { "description" : "Webhook URL", "type" : "string" @@ -2371,6 +2375,23 @@ "required" : [ "creatingUser", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "storageConfigUUID", "timeTakenPartial" ], "type" : "object" }, + "Basic Auth information" : { + "allOf" : [ { + "$ref" : "#/definitions/HTTP Auth information" + }, { + "properties" : { + "password" : { + "description" : "Password", + "type" : "string" + }, + "username" : { + "description" : "Username", + "type" : "string" + } + }, + "type" : "object" + } ] + }, "BootstarpBackupParams" : { "description" : "Backup parameters for bootstrapping", "properties" : { @@ -4371,6 +4392,17 @@ "required" : [ "clusters", "creatingUser", "kubernetesUpgradeSupported", "masterGFlags", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "tserverGFlags", "upgradeOption" ], "type" : "object" }, + "HTTP Auth information" : { + "discriminator" : "type", + "properties" : { + "type" : { + "description" : "HTTP Auth type", + "enum" : [ "NONE", "BASIC", "TOKEN" ], + "type" : "string" + } + }, + "type" : "object" + }, "HashedTimestampColumnFinderResponse" : { "properties" : { "current_database" : { @@ -5679,6 +5711,14 @@ }, "type" : "object" }, + "No Auth" : { + "allOf" : [ { + "$ref" : "#/definitions/HTTP Auth information" + }, { + "properties" : { }, + "type" : "object" + } ] + }, "NodeActionFormData" : { "properties" : { "nodeAction" : { @@ -9753,6 +9793,23 @@ "required" : [ "allowInsecure", "clusters", "creatingUser", "enableClientToNodeEncrypt", "enableNodeToNodeEncrypt", "kubernetesUpgradeSupported", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "upgradeOption" ], "type" : "object" }, + "Token Auth information" : { + "allOf" : [ { + "$ref" : "#/definitions/HTTP Auth information" + }, { + "properties" : { + "tokenHeader" : { + "description" : "Header name", + "type" : "string" + }, + "tokenValue" : { + "description" : "Token value", + "type" : "string" + } + }, + "type" : "object" + } ] + }, "TriggerHealthCheckResult" : { "description" : "The response type for triggering a health check. Contains the timestamp of when the health check was triggered.", "properties" : { diff --git a/managed/src/test/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebhookTest.java b/managed/src/test/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebhookTest.java index 5d44637d1f57..16c6fb93c575 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebhookTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/alerts/impl/AlertChannelWebhookTest.java @@ -20,8 +20,12 @@ import com.yugabyte.yw.models.AlertChannel; import com.yugabyte.yw.models.AlertChannel.ChannelType; import com.yugabyte.yw.models.Customer; +import com.yugabyte.yw.models.helpers.auth.BasicAuth; +import com.yugabyte.yw.models.helpers.auth.HttpAuth; +import com.yugabyte.yw.models.helpers.auth.TokenAuth; import java.io.IOException; import java.nio.charset.Charset; +import java.util.function.Consumer; import okhttp3.HttpUrl; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -105,6 +109,32 @@ public void test() throws PlatformNotificationException, IOException, Interrupte } } + @Test + public void testBasicAuth() + throws PlatformNotificationException, IOException, InterruptedException { + BasicAuth basicAuth = new BasicAuth(); + basicAuth.setUsername("testUser"); + basicAuth.setPassword("testPassword"); + testAuth( + basicAuth, + recordedRequest -> + assertThat( + recordedRequest.getHeader("Authorization"), + equalTo("Basic dGVzdFVzZXI6dGVzdFBhc3N3b3Jk"))); + } + + @Test + public void testTokenAuth() + throws PlatformNotificationException, IOException, InterruptedException { + TokenAuth tokenAuth = new TokenAuth(); + tokenAuth.setTokenHeader("testHeader"); + tokenAuth.setTokenValue("testValue"); + testAuth( + tokenAuth, + recordedRequest -> + assertThat(recordedRequest.getHeader("testHeader"), equalTo("testValue"))); + } + @Test public void testFailure() throws IOException, PlatformNotificationException { try (MockWebServer server = new MockWebServer()) { @@ -129,4 +159,27 @@ public void testFailure() throws IOException, PlatformNotificationException { + "error response 500 received with body {\"error\":\"not_ok\"}")); } } + + private void testAuth(HttpAuth auth, Consumer asserts) + throws PlatformNotificationException, IOException, InterruptedException { + try (MockWebServer server = new MockWebServer()) { + server.start(); + HttpUrl baseUrl = server.url(WEBHOOK_TEST_PATH); + server.enqueue(new MockResponse().setBody("{\"status\":\"ok\"}")); + + AlertChannel channelConfig = new AlertChannel(); + channelConfig.setName("Channel name"); + AlertChannelWebHookParams params = new AlertChannelWebHookParams(); + params.setWebhookUrl(baseUrl.toString()); + params.setHttpAuth(auth); + channelConfig.setParams(params); + + Alert alert = ModelFactory.createAlert(defaultCustomer); + channel.sendNotification(defaultCustomer, alert, channelConfig, alertChannelTemplatesExt); + + RecordedRequest request = server.takeRequest(); + assertThat(request.getPath(), is(WEBHOOK_TEST_PATH)); + asserts.accept(request); + } + } } From d5725e73b5a0a9d0dc424dac5c3dfc7c456ef152 Mon Sep 17 00:00:00 2001 From: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> Date: Thu, 2 Mar 2023 10:35:58 -0500 Subject: [PATCH 38/81] Update _index.md (#16274) --- .../yugabyte-platform/administer-yugabyte-platform/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/_index.md b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/_index.md index e5bb3267c12a..4f16765e8f37 100644 --- a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/_index.md +++ b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/_index.md @@ -15,7 +15,7 @@ menu: type: indexpage --- -YugabyteDB Anywhere can create a YugabyteDB universe with many instances (virtual machines, pods, machines, and so on, provided by IaaS), grouped together to form one logical distributed database. Each universe includes one primary cluster and, optionally, one or more read replica clusters. All instances belonging to a cluster run on the same type of cloud provider instance type.

+YugabyteDB Anywhere can create a YugabyteDB universe with many instances (virtual machines, pods, machines, and so on, provided by IaaS), grouped together to form one logical distributed database. Each universe includes one primary cluster and, optionally, one or more read replica clusters. All instances belonging to a cluster run on the same type of cloud provider instance type.
From 1f4ff775fc368ffc9a6e071b5c11abaaa608441b Mon Sep 17 00:00:00 2001 From: Aman Date: Thu, 2 Mar 2023 16:45:24 +0530 Subject: [PATCH 39/81] UI Fix to detect registry value from the API response Summary: UI Fix to detect registry value from the API response Test Plan: Tested locally by updating the UI and value was getting populated. # Tip: In Git and Mercurial, use a branch like "T123" to automatically associate # changes with the corresponding task. Reviewers: sneelakantan, anabaria, kkannan Reviewed By: kkannan Subscribers: jenkins-bot, svarshney, jmak, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23275 --- .../PublicCloud/Kubernetes/CreateKubernetesConfiguration.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/managed/ui/src/components/config/PublicCloud/Kubernetes/CreateKubernetesConfiguration.js b/managed/ui/src/components/config/PublicCloud/Kubernetes/CreateKubernetesConfiguration.js index 43b67d8470fc..9092031b51d6 100644 --- a/managed/ui/src/components/config/PublicCloud/Kubernetes/CreateKubernetesConfiguration.js +++ b/managed/ui/src/components/config/PublicCloud/Kubernetes/CreateKubernetesConfiguration.js @@ -134,13 +134,14 @@ class CreateKubernetesConfiguration extends Component { this.props .fetchKubenetesConfig() .then((resp) => { - const { KUBECONFIG_PULL_SECRET_NAME, KUBECONFIG_PULL_SECRET_CONTENT } = resp.data.config; + const { KUBECONFIG_PULL_SECRET_NAME, KUBECONFIG_PULL_SECRET_CONTENT, KUBECONFIG_IMAGE_REGISTRY} = resp.data.config; const { regionList } = resp.data; const fileObj = new File([KUBECONFIG_PULL_SECRET_CONTENT], KUBECONFIG_PULL_SECRET_NAME, { type: 'text/plain', lastModified: new Date().getTime() }); setFieldValue('pullSecret', fileObj); + setFieldValue('imageRegistry', KUBECONFIG_IMAGE_REGISTRY); const parsedRegionList = regionList.map((r) => { let regionCode = {}; From 3d24e6c5560939a825a5de142501636134af6e83 Mon Sep 17 00:00:00 2001 From: Anubhav Srivastava Date: Tue, 28 Feb 2023 19:52:05 -0500 Subject: [PATCH 40/81] [#14233] Pass placement info in HandleAddIfWrongPlacement Summary: We were not passing the placement info to `CanAddTabletToTabletServer` in one HandleAddIfWrongPlacement case. This resulted in the load balancer being able to move tablets from a blacklisted tserver to any tserver, including those outside the placement policy. Test Plan: Three tests were added as part of this diff. LoadBalancerPlacementPolicyTest.BlacklistedAdd is a test for the new behavior and the other two are regression tests and passed before this diff. - LoadBalancerPlacementPolicyTest.UnderreplicatedAdd: HandleAddIfMissingPlacement should not add replicas to zones outside the placement policy - LoadBalancerPlacementPolicyTest.BlacklistedAdd: HandleAddIfWrongPlacement should not add replicas to zones outside the placement policy (would fail prior to this diff) - LoadBalancerPlacementPolicyTest.AlterPlacement: Test that if the placement info is changed from a,b,c to a,b,d, the replicas move accordingly. Reviewers: jhe, rahuldesirazu Reviewed By: jhe, rahuldesirazu Subscribers: jhe, ybase, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D23236 --- .../load_balancer_placement_policy-test.cc | 206 +++++++++++++----- src/yb/master/cluster_balance.cc | 7 +- src/yb/master/cluster_balance_util.cc | 7 +- src/yb/master/cluster_balance_util.h | 3 +- 4 files changed, 156 insertions(+), 67 deletions(-) diff --git a/src/yb/integration-tests/load_balancer_placement_policy-test.cc b/src/yb/integration-tests/load_balancer_placement_policy-test.cc index 19e707723e8d..e5148f0abf41 100644 --- a/src/yb/integration-tests/load_balancer_placement_policy-test.cc +++ b/src/yb/integration-tests/load_balancer_placement_policy-test.cc @@ -33,12 +33,15 @@ #include "yb/util/net/net_fwd.h" #include "yb/util/result.h" #include "yb/util/test_macros.h" +#include "yb/util/tsan_util.h" using std::string; using std::vector; using namespace std::literals; +DECLARE_int32(catalog_manager_bg_task_wait_ms); + namespace yb { namespace integration_tests { @@ -225,64 +228,6 @@ TEST_F(LoadBalancerPlacementPolicyTest, CreateTableWithPlacementPolicyTest) { } } -class LoadBalancerReadReplicaPlacementPolicyTest : public LoadBalancerPlacementPolicyTest { - protected: - void TestBlacklist(bool use_empty_table_placement) { - const string kReadReplicaPlacementUuid = "read_replica"; - // Add 2 read replicas to cluster placement policy. - size_t num_tservers = num_tablet_servers(); - AddNewTserverToLocation("c", "r", "z0", ++num_tservers, kReadReplicaPlacementUuid); - AddNewTserverToLocation("c", "r", "z0", ++num_tservers, kReadReplicaPlacementUuid); - ASSERT_EQ(num_tservers, 5); - - ASSERT_OK(yb_admin_client_->AddReadReplicaPlacementInfo( - "c.r.z0:0", 1 /* replication_factor */, kReadReplicaPlacementUuid)); - - DeleteTable(); - if (use_empty_table_placement) { - master::ReplicationInfoPB ri; - ASSERT_OK(NewTableCreator()->table_name(table_name()) - .schema(&schema_).replication_info(ri).Create()); - } else { - ASSERT_OK(NewTableCreator()->table_name(table_name()).schema(&schema_).Create()); - } - - // There should be 2 tablets on each of the read replicas since we start with 4 tablets and - // the replication factor for read replicas is 1. - // Note that we shouldn't have to wait for the load balancer here, since the table creation - // should evenly spread the tablets across both read replicas. - vector counts_per_ts; - GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); - vector expected_counts_per_ts = {4, 4, 4, 2, 2}; - ASSERT_VECTORS_EQ(expected_counts_per_ts, counts_per_ts); - - // Blacklist one of the read replicas. The tablets should all move to the other read replica. - ASSERT_OK(external_mini_cluster()->AddTServerToBlacklist( - external_mini_cluster()->master(), - external_mini_cluster()->tablet_server(4))); - WaitForLoadBalancer(); - GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); - expected_counts_per_ts = {4, 4, 4, 4, 0}; - ASSERT_VECTORS_EQ(expected_counts_per_ts, counts_per_ts); - - // Clear the blacklist. The tablets should spread evenly across both read replicas. - ASSERT_OK(external_mini_cluster()->ClearBlacklist(external_mini_cluster()->master())); - WaitForLoadBalancer(); - GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); - expected_counts_per_ts = {4, 4, 4, 2, 2}; - ASSERT_VECTORS_EQ(expected_counts_per_ts, counts_per_ts); - } -}; - -TEST_F(LoadBalancerReadReplicaPlacementPolicyTest, Blacklist) { - TestBlacklist(false /* use_empty_table_placement */); -} - -// Regression test for GitHub issue #15698. -TEST_F(LoadBalancerReadReplicaPlacementPolicyTest, BlacklistWithEmptyPlacement) { - TestBlacklist(true /* use_empty_table_placement */); -} - TEST_F(LoadBalancerPlacementPolicyTest, PlacementPolicyTest) { // Set cluster placement policy. ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("c.r.z0,c.r.z1,c.r.z2", 3, "")); @@ -430,6 +375,92 @@ TEST_F(LoadBalancerPlacementPolicyTest, AlterPlacementDataConsistencyTest) { placement_table, ClusterVerifier::EXACTLY, rows_inserted)); } +// HandleAddIfMissingPlacement should not add replicas to zones outside the placement policy. +TEST_F(LoadBalancerPlacementPolicyTest, UnderreplicatedAdd) { + const int consider_failed_sec = 3; + ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("c.r.z0,c.r.z1,c.r.z2", 3, "")); + + // Add a tserver in a new zone that is not part of the placement info. + const int new_num_tservers = 4; + AddNewTserverToZone("z3", new_num_tservers); + + ASSERT_OK(external_mini_cluster()->SetFlagOnTServers( + "follower_unavailable_considered_failed_sec", std::to_string(consider_failed_sec))); + external_mini_cluster()->tablet_server(0)->Shutdown(SafeShutdown::kTrue); + ASSERT_OK(external_mini_cluster()->WaitForTabletServerCount( + 3 /* num_tservers */, 10s /* timeout */)); + + // Wait for ts0 removed from quorum. + vector counts_per_ts; + const vector expected_counts_per_ts = {0, 4, 4, 0}; + ASSERT_OK(WaitFor([&] { + GetLoadOnTservers(table_name().table_name(), new_num_tservers, &counts_per_ts); + return counts_per_ts == expected_counts_per_ts; + }, 10s * kTimeMultiplier, "Wait for ts0 removed from quorum.")); + + // Should not add a replica in ts3 since that does not fix the under-replication in z0. + SleepFor(FLAGS_catalog_manager_bg_task_wait_ms * 2ms); + WaitForLoadBalancerToBeIdle(); + GetLoadOnTservers(table_name().table_name(), new_num_tservers, &counts_per_ts); + ASSERT_EQ(counts_per_ts, expected_counts_per_ts); + + ASSERT_OK(external_mini_cluster()->tablet_server(0)->Start()); +} + +// HandleAddIfWrongPlacement should not add replicas to zones outside the placement policy when +// moving off of a blacklisted node. +TEST_F(LoadBalancerPlacementPolicyTest, BlacklistedAdd) { + ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("c.r.z0,c.r.z1,c.r.z2", 3, "")); + + // Add a tserver in a new zone that is not part of the placement info. + int num_tservers = 4; + AddNewTserverToZone("z3", num_tservers); + + // Blacklist a tserver to give it "wrong" placement. + ASSERT_OK(external_mini_cluster()->AddTServerToBlacklist( + external_mini_cluster()->GetLeaderMaster(), + external_mini_cluster()->tablet_server(0) + )); + + SleepFor(3s * kTimeMultiplier); + WaitForLoadBalancerToBeIdle(); + + // Should not move from ts0 as we do not have an alternative in the same zone. + vector counts_per_ts; + vector expected_counts_per_ts = {4, 4, 4, 0}; + GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); + ASSERT_VECTORS_EQ(counts_per_ts, expected_counts_per_ts); + + // Should move from the ts0 replica to other tserver in zone 0 (ts4). + ++num_tservers; + AddNewTserverToZone("z0", num_tservers); + WaitForLoadBalanceCompletion(); + + expected_counts_per_ts = {0, 4, 4, 0, 4}; + GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); + ASSERT_VECTORS_EQ(counts_per_ts, expected_counts_per_ts); +} + +// HandleAddIfWrongPlacement should move replicas to the appropriate zones after placement is +// altered. +TEST_F(LoadBalancerPlacementPolicyTest, AlterPlacement) { + ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("c.r.z0,c.r.z1,c.r.z2", 3, "")); + + // Add a tserver in a new zone that is not part of the placement info. + const int new_num_tservers = 4; + AddNewTserverToZone("z3", new_num_tservers); + + ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("c.r.z1,c.r.z2,c.r.z3", 3, "")); + WaitForLoadBalanceCompletion(); + + // HandleAddIfMissingPlacement should add a ts3 replica to fix minimum placement in z0, then + // HandleRemoveIfWrongPlacement should remove the ts0 replica to fix the over-replication. + vector counts_per_ts; + vector expected_counts_per_ts = {0, 4, 4, 4}; + GetLoadOnTservers(table_name().table_name(), new_num_tservers, &counts_per_ts); + ASSERT_VECTORS_EQ(counts_per_ts, expected_counts_per_ts); +} + TEST_F(LoadBalancerPlacementPolicyTest, ModifyPlacementUUIDTest) { // Set cluster placement policy. ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("c.r.z0,c.r.z1,c.r.z2", 3, "")); @@ -476,7 +507,6 @@ TEST_F(LoadBalancerPlacementPolicyTest, ModifyPlacementUUIDTest) { // tablets allotted to it. ASSERT_EQ(counts_per_ts[3], 0); ASSERT_EQ(counts_per_ts[4], 4); - } TEST_F(LoadBalancerPlacementPolicyTest, PrefixPlacementTest) { @@ -718,5 +748,63 @@ TEST_F(LoadBalancerPlacementPolicyTest, PrefixPlacementTest) { // FIN: Thank you all for watching, have a great day ahead! } +class LoadBalancerReadReplicaPlacementPolicyTest : public LoadBalancerPlacementPolicyTest { + protected: + void TestBlacklist(bool use_empty_table_placement) { + const string kReadReplicaPlacementUuid = "read_replica"; + // Add 2 read replicas to cluster placement policy. + size_t num_tservers = num_tablet_servers(); + AddNewTserverToLocation("c", "r", "z0", ++num_tservers, kReadReplicaPlacementUuid); + AddNewTserverToLocation("c", "r", "z0", ++num_tservers, kReadReplicaPlacementUuid); + ASSERT_EQ(num_tservers, 5); + + ASSERT_OK(yb_admin_client_->AddReadReplicaPlacementInfo( + "c.r.z0:0", 1 /* replication_factor */, kReadReplicaPlacementUuid)); + + DeleteTable(); + if (use_empty_table_placement) { + master::ReplicationInfoPB ri; + ASSERT_OK(NewTableCreator()->table_name(table_name()) + .schema(&schema_).replication_info(ri).Create()); + } else { + ASSERT_OK(NewTableCreator()->table_name(table_name()).schema(&schema_).Create()); + } + + // There should be 2 tablets on each of the read replicas since we start with 4 tablets and + // the replication factor for read replicas is 1. + // Note that we shouldn't have to wait for the load balancer here, since the table creation + // should evenly spread the tablets across both read replicas. + vector counts_per_ts; + GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); + vector expected_counts_per_ts = {4, 4, 4, 2, 2}; + ASSERT_VECTORS_EQ(expected_counts_per_ts, counts_per_ts); + + // Blacklist one of the read replicas. The tablets should all move to the other read replica. + ASSERT_OK(external_mini_cluster()->AddTServerToBlacklist( + external_mini_cluster()->master(), + external_mini_cluster()->tablet_server(4))); + WaitForLoadBalancer(); + GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); + expected_counts_per_ts = {4, 4, 4, 4, 0}; + ASSERT_VECTORS_EQ(expected_counts_per_ts, counts_per_ts); + + // Clear the blacklist. The tablets should spread evenly across both read replicas. + ASSERT_OK(external_mini_cluster()->ClearBlacklist(external_mini_cluster()->master())); + WaitForLoadBalancer(); + GetLoadOnTservers(table_name().table_name(), num_tservers, &counts_per_ts); + expected_counts_per_ts = {4, 4, 4, 2, 2}; + ASSERT_VECTORS_EQ(expected_counts_per_ts, counts_per_ts); + } +}; + +TEST_F(LoadBalancerReadReplicaPlacementPolicyTest, Blacklist) { + TestBlacklist(false /* use_empty_table_placement */); +} + +// Regression test for GitHub issue #15698. +TEST_F(LoadBalancerReadReplicaPlacementPolicyTest, BlacklistWithEmptyPlacement) { + TestBlacklist(true /* use_empty_table_placement */); +} + } // namespace integration_tests } // namespace yb diff --git a/src/yb/master/cluster_balance.cc b/src/yb/master/cluster_balance.cc index f925ed8274d8..81bc90c9d494 100644 --- a/src/yb/master/cluster_balance.cc +++ b/src/yb/master/cluster_balance.cc @@ -775,7 +775,8 @@ Result ClusterLoadBalancer::HandleAddIfMissingPlacement( // that we can use this tablet server. if (placement_info.placement_blocks().empty()) { // No need to check placement info, as there is none. - can_choose_ts = VERIFY_RESULT(state_->CanAddTabletToTabletServer(tablet_id, ts_uuid)); + can_choose_ts = VERIFY_RESULT( + state_->CanAddTabletToTabletServer(tablet_id, ts_uuid, nullptr /* placement_info */)); } else { // We added a tablet to the set with missing replicas both if it is under-replicated, and we // added a placement to the tablet_meta under_replicated_placements if the num replicas in @@ -1396,7 +1397,7 @@ Result ClusterLoadBalancer::HandleLeaderMoves( Status ClusterLoadBalancer::MoveReplica( const TabletId& tablet_id, const TabletServerId& from_ts, const TabletServerId& to_ts) { - LOG(INFO) << Substitute("Moving tablet $0 from $1 to $2", tablet_id, from_ts, to_ts); + LOG(INFO) << Substitute("Moving replica $0 from $1 to $2", tablet_id, from_ts, to_ts); RETURN_NOT_OK(SendReplicaChanges(GetTabletMap().at(tablet_id), to_ts, true /* is_add */, true /* should_remove_leader */)); RETURN_NOT_OK(state_->AddReplica(tablet_id, to_ts)); @@ -1405,7 +1406,7 @@ Status ClusterLoadBalancer::MoveReplica( } Status ClusterLoadBalancer::AddReplica(const TabletId& tablet_id, const TabletServerId& to_ts) { - LOG(INFO) << Substitute("Adding tablet $0 to $1", tablet_id, to_ts); + LOG(INFO) << Substitute("Adding replica $0 to $1", tablet_id, to_ts); // This is an add operation, so the "should_remove_leader" flag is irrelevant. RETURN_NOT_OK(SendReplicaChanges(GetTabletMap().at(tablet_id), to_ts, true /* is_add */, true /* should_remove_leader */)); diff --git a/src/yb/master/cluster_balance_util.cc b/src/yb/master/cluster_balance_util.cc index 58a7bfa81f25..a93f2249d7cd 100644 --- a/src/yb/master/cluster_balance_util.cc +++ b/src/yb/master/cluster_balance_util.cc @@ -445,8 +445,9 @@ Result PerTableLoadState::CanAddTabletToTabletServer( } // If we ask to use placement information, check against it. if (placement_info && !GetValidPlacement(to_ts, placement_info).has_value()) { - YB_LOG_EVERY_N_SECS(INFO, 30) << "tablet server " << to_ts << " has invalid placement info. " - << "Not allowing it to take more tablets."; + YB_LOG_EVERY_N_SECS(INFO, 30) << "tablet server " << to_ts << " has placement info " + << "incompatible with tablet " << tablet_id << ". Not allowing " + << "it to host this tablet."; return false; } // If this server has a pending tablet delete, don't use it. @@ -497,7 +498,7 @@ Result PerTableLoadState::CanSelectWrongReplicaToMove( VERIFY_RESULT(CanAddTabletToTabletServer(tablet_id, to_uuid, &placement_info))) { found_match = true; } else { - if (VERIFY_RESULT(CanAddTabletToTabletServer(tablet_id, to_uuid))) { + if (VERIFY_RESULT(CanAddTabletToTabletServer(tablet_id, to_uuid, &placement_info))) { // If we have placement information, we want to only pick the tablet if it's moving // to the same placement, so we guarantee we're keeping the same type of distribution. // Since we allow prefixes as well, we can still respect the placement of this tablet diff --git a/src/yb/master/cluster_balance_util.h b/src/yb/master/cluster_balance_util.h index b0d561fa7856..a6577a9ec17a 100644 --- a/src/yb/master/cluster_balance_util.h +++ b/src/yb/master/cluster_balance_util.h @@ -296,8 +296,7 @@ class PerTableLoadState { virtual void UpdateTabletServer(std::shared_ptr ts_desc); Result CanAddTabletToTabletServer( - const TabletId& tablet_id, const TabletServerId& to_ts, - const PlacementInfoPB* placement_info = nullptr); + const TabletId& tablet_id, const TabletServerId& to_ts, const PlacementInfoPB* placement_info); // For a TS specified by ts_uuid, this function checks if there is a placement // block in placement_info where this TS can be placed. If there doesn't exist From b5ec525a52b1c6621fdb4aa229b51ba17cb9a9bf Mon Sep 17 00:00:00 2001 From: Aishwarya Chakravarthy Date: Thu, 2 Mar 2023 11:46:49 -0500 Subject: [PATCH 41/81] [docs] C# smart driver page (#16217) * added c# smart driver page * added reference section * merged master so added minor change * yb_servers_refresh_interval parameter * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/reference/drivers/ysql-client-drivers.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/reference/drivers/ysql-client-drivers.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/reference/drivers/ysql-client-drivers.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/reference/drivers/ysql-client-drivers.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * use the driver change remaining * added use the driver details * more changes from review * added driver version * SSL steps and YB version * added links for ssl * Update docs/content/preview/drivers-orms/csharp/ysql.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> * Update docs/content/preview/reference/drivers/csharp/yb-npgsql-reference.md Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> --------- Co-authored-by: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> --- docs/content/preview/drivers-orms/_index.md | 1 + .../preview/drivers-orms/csharp/_index.md | 11 +- .../drivers-orms/csharp/postgres-npgsql.md | 19 +- .../preview/drivers-orms/csharp/ycql.md | 2 +- .../preview/drivers-orms/csharp/ysql.md | 208 ++++++++++++++++++ .../preview/drivers-orms/smart-drivers.md | 1 + .../preview/reference/drivers/_index.md | 2 +- .../csharp/postgres-npgsql-reference.md | 11 +- .../drivers/csharp/yb-npgsql-reference.md | 204 +++++++++++++++++ .../reference/drivers/ysql-client-drivers.md | 34 ++- 10 files changed, 480 insertions(+), 13 deletions(-) create mode 100644 docs/content/preview/drivers-orms/csharp/ysql.md create mode 100644 docs/content/preview/reference/drivers/csharp/yb-npgsql-reference.md diff --git a/docs/content/preview/drivers-orms/_index.md b/docs/content/preview/drivers-orms/_index.md index 0e5ba316706b..42e491ad4e53 100644 --- a/docs/content/preview/drivers-orms/_index.md +++ b/docs/content/preview/drivers-orms/_index.md @@ -184,6 +184,7 @@ The following libraries are officially supported by YugabyteDB. | Driver/ORM | Support Level | Example apps | | :--------- | :------------ | :----------- | +| [YugabyteDB C# Smart Driver for YSQL](csharp/ysql/) | Full | [CRUD](csharp/ysql/) | | [PostgreSQL Npgsql Driver](csharp/postgres-npgsql/) | Full | [CRUD](csharp/postgres-npgsql/) | | [YugabyteDB C# Driver for YCQL](csharp/ycql/) | Full | [CRUD](csharp/ycql/) | | [Entity Framework](csharp/entityframework/) | Full | [CRUD](csharp/entityframework/) | diff --git a/docs/content/preview/drivers-orms/csharp/_index.md b/docs/content/preview/drivers-orms/csharp/_index.md index 05f119d6b682..466a15ea8cfd 100644 --- a/docs/content/preview/drivers-orms/csharp/_index.md +++ b/docs/content/preview/drivers-orms/csharp/_index.md @@ -19,7 +19,8 @@ The following projects can be used to implement C# applications using the Yugaby | Project | Documentation and Guides | Latest Driver Version | Supported YugabyteDB Version | | :------ | :----------------------- | :-------------------- | :--------------------------- | -| PostgreSQL Npgsql Driver | [Documentation](postgres-npgsql/)
[Reference](../../reference/drivers/csharp/postgres-npgsql-reference/) | [6.0.3](https://www.nuget.org/packages/Npgsql/) | 2.6 and above +| YugabyteDB C# Driver for YSQL [Recommended] | [Documentation](ysql/)
[Reference](../../reference/drivers/csharp/yb-npgsql-reference/) | [8.0.0-yb-1-beta](https://www.nuget.org/packages/NpgsqlYugabyteDB/) | 2.8 and later +| PostgreSQL Npgsql Driver | [Documentation](postgres-npgsql/)
[Reference](../../reference/drivers/csharp/postgres-npgsql-reference/) | [6.0.3](https://www.nuget.org/packages/Npgsql/) | 2.6 and later | YugabyteDB C# Driver for YCQL | [Documentation](ycql/) | | Project | Documentation and Guides | Example Apps | @@ -27,9 +28,9 @@ The following projects can be used to implement C# applications using the Yugaby | Entity Framework | [Documentation](entityframework/)
[Hello World](../orms/csharp/ysql-entity-framework/) | [Entity Framework ORM App](https://github.com/YugabyteDB-Samples/orm-examples/tree/master/csharp/entityframework) | | Dapper | [Hello World](../orms/csharp/ysql-dapper/) | [Dapper ORM App](https://github.com/YugabyteDB-Samples/orm-examples/tree/master/csharp/dapper/DapperORM) | -Learn how to establish a connection to a YugabyteDB database and begin basic CRUD operations by referring to [Connect an app](postgres-npgsql/) or [Use an ORM](entityframework/). +Learn how to establish a connection to a YugabyteDB database and begin basic CRUD operations by referring to [Connect an app](ysql/) or [Use an ORM](entityframework/). -For reference documentation, including using projects with SSL, refer to the [drivers and ORMs reference](../../reference/drivers/csharp/postgres-npgsql-reference/) pages. +For reference documentation, including using projects with SSL, refer to the [drivers and ORMs reference](../../reference/drivers/csharp/yb-npgsql-reference/) pages. ## Prerequisites @@ -41,7 +42,7 @@ To develop C# applications for YugabyteDB, you need the following: - **Create a C# project**\ For ease-of-use, use an integrated development environment (IDE) such as Visual Studio. To download and install Visual Studio, visit the [Visual Studio Downloads](https://visualstudio.microsoft.com/downloads/) page. - To create a C# project in Visual Studio, select [Console Application](https://docs.microsoft.com/en-us/dotnet/core/tutorials/with-visual-studio?pivots=dotnet-6-0) as template when creating a new project. - - If you are not using an IDE, use the dotnet command: + - If you are not using an IDE, use the following dotnet command: ```csharp dotnet new console -o new_project_name @@ -53,4 +54,4 @@ To develop C# applications for YugabyteDB, you need the following: ## Next step -- [Connect an app](postgres-npgsql/) +- [Connect an app](ysql/) diff --git a/docs/content/preview/drivers-orms/csharp/postgres-npgsql.md b/docs/content/preview/drivers-orms/csharp/postgres-npgsql.md index 8f89276011aa..6a0e3c00d608 100644 --- a/docs/content/preview/drivers-orms/csharp/postgres-npgsql.md +++ b/docs/content/preview/drivers-orms/csharp/postgres-npgsql.md @@ -7,13 +7,13 @@ menu: preview: identifier: postgres-npgsql-driver parent: csharp-drivers - weight: 400 + weight: 420 type: docs ---
  • - + YSQL
    • @@ -26,6 +26,13 @@ type: docs
      +
    • + + + YugabyteDB Npgsql Smart Driver + +
      • +
    • @@ -256,7 +263,13 @@ namespace Yugabyte_CSharp_Demo } ``` -To run the project `Program.cs`, choose _Run -> Start Without Debugging_. +## Run the application + +To run the project `Program.cs` in Visual Studio Code, from the **Run** menu, choose **Start Without Debugging**. If you aren't using an IDE, enter the following command: + +```csharp +dotnet run +``` You should see output similar to the following if you're using SSL: diff --git a/docs/content/preview/drivers-orms/csharp/ycql.md b/docs/content/preview/drivers-orms/csharp/ycql.md index 692d34824166..e6c22bb0e52e 100644 --- a/docs/content/preview/drivers-orms/csharp/ycql.md +++ b/docs/content/preview/drivers-orms/csharp/ycql.md @@ -13,7 +13,7 @@ type: docs
      • - + YSQL
        • diff --git a/docs/content/preview/drivers-orms/csharp/ysql.md b/docs/content/preview/drivers-orms/csharp/ysql.md new file mode 100644 index 000000000000..a08c2fa158c5 --- /dev/null +++ b/docs/content/preview/drivers-orms/csharp/ysql.md @@ -0,0 +1,208 @@ +--- +title: Connect an application +linkTitle: Connect an app +description: C# driver for YSQL +image: /images/section_icons/sample-data/s_s1-sampledata-3x.png +menu: + preview: + identifier: csharp-driver-ysql + parent: csharp-drivers + weight: 400 +type: docs +--- + + + + + +The [Yugabyte Npgsql smart driver](https://github.com/yugabyte/npgsql) is a .NET driver for [YSQL](../../../api/ysql/) built on the [PostgreSQL Npgsql driver](https://github.com/npgsql/npgsql/tree/main/src/Npgsql), with additional [connection load balancing](../../smart-drivers/) features. + +{{< note title="YugabyteDB Managed" >}} + +To use smart driver load balancing features when connecting to clusters in YugabyteDB Managed, applications must be deployed in a VPC that has been peered with the cluster VPC. For applications that access the cluster from a non-peered network, use the upstream PostgreSQL driver instead; in this case, the cluster performs the load balancing. Applications that use smart drivers from non-peered networks fall back to the upstream driver behaviour automatically. For more information, refer to [Using smart drivers with YugabyteDB Managed](../../smart-drivers/#using-smart-drivers-with-yugabytedb-managed). + +{{< /note >}} + +## CRUD operations + +The following sections demonstrate how to perform common tasks required for C# application development using the Yugabyte Npgsql smart driver APIs. + +To start building your application, make sure you have met the [prerequisites](../#prerequisites). + +### Step 1: Add the Npgsql driver dependency + +If you are using Visual Studio, add the Npgsql package to your project as follows: + +1. Right-click **Dependencies** and choose **Manage Nuget Packages**. +1. Search for `NpgsqlYugabyteDB` and click **Add Package**. You may need to click the **Include prereleases** checkbox. + +To add the Npgsql package to your project when not using an IDE, use the following `dotnet` command: + +```csharp +dotnet add package NpgsqlYugabyteDB +``` + +or any of the other methods mentioned on the [nuget page](https://www.nuget.org/packages/NpgsqlYugabyteDB/) for NpgsqlYugabyteDB. + +### Step 2: Set up the database connection + +After setting up the dependencies, implement a C# client application that uses the Npgsql YugabyteDB driver to connect to your YugabyteDB cluster and run a query on the sample data. + +Import YBNpgsql and use the `NpgsqlConnection` class for getting connection objects for the YugabyteDB database that can be used for performing DDLs and DMLs against the database. + +The following table describes the connection parameters required to connect, including [smart driver parameters](../../smart-drivers/) for uniform and topology load balancing. + +| Parameter | Description | Default | +| :-------- | :---------- | :------ | +| Host | Host name of the YugabyteDB instance. You can also enter [multiple addresses](#use-multiple-addresses). | localhost +| Port | Listen port for YSQL | 5433 +| Database | Database name | yugabyte +| Username | User connecting to the database | yugabyte +| Password | Password for the user | yugabyte +| Load Balance Hosts | [Uniform load balancing](../../smart-drivers/#cluster-aware-connection-load-balancing) | False | +| YB Servers Refresh Interval | If Load Balance Hosts is true, the interval in seconds to refresh the servers list | 300 | +| Topology Keys | [Topology-aware load balancing](../../smart-drivers/#topology-aware-connection-load-balancing) | Null | + +{{< note title ="Note" >}} +The behaviour of `Load Balance Hosts` is different in YugabyteDB Npgsql Smart Driver as compared to the upstream driver. The upstream driver balances connections on the list of hosts provided in the `Host` property, whereas the smart driver balances the connections on the list of servers returned by the `yb_servers()` function. +{{< /note >}} + +The following is an example of a basic connection string for connecting to YugabyteDB: + +```csharp +var connStringBuilder = "Host=localhost;Port=5433;Database=yugabyte;Username=yugabyte;Password=password;Load Balance Hosts=true" +NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder) +``` + +After the driver establishes the initial connection, it fetches the list of available servers from the cluster, and load-balances subsequent connection requests across these servers. + +#### Use multiple addresses + +You can specify multiple hosts in the connection string to provide alternative options during the initial connection in case the primary address fails. Delimit the addresses using commas, as follows: + +```csharp +var connStringBuilder = "Host=127.0.0.1,127.0.0.2,127.0.0.3;Port=5433;Database=yugabyte;Username=yugabyte;Password=password;Load Balance Hosts=true" +NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder) +``` + +#### Use topology-aware load balancing + +To use topology-aware load balancing, specify the topology keys by setting the `Topology Keys` parameter, as per the following example: + +```csharp +var connStringBuilder = "Host=127.0.0.1,127.0.0.2,127.0.0.3;Port=5433;Database=yugabyte;Username=yugabyte;Password=password;Load Balance Hosts=true;Topology Keys=cloud.region.zone" +NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder) +``` + +You can pass multiple keys to the `Topology Keys` property, and give each of them a preference value, as per the following example: + +```csharp +var connStringBuilder = "Host=127.0.0.1,127.0.0.2,127.0.0.3;Port=5433;Database=yugabyte;Username=yugabyte;Password=password;Load Balance Hosts=true;Topology Keys=cloud1.region1.zone1:1,cloud2.region2.zone2:2"; +NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder) +``` + +#### Use SSL + +The YugabyteDB Npgsql smart driver support for SSL is the same as for the upstream driver. To set up the driver properties to configure the credentials and SSL certificates for connecting to your cluster, refer to [Use SSL](../postgres-npgsql/#use-ssl). + +### Step 3 : Write your application + +Copy the following code to the `Program.cs` file to set up YugbyteDB tables and query the table contents from the C# client. Replace the connection string `connStringBuilder` with the credentials of your cluster, and SSL certificates if required. + +```csharp +using System; +using YBNpgsql; +namespace Yugabyte_CSharp_Demo +{ + class Program + { + static void Main(string[] args) + { + var connStringBuilder = "host=localhost;port=5433;database=yugabyte;userid=yugabyte;password=xxx;Load Balance Hosts=true"; + NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder); + try + { + conn.Open(); + NpgsqlCommand empCreateCmd = new NpgsqlCommand("CREATE TABLE employee (id int PRIMARY KEY, name varchar, age int, language varchar);", conn); + empCreateCmd.ExecuteNonQuery(); + Console.WriteLine("Created table Employee"); + NpgsqlCommand empInsertCmd = new NpgsqlCommand("INSERT INTO employee (id, name, age, language) VALUES (1, 'John', 35, 'CSharp');", conn); + int numRows = empInsertCmd.ExecuteNonQuery(); + Console.WriteLine("Inserted data (1, 'John', 35, 'CSharp')"); + NpgsqlCommand empPrepCmd = new NpgsqlCommand("SELECT name, age, language FROM employee WHERE id = @EmployeeId", conn); + empPrepCmd.Parameters.Add("@EmployeeId", YBNpgsqlTypes.NpgsqlDbType.Integer); + empPrepCmd.Parameters["@EmployeeId"].Value = 1; + NpgsqlDataReader reader = empPrepCmd.ExecuteReader(); + Console.WriteLine("Query returned:\nName\tAge\tLanguage"); + while (reader.Read()) + { + Console.WriteLine("{0}\t{1}\t{2}", reader.GetString(0), reader.GetInt32(1), reader.GetString(2)); + } + } + catch (Exception ex) + { + Console.WriteLine("Failure: " + ex.Message); + } + finally + { + if (conn.State != System.Data.ConnectionState.Closed) + { + conn.Close(); + } + } + } + } +} +``` + +## Run the application + +To run the project `Program.cs` in Visual Studio Code, from the **Run** menu, choose **Start Without Debugging**. If you aren't using an IDE, enter the following command: + +```csharp +dotnet run +``` + +You should see output similar to the following: + +```output +Created table Employee +Inserted data (1, 'John', 35, 'CSharp') +Query returned: +Name Age Language +John 35 CSharp +``` + +## Learn more + +- [YugabyteDB smart drivers for YSQL](../../smart-drivers/) +- [Smart Driver architecture](https://github.com/yugabyte/yugabyte-db/blob/master/architecture/design/smart-driver.md) +- [YugabyteDB Npgsql Smart Driver reference](../../../reference/drivers/csharp/yb-npgsql-reference/) +- Build C# applications using [Entity Framework ORM](../entityframework) diff --git a/docs/content/preview/drivers-orms/smart-drivers.md b/docs/content/preview/drivers-orms/smart-drivers.md index 4727063af15c..4a509d1f393e 100644 --- a/docs/content/preview/drivers-orms/smart-drivers.md +++ b/docs/content/preview/drivers-orms/smart-drivers.md @@ -22,6 +22,7 @@ Yugabyte has developed the following smart drivers for YSQL, available as open s | [YugabyteDB PGX Driver for Go](https://github.com/yugabyte/pgx) | jackc/pgx | [Documentation](../go/yb-pgx/) | | [YugabyteDB Psycopg2 Driver for Python](https://github.com/yugabyte/psycopg2) | PostgreSQL psycopg2 | [Documentation](../python/yugabyte-psycopg2/) | | [YugabyteDB node-postgres Driver for Node.js](https://github.com/yugabyte/node-postgres) | node-postgres | [Documentation](../nodejs/yugabyte-node-driver/) | +| [YugabyteDB Npgsql Driver for C#](https://github.com/yugabyte/npgsql) | PostgreSQL Npgsql Driver | [Documentation](../csharp/ysql/) | All YugabyteDB smart driver libraries are actively maintained, and receive bug fixes, performance enhancements, and security patches. diff --git a/docs/content/preview/reference/drivers/_index.md b/docs/content/preview/reference/drivers/_index.md index a79071fc2142..0bf4e34855a7 100644 --- a/docs/content/preview/reference/drivers/_index.md +++ b/docs/content/preview/reference/drivers/_index.md @@ -46,7 +46,7 @@ type: indexpage
      • - + C# diff --git a/docs/content/preview/reference/drivers/csharp/postgres-npgsql-reference.md b/docs/content/preview/reference/drivers/csharp/postgres-npgsql-reference.md index 51d35c08fd6d..7c6df19e8a4a 100644 --- a/docs/content/preview/reference/drivers/csharp/postgres-npgsql-reference.md +++ b/docs/content/preview/reference/drivers/csharp/postgres-npgsql-reference.md @@ -9,16 +9,23 @@ menu: name: C# Drivers identifier: ref-postgres-npgsql-driver parent: drivers - weight: 600 + weight: 620 type: docs ---
          +
        • + + + YugabyteDB Npgsql Smart Driver + +
          • +
        • - PostgreSQL Npgsql + PostgreSQL Npgsql Driver
          • diff --git a/docs/content/preview/reference/drivers/csharp/yb-npgsql-reference.md b/docs/content/preview/reference/drivers/csharp/yb-npgsql-reference.md new file mode 100644 index 000000000000..7c603f7417b2 --- /dev/null +++ b/docs/content/preview/reference/drivers/csharp/yb-npgsql-reference.md @@ -0,0 +1,204 @@ +--- +title: C# Drivers +linkTitle: C# Drivers +description: C# Drivers for YSQL +headcontent: C# Drivers for YSQL +image: /images/section_icons/sample-data/s_s1-sampledata-3x.png +menu: + preview: + name: C# Drivers + identifier: ref-yb-npgsql-driver + parent: drivers + weight: 600 +type: docs +--- + + + +Yugabyte Npgsql smart driver is a .NET driver for [YSQL](../../../../api/ysql/) based on [PostgreSQL Npgsql driver](https://github.com/npgsql/npgsql/tree/main/src/Npgsql), with additional connection load balancing features. + +For more information on the Yugabyte Npgsql smart driver, see the following: + +- [YugabyteDB smart drivers for YSQL](../../../../drivers-orms/smart-drivers/) +- [CRUD operations](../../../../drivers-orms/go/ysql) +- [GitHub repository](https://github.com/yugabyte/npgsql) +- [Smart Driver architecture](https://github.com/yugabyte/yugabyte-db/blob/master/architecture/design/smart-driver.md) + +## Download the driver dependency + +If you are using Visual Studio IDE, add the NpgsqlYugabyteDB package to your project as follows: + +1. Right-click **Dependencies** and choose **Manage Nuget Packages** +1. Search for `NpgsqlYugabyteDB` and click **Add Package**. You may need to click the **Include prereleases** checkbox. + +To add the NpgsqlYugabyteDB package to your project when not using an IDE, use the following `dotnet` command: + +```csharp +dotnet add package NpgsqlYugabyteDB +``` + +or any of the other methods mentioned on the [nuget page](https://www.nuget.org/packages/Npgsql/) for NpgsqlYugabyteDB. + +## Fundamentals + +Learn how to perform common tasks required for C# application development using the Npgsql YugabyteDB driver. + +### Load balancing connection properties + +The following connection properties need to be added to enable load balancing: + +- `Load Balance Hosts` - enable cluster-aware load balancing by setting this property to `true`; disabled by default. +- `Topology Keys` - provide comma-separated geo-location values to enable topology-aware load balancing. Geo-locations can be provided as `cloud.region.zone`. Specify all zones in a region as `cloud.region.*`. To designate fallback locations for when the primary location is unreachable, specify a priority in the form `:n`, where `n` is the order of precedence. For example, `cloud1.datacenter1.rack1:1,cloud1.datacenter1.rack2:2`. + +By default, the driver refreshes the list of nodes every 300 seconds (5 minutes). You can change this value by including the `YB Servers Refresh Interval` connection parameter. + +### Use the driver + +To use the driver, pass new connection properties for load balancing in the connection URL or properties pool. + +To enable uniform load balancing across all servers, you set the `Load Balance Hosts` property to `true` in the URL, as per the following example: + +```csharp +var connStringBuilder = "Host=127.0.0.1,127.0.0.2,127.0.0.3;Port=5433;Database=yugabyte;Username=yugabyte;Password=password;Load Balance Hosts=true;" +NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder) +``` + +You can specify [multiple hosts](../../../../drivers-orms/csharp/ysql/#use-multiple-addresses) in the connection string in case the primary address fails. After the driver establishes the initial connection, it fetches the list of available servers from the universe, and performs load balancing of subsequent connection requests across these servers. + +To specify topology keys, you set the `Topology Keys` property to comma separated values, as per the following example: + +```go +var connStringBuilder = "Host=127.0.0.1,127.0.0.2,127.0.0.3;Port=5433;Database=yugabyte;Username=yugabyte;Password=password;Load Balance Hosts=true;Topology Keys=cloud.region.zone" +NpgsqlConnection conn = new NpgsqlConnection(connStringBuilder) +``` + +### Create table + +Tables can be created in YugabyteDB by passing the `CREATE TABLE` DDL statement to the `NpgsqlCommand` class and getting a command object, then calling the `ExecuteNonQuery()` method using this command object. + +```sql +CREATE TABLE employee (id int PRIMARY KEY, name varchar, age int, language varchar) +``` + +```csharp +conn.Open(); +NpgsqlCommand empCreateCmd = new NpgsqlCommand("CREATE TABLE employee (id int PRIMARY KEY, name varchar, age int, language varchar);", conn); +empCreateCmd.ExecuteNonQuery(); +``` + +### Read and write data + +#### Insert data + +To write data into YugabyteDB, execute the `INSERT` statement using the `NpgsqlCommand` class, get a command object, and then call the `ExecuteNonQuery()` method using this command object. + +```sql +INSERT INTO employee (id, name, age, language) VALUES (1, 'John', 35, 'CSharp'); +``` + +```csharp +NpgsqlCommand empInsertCmd = new NpgsqlCommand("INSERT INTO employee (id, name, age, language) VALUES (1, 'John', 35, 'CSharp');", conn); +int numRows = empInsertCmd.ExecuteNonQuery(); +``` + +#### Query data + +To query data from YugabyteDB tables, execute the `SELECT` statement using the `NpgsqlCommand` class, get a command object, and then call the `ExecuteReader()` function using the object. Loop through the reader to get the list of returned rows. + +```sql +SELECT * from employee where id=1; +``` + +```csharp +NpgsqlCommand empPrepCmd = new NpgsqlCommand("SELECT name, age, language FROM employee WHERE id = @EmployeeId", conn); +empPrepCmd.Parameters.Add("@EmployeeId", YBNpgsqlTypes.NpgsqlDbType.Integer); + +empPrepCmd.Parameters["@EmployeeId"].Value = 1; +NpgsqlDataReader reader = empPrepCmd.ExecuteReader(); + +Console.WriteLine("Query returned:\nName\tAge\tLanguage"); +while (reader.Read()) +{ + Console.WriteLine("{0}\t{1}\t{2}", reader.GetString(0), reader.GetInt32(1), reader.GetString(2)); +} +``` + +### Configure SSL/TLS + +The YugabyteDB Npgsql smart driver support for SSL is the same as for the upstream driver. For information on using SSL/TLS for your application, refer to the .NET Npgsql driver's [Configure SSL/TLS](../postgres-npgsql-reference/#configure-ssl-tls) instructions. + + diff --git a/docs/content/preview/reference/drivers/ysql-client-drivers.md b/docs/content/preview/reference/drivers/ysql-client-drivers.md index b4b01e4770a9..e03ae447bf9b 100644 --- a/docs/content/preview/reference/drivers/ysql-client-drivers.md +++ b/docs/content/preview/reference/drivers/ysql-client-drivers.md @@ -76,13 +76,21 @@ $ make install ## C\# +### Npgsql YugabyteDB Smart Driver + +The YugabyteDB Npgsql Smart Driver is a .NET driver for YSQL built on the PostgreSQL Npgsql driver, with features that eliminate the need for external load balancers. + +For information on the YugabyteDB Npgsql Smart Driver and its load balancing features, see [YugabyteDB Npgsql Smart Driver](../../../reference/drivers/csharp/yb-npgsql-reference/). + +For building a sample C# application with the YugabyteDB Npgsql driver, see [Connect an application](../../../drivers-orms/csharp/ysql/). + ### Npgsql [Npgsql](https://www.npgsql.org/) is an open source ADO.NET Data Provider for PostgreSQL that enables C# applications to connect and interact with PostgreSQL databases. Npgsql is based on [libpq](#libpq) and supports the [SCRAM-SHA-256 authentication method](../../../secure/authentication/password-authentication/#scram-sha-256). For details on Npgsql, refer to the [Npgsql documentation](https://www.npgsql.org/doc/). -For a tutorial on building a sample C# application with Npgsql, see [Connect an application](../../../drivers-orms/csharp/postgres-npgsql/). +For building a sample C# application with Npgsql, see [Connect an application](../../../drivers-orms/csharp/postgres-npgsql/). #### Install the driver @@ -112,6 +120,14 @@ connStringBuilder.ServerCompatibilityMode = ServerCompatibilityMode.NoTypeLoadin ## Go +### YugabyteDB PGX Smart Driver + +The YugabyteDB PGX smart driver is a distributed Go driver for YSQL based on jackc/pgx, with a additional connection load balancing features. + +For information on the YugabyteDB PGX smart driver and its load balancing features, see [YugabyteDB PGX smart driver](../../../reference/drivers/go/yb-pgx-reference/). + +For building a sample Go application with the YugabyteDB PGX driver, see [Connect an application](../../../drivers-orms/go/yb-pgx/). + ### Go PostgreSQL driver (pq) The [Go PostgreSQL driver package (pq)](https://pkg.go.dev/github.com/lib/pq?tab=doc) is a Go PostgreSQL driver for the `database/sql` package. pq is not based on [libpq](#libpq), but supports the [SCRAM-SHA-256 authentication method](../../../secure/authentication/password-authentication/#scram-sha-256). @@ -160,6 +176,14 @@ To get the latest versions for projects using [Apache Maven](https://maven.apach ## Node.js +### YugabyteDB node-postgres Smart Driver + +The YugabyteDB node-postgres Smart Driver is a distributed Node.js driver for YSQL built on the PostgreSQL node-postgres driver, with features that eliminate the need for external load balancers. + +For information on the YugabyteDB node-postgres Smart Driver and its load balancing features, see [YugabyteDB node-postgres Smart Driver](../../../reference/drivers/nodejs/yugabyte-pg-reference/). + +For building a sample node.js application with the YugabyteDB node-postgres driver, see [Connect an application](../../../drivers-orms/nodejs/yugabyte-node-driver/). + ### node-postgres [node-postgres](https://node-postgres.com/) is a collection of Node.js modules for interacting with PostgreSQL databases. node-postgres optionally uses [libpq](#libpq) and supports the [SCRAM-SHA-256 authentication method](../../../secure/authentication/password-authentication/#scram-sha-256). @@ -198,6 +222,14 @@ CentOS users can install the driver using the `sudo yum install php-pgsql` comma ## Python +### YugabyteDB Psycopg2 Smart Driver + +The YugabyteDB Psycopg2 Smart Driver is a distributed Node.js driver for YSQL built on the PostgreSQL Psycopg2 driver, with features that eliminate the need for external load balancers. + +For information on the YugabyteDB Psycopg2 Smart Driver and its load balancing features, see [YugabyteDB Psycopg2 Smart Driver](../../../reference/drivers/python/yugabyte-psycopg2-reference/). + +For building a sample Python application with the YugabyteDB Psycopg2 driver, see [Connect an application](../../../drivers-orms/python/yugabyte-psycopg2/). + ### psycopg2 [Psycopg](https://www.psycopg.org/) is the popular PostgreSQL database adapter for the Python programming language. psycopg2 is based on [libpq](#libpq) and supports the [SCRAM-SHA-256 authentication method](../../../secure/authentication/password-authentication/#scram-sha-256). From a652b1041eef1fac605364e83b00219347d66838 Mon Sep 17 00:00:00 2001 From: saratkumar-yb Date: Thu, 2 Mar 2023 09:46:11 +0000 Subject: [PATCH 42/81] Fix for shellcheck failures caused due to D22071 Summary: Github CI shellcheck started failing because of D22071 change. Test Plan: Github actions CI check should pass along with code review pipeline. Reviewers: hsunder, steve.varnau Reviewed By: steve.varnau Differential Revision: https://phabricator.dev.yugabyte.com/D23273 --- build-support/common-build-env.sh | 5 ++++- build-support/jenkins/build.sh | 8 ++------ build-support/jenkins/test.sh | 1 + 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build-support/common-build-env.sh b/build-support/common-build-env.sh index a9ad1144a27f..f69c6c11fa45 100644 --- a/build-support/common-build-env.sh +++ b/build-support/common-build-env.sh @@ -2171,10 +2171,13 @@ run_shellcheck() { build-support/common-build-env.sh build-support/common-cli-env.sh build-support/common-test-env.sh + build-support/jenkins/common-lto.sh build-support/compiler-wrappers/compiler-wrapper.sh build-support/find_linuxbrew.sh - build-support/jenkins/build-and-test.sh + build-support/jenkins/build.sh + build-support/jenkins/test.sh build-support/jenkins/yb-jenkins-build.sh + build-support/jenkins/yb-jenkins-test.sh build-support/run-test.sh yb_build.sh ) diff --git a/build-support/jenkins/build.sh b/build-support/jenkins/build.sh index c8a5aeaee388..17b8761e21c7 100755 --- a/build-support/jenkins/build.sh +++ b/build-support/jenkins/build.sh @@ -152,6 +152,7 @@ log "Removing old JSON-based test report files" activate_virtualenv set_pythonpath +# shellcheck source=build-support/jenkins/common-lto.sh . "${BASH_SOURCE%/*}/common-lto.sh" # ------------------------------------------------------------------------------------------------- @@ -284,11 +285,6 @@ fi YB_SKIP_CPP_COMPILATION=${YB_SKIP_CPP_COMPILATION:-0} YB_COMPILE_ONLY=${YB_COMPILE_ONLY:-0} -CTEST_OUTPUT_PATH="${BUILD_ROOT}"/ctest.log -CTEST_FULL_OUTPUT_PATH="${BUILD_ROOT}"/ctest-full.log - -TEST_LOG_DIR="${BUILD_ROOT}/test-logs" - configure_remote_compilation export NO_REBUILD_THIRDPARTY=1 @@ -607,4 +603,4 @@ else time "${YB_SRC_ROOT}/yb_build.sh" "${BUILD_TYPE}" --build-yugabyted-ui --skip-java fi -exit ${EXIT_STATUS} \ No newline at end of file +exit ${EXIT_STATUS} diff --git a/build-support/jenkins/test.sh b/build-support/jenkins/test.sh index 856bd71d3d25..81b1acfb4501 100755 --- a/build-support/jenkins/test.sh +++ b/build-support/jenkins/test.sh @@ -97,6 +97,7 @@ fi activate_virtualenv set_pythonpath +# shellcheck source=build-support/jenkins/common-lto.sh . "${BASH_SOURCE%/*}/common-lto.sh" # ------------------------------------------------------------------------------------------------- From a47e3f4826d4c174ad97a96fea48271d1b799f2f Mon Sep 17 00:00:00 2001 From: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> Date: Thu, 2 Mar 2023 13:20:05 -0500 Subject: [PATCH 43/81] [docs] [PLAT-7589] Azure KMS to Key Vault (#16282) --- .../preview/releases/release-notes/v2.15.md | 4 +- .../preview/releases/release-notes/v2.17.md | 4 - .../preview/releases/yba-releases/v2.16.md | 2 +- .../preview/releases/yba-releases/v2.17.md | 28 +---- .../security/create-kms-config/aws-kms.md | 64 +++++----- .../security/create-kms-config/azure-kms.md | 20 ++-- .../security/create-kms-config/google-kms.md | 14 +-- .../create-kms-config/hashicorp-kms.md | 111 +++++++++--------- .../security/create-kms-config/aws-kms.md | 6 +- .../security/create-kms-config/azure-kms.md | 18 +-- .../security/create-kms-config/google-kms.md | 12 +- .../create-kms-config/hashicorp-kms.md | 111 +++++++++--------- 12 files changed, 183 insertions(+), 211 deletions(-) diff --git a/docs/content/preview/releases/release-notes/v2.15.md b/docs/content/preview/releases/release-notes/v2.15.md index 000349e7efd3..28cc02ffc3f0 100644 --- a/docs/content/preview/releases/release-notes/v2.15.md +++ b/docs/content/preview/releases/release-notes/v2.15.md @@ -101,7 +101,7 @@ docker pull yugabytedb/yugabyte:2.15.3.0-b231 * [PLAT-3784] UI support for GCP KMS * [PLAT-3826] Switching default image to AlmaLinux8 for cloud providers -* [PLAT-4164] UI support for Azure KMS +* [PLAT-4164] UI support for Azure Key Vault * [PLAT-4704] [PLAT-5099] Add YBC Management APIs * [PLAT-4732] Support non-root Yba-installer workflow * [PLAT-4754] [DB-2913] Support expanding multi-AZ universe while a AZ is down @@ -112,7 +112,7 @@ docker pull yugabytedb/yugabyte:2.15.3.0-b231 * [PLAT-5090] Create Incremental backups * [PLAT-5142] Add API to upgrade k8s overrides. * [PLAT-5221] Add edit functionality for GCP KMS -* [PLAT-5254] Add edit functionality for Azure KMS +* [PLAT-5254] Add edit functionality for Azure Key Vault #### Database diff --git a/docs/content/preview/releases/release-notes/v2.17.md b/docs/content/preview/releases/release-notes/v2.17.md index 33e37eacd6ef..e5fce2fea3ec 100644 --- a/docs/content/preview/releases/release-notes/v2.17.md +++ b/docs/content/preview/releases/release-notes/v2.17.md @@ -157,7 +157,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [[12068](https://github.com/yugabyte/yugabyte-db/issues/12068)] [[14047](https://github.com/yugabyte/yugabyte-db/issues/14047)] [xCluster] Fix shutdown path of CDCConsumer * [[12307](https://github.com/yugabyte/yugabyte-db/issues/12307)] [YSQL] [Upgrade] Inserts to pg_depend should update YB pinned object cache * [[12442](https://github.com/yugabyte/yugabyte-db/issues/12442)] [YSQL] Add colocation_id to pg_dump for materialized views -* [[12442](https://github.com/yugabyte/yugabyte-db/issues/12442)] [YSQL] Add colocation_id to pg_dump for materialized views * [[12899](https://github.com/yugabyte/yugabyte-db/issues/12899)] [[14862](https://github.com/yugabyte/yugabyte-db/issues/14862)] [DocDB] Handle system keys in ScanChoices/Iterator * [[13069](https://github.com/yugabyte/yugabyte-db/issues/13069)] [DocDB] Drop Database removes metadata from tablet, persists and replicates this change to superblock * [[13102](https://github.com/yugabyte/yugabyte-db/issues/13102)] [DocDB] Fix deadlock in DoSplitTablet. @@ -204,7 +203,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [[14424](https://github.com/yugabyte/yugabyte-db/issues/14424)] [YCQL] Persist where predicate in IndexInfo for PartialIndexes * [[14426](https://github.com/yugabyte/yugabyte-db/issues/14426)] [yugabyted] [QA] Added validation for hostnames provided through CLI. * [[14442](https://github.com/yugabyte/yugabyte-db/issues/14442)] [xCluster] Fix Transaction Consistency Issues -* [[14442](https://github.com/yugabyte/yugabyte-db/issues/14442)] [xCluster] Fix Transaction Consistency Issues * [[14471](https://github.com/yugabyte/yugabyte-db/issues/14471)] [YSQL] Fix crash upon concurrent DMLs on two tables with foreign key relationship * [[14479](https://github.com/yugabyte/yugabyte-db/issues/14479)] [CDCSDK] Add bool parameter to error out of UpdateCdcReplicatedIndex RPC fails on any TServer * [[14489](https://github.com/yugabyte/yugabyte-db/issues/14489)] [DocDB] fix term mismatch caused by stale consensus meta data passed to InitTabletPeer @@ -274,14 +272,12 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [[14974](https://github.com/yugabyte/yugabyte-db/issues/14974)] [YSQL] Fix name conflict in aset realloc * [[14987](https://github.com/yugabyte/yugabyte-db/issues/14987)] [[14988](https://github.com/yugabyte/yugabyte-db/issues/14988)] [xCluster] Check for system replication in IsSetupUniverseReplicationDone * [[15001](https://github.com/yugabyte/yugabyte-db/issues/15001)] Adjust XCluster DDL Verification to Serial Equivalence After PackedColumns -* [[15001](https://github.com/yugabyte/yugabyte-db/issues/15001)] Adjust XCluster DDL Verification to Serial Equivalence After PackedColumns * [[15004](https://github.com/yugabyte/yugabyte-db/issues/15004)] [CDCSDK] Dataloss when streaming transaction intents for large tables * [[15006](https://github.com/yugabyte/yugabyte-db/issues/15006)] [DocDB] set same error to operations with same request id when retrying a batcher * [[15016](https://github.com/yugabyte/yugabyte-db/issues/15016)] [DocDB] YSQL select max() returns inconsistent result after tablet splitting * [[15016](https://github.com/yugabyte/yugabyte-db/issues/15016)] Fixing heap use after free (caught by asan) * [[15016](https://github.com/yugabyte/yugabyte-db/issues/15016)] Recalculation of a partition key only when it is really necessary * [[15025](https://github.com/yugabyte/yugabyte-db/issues/15025)] [YCQL] Fixing Index updating by DELETE for a deleted row -* [[15025](https://github.com/yugabyte/yugabyte-db/issues/15025)] [YCQL] Fixing Index updating by DELETE for a deleted row * [[15027](https://github.com/yugabyte/yugabyte-db/issues/15027)] [DocDB] Fix to compute the middle of middle key for tablet splitting * [[15032](https://github.com/yugabyte/yugabyte-db/issues/15032)] [CDCSDK] Alter table support with CDC snapshot * [[15044](https://github.com/yugabyte/yugabyte-db/issues/15044)] [YSQL] Fix initial_sys_catalog_snapshot dependencies diff --git a/docs/content/preview/releases/yba-releases/v2.16.md b/docs/content/preview/releases/yba-releases/v2.16.md index 55d3cb95b5e1..e38e618fd45d 100644 --- a/docs/content/preview/releases/yba-releases/v2.16.md +++ b/docs/content/preview/releases/yba-releases/v2.16.md @@ -270,7 +270,7 @@ docker pull yugabytedb/yugabyte:2.16.0.0-b90 * [PLAT-6037] Prevent GFlags deletion through the Non-Restart upgrade option * [PLAT-6042] Fix Kubernetes CPU usage graph * [PLAT-6059] Fix configure_universe API -* [PLAT-6074] Decrypt against all key versions for a key in azure KMS +* [PLAT-6074] Decrypt against all key versions for a key in Azure Key Vault * [PLAT-6080] [Backup] [UI] Not able to select backups checkbox * [PLAT-6091] [PLAT-6126] Onprem custom home directory, Throw Bad Request in the case of a ybc backup restore to a non-ybc universe * [PLAT-6104] [Incremental Backups] [UI] Throw UI warning for full backup deletion request when associated incremental backup in progress diff --git a/docs/content/preview/releases/yba-releases/v2.17.md b/docs/content/preview/releases/yba-releases/v2.17.md index 6802e2350d77..0cf09f6d536d 100644 --- a/docs/content/preview/releases/yba-releases/v2.17.md +++ b/docs/content/preview/releases/yba-releases/v2.17.md @@ -40,7 +40,7 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-4402] [PLAT-5540] Volume resize for k8s universes. * [PLAT-6195] Incremental Backup is now available as a Beta feature * [PLAT-6256] AmazonLinux2 based machines support in YBA. -* [PLAT-6910] [PLAT-6921] [PLAT-6992] PITR is now available for software vesion 2.14 and up +* [PLAT-6910] [PLAT-6921] [PLAT-6992] PITR is now available for software version 2.14 and up ### Improvements @@ -48,7 +48,7 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-470] [k8s] Simplified k8s provider config for deploying into the same cluster as YW (#7394) * [PLAT-2677] [PLAT-4687] Add GFlags metadata dynamically from the DB package into the platform for every kind of releases * [PLAT-3595] Added a shell command to run kubectl diff before an upgrade and send output to log. -* [PLAT-4465] Add cert-manager support for generating tls cert for Yugaware UI +* [PLAT-4465] Add cert-manager support for generating TLS cert for Yugaware UI * [PLAT-4563] Add Tablet splitting docDB metrics * [PLAT-4591] [PLAT-4593] [K8s] [UI] Allow specifying universe level helm overrides in UI * [PLAT-4942] [PLAT-6501] Improve bundle load time of JS code and move from webpack to esbuild @@ -106,7 +106,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6211] Add alerts for un-protected private ssh keys * [PLAT-6219] Condense and reduce the total number of metrics * [PLAT-6239] Upgrade the postgres version to 14.4 -* [PLAT-6239] Upgrade the postgres version to 14.4 * [PLAT-6242] [PLAT-6243] Implement gRPC equivalent of SSH RemoteShell to communicate with node agent + ansible over gRPC * [PLAT-6254] Install and upgrade improvements: 2-directory install location, upgrade to an alt-install directory, revamped preflight checks for upgrades * [PLAT-6256] AmazonLinux2 based machines support in YBA. @@ -141,7 +140,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-3763] Set kubeconfig file permissions to 600 when writing the file * [PLAT-3812] Verify that upgrade tasks don't need all the intent from ui * [PLAT-3915] [PLAT-6647] Read replica cluster should inherit YSQL enabled property from primary cluster -* [PLAT-3915] Read replica cluster should inherit ysql enabled property from primary cluster * [PLAT-3926] Generate unique self-signed cert for nginx on first run * [PLAT-4006] Get progress in terms of tablet numbers during data migration. * [PLAT-4007] Do not wait for LB for added nodes @@ -288,7 +286,7 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6065] Always Show YSQL and YCQL filter option in live queries tab * [PLAT-6070] Fix multi provider instance type caching * [PLAT-6071] [Master] Universe Task page throws JS console error and page is blank -* [PLAT-6074] Decrypt against all key versions for a key in azure KMS +* [PLAT-6074] Decrypt against all key versions for a key in Azure Key Vault * [PLAT-6076] Increase paramiko ssh-client banner_timeout * [PLAT-6078] Make dropdown scrollable in case of top K metrics * [PLAT-6080] [Backup] [UI] Not able to select backups checkbox @@ -314,7 +312,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6154] Avoid deletion of expired backups with missing storage config * [PLAT-6160] Fix the VM Image upgrade issue with YBC. * [PLAT-6168] Fix intermittent failures on node reboot -* [PLAT-6168] Fix intermittent failures on node reboot * [PLAT-6169] switching Azure default image back to CentOS 7 * [PLAT-6171] Move expired backup deletion logic to the backup garbage collector * [PLAT-6181] Make dropdown scrollable in Runtime Config UI for Universe and Providers @@ -328,8 +325,7 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6205] Prevent commenting of trust based auths for postgres connection * [PLAT-6210] Check all service status after install * [PLAT-6218] Change naming of auth enforced in reference.conf and make default value as false -* [PLAT-6221] YSQL or YCQL auth enforce config should be checked in backend for Create Universe -* [PLAT-6227] Handle the scenario of the universe paused when making the PITR API calls +* [PLAT-6221] YSQL or YCQL auth enforce config should be checked in backend for Create Universe * [PLAT-6227] Handle the scenario of the universe paused when making the PITR API calls * [PLAT-6231] [K8s] update log_cleanup.sh to recover from full disk * [PLAT-6233] Fix for vol resize failure during shrink @@ -337,7 +333,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6237] Fix the universe API calls to send enableYbc as true only during create universe and software upgrade * [PLAT-6238] [UI] Add k8s info component in support bundle * [PLAT-6239] Adding Update Package task to enum -* [PLAT-6239] Adding Update Package task to enum * [PLAT-6241] Adding log file for YBA in yba-installer * [PLAT-6246] [xCluster] Delete xCluster task should not change status if it is in DeletedUniverse status * [PLAT-6253] Restrict kubernetes overrides validation to only top level keys. @@ -365,7 +360,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6344] Customers can bring in linux users. Also confirm before we create a user * [Plat-6344] User check didn't get rebased correctly * [PLAT-6351] Allowing configuration of yba-installer Install Root -* [PLAT-6354] Remove LSM graphs that are no longer in metrics.yml * [PLAT-6354] Remove LSM graphs that are no longer in metrics.yml * [PLAT-6357] Renewable Vault token not renewed by YB * [PLAT-6358] Allow defining prometheus remote write config in helm chart @@ -373,29 +367,23 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6363] API annotation to vmImages map * [PLAT-6367] Fix SdkClientException for no region found in default region chain AWS * [PLAT-6375] Fix protection level in auth config if using existing crypto key -* [PLAT-6377] Seting instance type in node details of universe +* [PLAT-6377] Setting instance type in node details of universe * [PLAT-6379] refactor cmd to follow cobra standards * [PLAT-6381] Remove access_key_code from api_examples of kubernetes API universe create call. * [PLAT-6396] Show error message from request in toast * [PLAT-6397] Move installThirdPartyPackageTask as part of backup/restore task group -* [PLAT-6397] Move installThirdPartyPackageTask as part of backup/restore task group * [PLAT-6405] [PLAT-6437] Declare and use mutable keys with metadata in Java code * [PLAT-6424] Fixing yb_platform_backup.sh to work in docker environments * [PLAT-6438] [PLAT-6383] Add scope strictness and data validation when setting Runtime keys * [PLAT-6441] YBA-Installer Refactor * [PLAT-6443] S3 backup failing with s3cmd 2.3.0 version when access_token specified -* [PLAT-6443] S3 backup failing with s3cmd 2.3.0 version when access_token specified * [PLAT-6444] Configure Throttle Parameters throws 404 and portal UI becomes blank when backup is in progress -* [PLAT-6444] Configure Throttle Parameters throws 404 and portal UI becomes blank when backup is in progress -* [PLAT-6452] Remove old key from Master Advanced metric master_lsm_rocksdb_num_seek_or_next * [PLAT-6452] Remove old key from Master Advanced metric master_lsm_rocksdb_num_seek_or_next * [PLAT-6456] Update generate_kubeconfig.py to work on OpenShift 4.11 * [PLAT-6458] Couldn't set backup interval to 1 Hour, getting error as "Incremental backup interval must be less than full backup" -* [PLAT-6458] Couldn't set backup interval to 1 Hour, getting error as "Incremental backup interval must be less than full backup" * [PLAT-6459] Fix RemoteShell in devops python code to work with run_node_action.py * [PLAT-6460] Change the UI check during restore of YCQl tables backup to only check for non existence of the specific tables listed instead of forcing users to always restore to a non existent namespace. * [PLAT-6464] [Openshift] Support bundle creation failed with group id '1000720000' is too big -* [PLAT-6464] [Openshift] Support bundle creation failed with group id '1000720000' is too big * [PLAT-6467] [UI] Show protection level in GCP KMS UI * [PLAT-6475] [PLAT-6787] [PLAT-6795] Fix systemd Resume universe failures, Restart ybc on hot cert reload, and Do nothing in the case of nonRestartUpgrade * [PLAT-6476] Ensure we use the correct password for keystore (server.ks) @@ -407,7 +395,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6497] [Master] Support Bundle Download failed with Error while trying to download the node level component files * [PLAT-6499] Copy the ybc packages to /opt/yugabyte/ybc/release if MOVE_YBC_PKG=1 * [PLAT-6509] [Read Replica] Wrong number of nodes in add read replica -* [PLAT-6509] [Read Replica] Wrong number of nodes in add read replica * [PLAT-6517] Configure waiter for snapshot creation * [PLAT-6517] Increase max attempts for waiter during snapshot creation * [PLAT-6519] [yugaware] Collect only YBDB pod's metrics from K8s nodes @@ -416,11 +403,9 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6527] Background Scheduled backup and any other tasks should honor lameduck and retry * [PLAT-6531] [UI] Backup config css is overriding global Divider's css * [PLAT-6544] Aborting a YB-Controller task from UI not propagating to YBC -* [PLAT-6544] Aborting a YB-Controller task from UI not propagating to YBC * [PLAT-6568] Fix for Minio endpoint using HTTPS * [PLAT-6570] Add script to enable attaching/detaching of universe * [PLAT-6577] The Parallel Threads text box is not visible when restoring a non ybc backup onto a universe that has ybc installed on it. -* [PLAT-6577] The Parallel Threads text box is not visible when restoring a non ybc backup onto a universe that has ybc installed on it. * [PLAT-6578] Showing wrong cluster uuid in third party software upgrade * [PLAT-6580] Configure Throttle Parameters vaules are not saved * [PLAT-6581] Fix JS exception when user clicks pods tab when K8 universe is in error state @@ -449,7 +434,6 @@ docker pull yugabytedb/yugabyte:2.17.1.0-b439 * [PLAT-6745] Blank page on canceling edit on the PITR date/time picker * [PLAT-6754] Add warning to create universe API toggles * [PLAT-6763] Fix Response size graph -* [PLAT-6763] Fix Response size graph * [PLAT-6766] [xCluster] X-cluster replication is failing due to NPE * [PLAT-6771] [PITR] [UI] Wrap the large keyspace/Database name within PITR modal * [PLAT-6775] Backup of custom install root @@ -564,7 +548,7 @@ docker pull yugabytedb/yugabyte:2.17.0.0-b24 * [PLAT-6042] Fix kubernetes CPU usage graph * [PLAT-6053] [PLAT-6054] Enable toggling force delete for xCluster config * [PLAT-6069] Fix: Incremental Backup location same as Full Backup Location -* [PLAT-6074] Decrypt against all key versions for a key in azure KMS +* [PLAT-6074] Decrypt against all key versions for a key in Azure Key Vault * [PLAT-6075] Fix: Bring backup duration for non-incremental and full backups * [PLAT-6080] [Backup] [UI] Not able to select backups checkbox * [PLAT-6123] [Backup] Show parallel thread if current backup is not YBC enabled diff --git a/docs/content/preview/yugabyte-platform/security/create-kms-config/aws-kms.md b/docs/content/preview/yugabyte-platform/security/create-kms-config/aws-kms.md index c289bd2a0493..00dec98dad2d 100644 --- a/docs/content/preview/yugabyte-platform/security/create-kms-config/aws-kms.md +++ b/docs/content/preview/yugabyte-platform/security/create-kms-config/aws-kms.md @@ -29,8 +29,8 @@ type: docs
        • }}" class="nav-link"> -   -   Azure KMS + + Azure Key Vault
          • @@ -43,7 +43,7 @@ type: docs
        -
        Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Amazon Web Services (AWS) KMS](https://aws.amazon.com/kms/). +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Amazon Web Services (AWS) KMS](https://aws.amazon.com/kms/). The AWS user associated with a KMS configuration requires the following minimum Identity and Access Management (IAM) KMS-related permissions: @@ -59,9 +59,9 @@ You can create a KMS configuration that uses AWS KMS as follows: 1. Use the YugabyteDB Anywhere UI to navigate to **Configs > Security > Encryption At Rest** to access the list of existing configurations. -2. Click **Create New Config**. +1. Click **Create New Config**. -3. Enter the following configuration details in the form: +1. Enter the following configuration details in the form: - **Configuration Name** — Enter a meaningful name for your configuration. - **KMS Provider** — Select **AWS KMS**. @@ -72,37 +72,37 @@ You can create a KMS configuration that uses AWS KMS as follows: - **Customer Master Key ID** — Enter the identifier for the CMK. If an identifier is not entered, a CMK ID will be auto-generated. - **AWS KMS Endpoint** — Specify the KMS endpoint to ensure that the encryption traffic is routed across your internal links without crossing into an external network. -4. Optionally, click **Upload CMK Policy** to select a custom policy file. The following is the default policy: +1. Optionally, click **Upload CMK Policy** to select a custom policy file. The following is the default policy: ```json - { - "Version": "2012-10-17", - "Id": "key-default-1", - "Statement": [ - { - "Sid": "Enable IAM User Permissions", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::root" - }, - "Action": "kms:*", - "Resource": "*" - }, - { - "Sid": "Allow access for Key Administrators", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam:::[user|role]{1}/[|]{1}" - }, - "Action": "kms:*", - "Resource": "*" - } - ] - } + { + "Version": "2012-10-17", + "Id": "key-default-1", + "Statement": [ + { + "Sid": "Enable IAM User Permissions", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::root" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "Allow access for Key Administrators", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam:::[user|role]{1}/[|]{1}" + }, + "Action": "kms:*", + "Resource": "*" + } + ] + } ``` -5. Click **Save**.
        +1. Click **Save**. Your new configuration should appear in the list of configurations. A saved KMS configuration can only be deleted if it is not in use by any existing universes. -6. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. +1. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. diff --git a/docs/content/preview/yugabyte-platform/security/create-kms-config/azure-kms.md b/docs/content/preview/yugabyte-platform/security/create-kms-config/azure-kms.md index b35f0434e25b..d0a056949b99 100644 --- a/docs/content/preview/yugabyte-platform/security/create-kms-config/azure-kms.md +++ b/docs/content/preview/yugabyte-platform/security/create-kms-config/azure-kms.md @@ -2,9 +2,7 @@ title: Create a KMS configuration using Azure headerTitle: Create a KMS configuration using Azure linkTitle: Create a KMS configuration -description: Use YugabyteDB Anywhere to create a KMS configuration for Azure KMS. -aliases: - - /preview/yugabyte-platform/security/create-kms-config +description: Use YugabyteDB Anywhere to create a KMS configuration for Azure Key Vault. menu: preview_yugabyte-platform: parent: security @@ -29,7 +27,7 @@ type: docs
      • }}" class="nav-link active"> -   Azure KMS + Azure Key Vault
      • @@ -39,9 +37,10 @@ type: docs
      -
      Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of Microsoft Azure KMS. -Conceptually, Azure KMS consists of a key vault containing one or more keys, with each key capable of having multiple versions. +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of Microsoft [Azure Key Vault](https://learn.microsoft.com/en-us/azure/key-vault/). + +Conceptually, Azure Key Vault consists of a key vault containing one or more keys, with each key capable of having multiple versions. Before defining a KMS configuration with YugabyteDB Anywhere, you need to create a key vault through the [Azure portal](https://docs.microsoft.com/en-us/azure/key-vault/general/quick-create-portal). The following settings are required: @@ -56,7 +55,7 @@ If you are planning to use an existing cryptographic key with the same name, it - Permitted operations should have at least WRAP_KEY and UNWRAP_KEY. - The key rotation policy should not be defined in order to avoid automatic rotation. -You can create a KMS configuration that uses Azure KMS, as follows: +You can create a KMS configuration that uses Azure Key Vault, as follows: 1. Use the YugabyteDB Anywhere UI to navigate to **Configs > Security > Encryption At Rest** to access the list of existing configurations. @@ -74,13 +73,12 @@ You can create a KMS configuration that uses Azure KMS, as follows: - **Key Algorithm** — The algorithm for the master key. Currently, only the RSA algorithm is supported. - **Key Size** — Select the size of the master key, in bits. Supported values are 2048 (default), 3072, and 4096. - ![img](/images/yp/security/azurekms-config.png) + ![Azure Key Vault configuration](/images/yp/security/azurekms-config.png) -1. Click **Save**.
      +1. Click **Save**. Your new configuration should appear in the list of configurations. A saved KMS configuration can only be deleted if it is not in use by any existing universes. 1. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. - -Note that YugabyteDB Anywhere does not manage the key vault and deleting the KMS configuration does not delete the key vault, master key, or key versions on Azure KMS. +Note that YugabyteDB Anywhere does not manage the key vault and deleting the KMS configuration does not delete the key vault, master key, or key versions on Azure Key Vault. diff --git a/docs/content/preview/yugabyte-platform/security/create-kms-config/google-kms.md b/docs/content/preview/yugabyte-platform/security/create-kms-config/google-kms.md index 71de99204f30..a57211ee0e8f 100644 --- a/docs/content/preview/yugabyte-platform/security/create-kms-config/google-kms.md +++ b/docs/content/preview/yugabyte-platform/security/create-kms-config/google-kms.md @@ -3,8 +3,6 @@ title: Create a KMS configuration using Google Cloud headerTitle: Create a KMS configuration using Google Cloud linkTitle: Create a KMS configuration description: Use YugabyteDB Anywhere to create a KMS configuration for Google Cloud KMS. -aliases: - - /preview/yugabyte-platform/security/create-kms-config menu: preview_yugabyte-platform: parent: security @@ -30,7 +28,7 @@ type: docs
    • }}" class="nav-link"> -     Azure KMS + Azure Key Vault
      • @@ -43,7 +41,7 @@ type: docs
    -
    Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Google Cloud KMS](https://cloud.google.com/security-key-management). +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Google Cloud KMS](https://cloud.google.com/security-key-management). Conceptually, Google Cloud KMS consists of a key ring containing one or more cryptographic keys, with each key capable of having multiple versions. @@ -69,7 +67,7 @@ You can create a KMS configuration that uses Google Cloud KMS, as follows: 1. Click **Create New Config**. -3. Enter the following configuration details in the form: +1. Enter the following configuration details in the form: - **Configuration Name** — Enter a meaningful name for your configuration. - **KMS Provider** — Select **GCP KMS**. @@ -82,12 +80,10 @@ You can create a KMS configuration that uses Google Cloud KMS, as follows: ![Google KMS](/images/yp/security/googlekms-config.png) -3. Click **Save**.
    +1. Click **Save**. Your new configuration should appear in the list of configurations. A saved KMS configuration can only be deleted if it is not in use by any existing universes. -4. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. - - +1. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. Note that YugabyteDB Anywhere does not manage the key ring and deleting the KMS configuration does not destroy the key ring, cryptographic key, or its versions on Google Cloud KMS. diff --git a/docs/content/preview/yugabyte-platform/security/create-kms-config/hashicorp-kms.md b/docs/content/preview/yugabyte-platform/security/create-kms-config/hashicorp-kms.md index fb9867ab49dc..0d24378bf378 100644 --- a/docs/content/preview/yugabyte-platform/security/create-kms-config/hashicorp-kms.md +++ b/docs/content/preview/yugabyte-platform/security/create-kms-config/hashicorp-kms.md @@ -3,8 +3,6 @@ title: Create a KMS configuration using HashiCorp Vault headerTitle: Create a KMS configuration using HashiCorp Vault linkTitle: Create a KMS configuration description: Use YugabyteDB Anywhere to create a KMS configuration for HashiCorp Vault. -aliases: - - /preview/yugabyte-platform/security/create-kms-config menu: preview_yugabyte-platform: parent: security @@ -29,7 +27,7 @@ type: docs
  • }}" class="nav-link"> -   Azure KMS + Azure Key Vault
  • @@ -39,7 +37,8 @@ type: docs
-
Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [HashiCorp Vault](https://www.vaultproject.io/) as a KMS. + +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [HashiCorp Vault](https://www.vaultproject.io/) as a KMS. ## Configure HashiCorp Vault @@ -51,78 +50,78 @@ Before you can start configuring HashiCorp Vault, install it on a virtual machin You need to configure HashiCorp Vault in order to use it with YugabyteDB Anywhere, as follows: -- Create a vault configuration file that references your nodes and specifies the address, as follows: +1. Create a vault configuration file that references your nodes and specifies the address, as follows: - ```properties - storage "raft" { - path = "./vault/data/" - node_id = "node1" - } + ```properties + storage "raft" { + path = "./vault/data/" + node_id = "node1" + } - listener "tcp" { - address = "127.0.0.1:8200" - tls_disable = "true" - } + listener "tcp" { + address = "127.0.0.1:8200" + tls_disable = "true" + } - api_addr = "http://127.0.0.1:8200" - cluster_addr = "https://127.0.0.1:8201" - ui = true - disable_mlock = true - default_lease_ttl = "768h" - max_lease_ttl = "8760h" - ``` + api_addr = "http://127.0.0.1:8200" + cluster_addr = "https://127.0.0.1:8201" + ui = true + disable_mlock = true + default_lease_ttl = "768h" + max_lease_ttl = "8760h" + ``` - Replace `127.0.0.1` with the vault web address. + Replace `127.0.0.1` with the vault web address. - For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). + For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). -- Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). +1. Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). -- Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). +1. Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). -- Enable the secret engine by executing the following command: +1. Enable the secret engine by executing the following command: - ```shell - vault secrets enable transit - ``` + ```shell + vault secrets enable transit + ``` - For more information, see [Transit Secrets Engine](https://www.vaultproject.io/docs/secrets/transit) and [Setup](https://www.vaultproject.io/docs/secrets/transit#setup). + For more information, see [Transit Secrets Engine](https://www.vaultproject.io/docs/secrets/transit) and [Setup](https://www.vaultproject.io/docs/secrets/transit#setup). -- Create the vault policy, as per the following sample: +1. Create the vault policy, as per the following sample: - ```properties - path "transit/*" { - capabilities = ["create", "read", "update", "delete", "list"] - } + ```properties + path "transit/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } - path "auth/token/lookup-self" { - capabilities = ["read"] - } + path "auth/token/lookup-self" { + capabilities = ["read"] + } - path "sys/capabilities-self" { - capabilities = ["read", "update"] - } + path "sys/capabilities-self" { + capabilities = ["read", "update"] + } - path "auth/token/renew-self" { - capabilities = ["update"] - } + path "auth/token/renew-self" { + capabilities = ["update"] + } - path "sys/*" { - capabilities = ["read"] - } - ``` + path "sys/*" { + capabilities = ["read"] + } + ``` -- Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: +1. Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: - ```shell - vault token create -no-default-policy -policy=trx - ``` + ```shell + vault token create -no-default-policy -policy=trx + ``` - You may also specify the following for your token: + You may also specify the following for your token: - - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. + - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. - - `period` — If specified, the token can be infinitely renewed. + - `period` — If specified, the token can be infinitely renewed. ## Create a KMS configuration @@ -139,7 +138,7 @@ You can create a new KMS configuration that uses HashiCorp Vault as follows: - **Vault Address** — Enter the web address of your vault. For example, `http://127.0.0.1:8200` - **Secret Token** — Enter the token you obtained from the vault. - **Secret Engine** — This is a read-only field with its value set to `transit`. It identifies the secret engine. - - **Mount Path** — Specify the path to the secret engine within the vault. The default value is `transit/`.
+ - **Mount Path** — Specify the path to the secret engine in the vault. The default value is `transit/`. ![Create config](/images/yp/security/hashicorp-config.png) diff --git a/docs/content/stable/yugabyte-platform/security/create-kms-config/aws-kms.md b/docs/content/stable/yugabyte-platform/security/create-kms-config/aws-kms.md index 96e3ecd4614c..48ca3e28f42e 100644 --- a/docs/content/stable/yugabyte-platform/security/create-kms-config/aws-kms.md +++ b/docs/content/stable/yugabyte-platform/security/create-kms-config/aws-kms.md @@ -28,7 +28,7 @@ type: docs
  • }}" class="nav-link">   -   Azure KMS + Azure Key Vault
    • @@ -41,7 +41,7 @@ type: docs -
    Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Amazon Web Services (AWS) KMS](https://aws.amazon.com/kms/). +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Amazon Web Services (AWS) KMS](https://aws.amazon.com/kms/). The AWS user associated with a KMS configuration requires the following minimum Identity and Access Management (IAM) KMS-related permissions: @@ -99,7 +99,7 @@ You can create a KMS configuration that uses AWS KMS as follows: } ``` -5. Click **Save**.
    +5. Click **Save**. Your new configuration should appear in the list of configurations. A saved KMS configuration can only be deleted if it is not in use by any existing universes. diff --git a/docs/content/stable/yugabyte-platform/security/create-kms-config/azure-kms.md b/docs/content/stable/yugabyte-platform/security/create-kms-config/azure-kms.md index e3eebb3eb347..a744e284a9b6 100644 --- a/docs/content/stable/yugabyte-platform/security/create-kms-config/azure-kms.md +++ b/docs/content/stable/yugabyte-platform/security/create-kms-config/azure-kms.md @@ -2,7 +2,7 @@ title: Create a KMS configuration using Azure headerTitle: Create a KMS configuration using Azure linkTitle: Create a KMS configuration -description: Use YugabyteDB Anywhere to create a KMS configuration for Azure KMS. +description: Use YugabyteDB Anywhere to create a KMS configuration for Azure Key Vault. menu: stable_yugabyte-platform: parent: security @@ -27,7 +27,7 @@ type: docs
  • }}" class="nav-link active"> -   Azure KMS + Azure Key Vault
  • @@ -37,9 +37,10 @@ type: docs
    • -
    Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of Microsoft Azure KMS. -Conceptually, Azure KMS consists of a key vault containing one or more keys, with each key capable of having multiple versions. +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of Microsoft Azure Key Vault. + +Conceptually, Azure Key Vault consists of a key vault containing one or more keys, with each key capable of having multiple versions. Before defining a KMS configuration with YugabyteDB Anywhere, you need to create a key vault through the [Azure portal](https://docs.microsoft.com/en-us/azure/key-vault/general/quick-create-portal). The following settings are required: @@ -54,7 +55,7 @@ If you are planning to use an existing cryptographic key with the same name, it - Permitted operations should have at least WRAP_KEY and UNWRAP_KEY. - The key rotation policy should not be defined in order to avoid automatic rotation. -You can create a KMS configuration that uses Azure KMS, as follows: +You can create a KMS configuration that uses Azure Key Vault, as follows: 1. Use the YugabyteDB Anywhere UI to navigate to **Configs > Security > Encryption At Rest** to access the list of existing configurations. @@ -72,13 +73,12 @@ You can create a KMS configuration that uses Azure KMS, as follows: - **Key Algorithm** — The algorithm for the master key. Currently, only the RSA algorithm is supported. - **Key Size** — Select the size of the master key, in bits. Supported values are 2048 (default), 3072, and 4096. - ![img](/images/yp/security/azurekms-config.png) + ![Azure Key Vault configuration](/images/yp/security/azurekms-config.png) -1. Click **Save**.
    +1. Click **Save**. Your new configuration should appear in the list of configurations. A saved KMS configuration can only be deleted if it is not in use by any existing universes. 1. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. - -Note that YugabyteDB Anywhere does not manage the key vault and deleting the KMS configuration does not delete the key vault, master key, or key versions on Azure KMS. +Note that YugabyteDB Anywhere does not manage the key vault and deleting the KMS configuration does not delete the key vault, master key, or key versions on Azure Key Vault. diff --git a/docs/content/stable/yugabyte-platform/security/create-kms-config/google-kms.md b/docs/content/stable/yugabyte-platform/security/create-kms-config/google-kms.md index 9d6af812d155..47504725fcec 100644 --- a/docs/content/stable/yugabyte-platform/security/create-kms-config/google-kms.md +++ b/docs/content/stable/yugabyte-platform/security/create-kms-config/google-kms.md @@ -28,7 +28,7 @@ type: docs
  • }}" class="nav-link"> -     Azure KMS + Azure Key Vault
    • @@ -41,7 +41,7 @@ type: docs -
    Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Google Cloud KMS](https://cloud.google.com/security-key-management). +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [Google Cloud KMS](https://cloud.google.com/security-key-management). Conceptually, Google Cloud KMS consists of a key ring containing one or more cryptographic keys, with each key capable of having multiple versions. @@ -67,7 +67,7 @@ You can create a KMS configuration that uses Google Cloud KMS, as follows: 1. Click **Create New Config**. -3. Enter the following configuration details in the form: +1. Enter the following configuration details in the form: - **Configuration Name** — Enter a meaningful name for your configuration. - **KMS Provider** — Select **GCP KMS**. @@ -80,12 +80,10 @@ You can create a KMS configuration that uses Google Cloud KMS, as follows: ![Google KMS](/images/yp/security/googlekms-config.png) -3. Click **Save**.
    +1. Click **Save**. Your new configuration should appear in the list of configurations. A saved KMS configuration can only be deleted if it is not in use by any existing universes. -4. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. - - +1. Optionally, to confirm that the information is correct, click **Show details**. Note that sensitive configuration values are displayed partially masked. Note that YugabyteDB Anywhere does not manage the key ring and deleting the KMS configuration does not destroy the key ring, cryptographic key, or its versions on Google Cloud KMS. diff --git a/docs/content/stable/yugabyte-platform/security/create-kms-config/hashicorp-kms.md b/docs/content/stable/yugabyte-platform/security/create-kms-config/hashicorp-kms.md index b73fbd1613f5..2c6fdff545ff 100644 --- a/docs/content/stable/yugabyte-platform/security/create-kms-config/hashicorp-kms.md +++ b/docs/content/stable/yugabyte-platform/security/create-kms-config/hashicorp-kms.md @@ -27,7 +27,7 @@ type: docs
  • }}" class="nav-link"> -   Azure KMS + Azure Key Vault
  • @@ -37,7 +37,8 @@ type: docs
    • -
    Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [HashiCorp Vault](https://www.vaultproject.io/) as a KMS. + +Encryption at rest uses universe keys to encrypt and decrypt universe data keys. You can use the YugabyteDB Anywhere UI to create key management service (KMS) configurations for generating the required universe keys for one or more YugabyteDB universes. Encryption at rest in YugabyteDB Anywhere supports the use of [HashiCorp Vault](https://www.vaultproject.io/) as a KMS. ## Configure HashiCorp Vault @@ -49,78 +50,78 @@ Before you can start configuring HashiCorp Vault, install it on a virtual machin You need to configure HashiCorp Vault in order to use it with YugabyteDB Anywhere, as follows: -- Create a vault configuration file that references your nodes and specifies the address, as follows: +1. Create a vault configuration file that references your nodes and specifies the address, as follows: - ```properties - storage "raft" { - path = "./vault/data/" - node_id = "node1" - } + ```properties + storage "raft" { + path = "./vault/data/" + node_id = "node1" + } - listener "tcp" { - address = "127.0.0.1:8200" - tls_disable = "true" - } + listener "tcp" { + address = "127.0.0.1:8200" + tls_disable = "true" + } - api_addr = "http://127.0.0.1:8200" - cluster_addr = "https://127.0.0.1:8201" - ui = true - disable_mlock = true - default_lease_ttl = "768h" - max_lease_ttl = "8760h" - ``` + api_addr = "http://127.0.0.1:8200" + cluster_addr = "https://127.0.0.1:8201" + ui = true + disable_mlock = true + default_lease_ttl = "768h" + max_lease_ttl = "8760h" + ``` - Replace `127.0.0.1` with the vault web address. + Replace `127.0.0.1` with the vault web address. - For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). + For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). -- Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). +1. Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). -- Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). +1. Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). -- Enable the secret engine by executing the following command: +1. Enable the secret engine by executing the following command: - ```shell - vault secrets enable transit - ``` + ```shell + vault secrets enable transit + ``` - For more information, see [Transit Secrets Engine](https://www.vaultproject.io/docs/secrets/transit) and [Setup](https://www.vaultproject.io/docs/secrets/transit#setup). + For more information, see [Transit Secrets Engine](https://www.vaultproject.io/docs/secrets/transit) and [Setup](https://www.vaultproject.io/docs/secrets/transit#setup). -- Create the vault policy, as per the following sample: +1. Create the vault policy, as per the following sample: - ```properties - path "transit/*" { - capabilities = ["create", "read", "update", "delete", "list"] - } + ```properties + path "transit/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } - path "auth/token/lookup-self" { - capabilities = ["read"] - } + path "auth/token/lookup-self" { + capabilities = ["read"] + } - path "sys/capabilities-self" { - capabilities = ["read", "update"] - } + path "sys/capabilities-self" { + capabilities = ["read", "update"] + } - path "auth/token/renew-self" { - capabilities = ["update"] - } + path "auth/token/renew-self" { + capabilities = ["update"] + } - path "sys/*" { - capabilities = ["read"] - } - ``` + path "sys/*" { + capabilities = ["read"] + } + ``` -- Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: +1. Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: - ```shell - vault token create -no-default-policy -policy=trx - ``` + ```shell + vault token create -no-default-policy -policy=trx + ``` - You may also specify the following for your token: + You may also specify the following for your token: - - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. + - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. - - `period` — If specified, the token can be infinitely renewed. + - `period` — If specified, the token can be infinitely renewed. ## Create a KMS configuration @@ -137,9 +138,9 @@ You can create a new KMS configuration that uses HashiCorp Vault as follows: - **Vault Address** — Enter the web address of your vault. For example, `http://127.0.0.1:8200` - **Secret Token** — Enter the token you obtained from the vault. - **Secret Engine** — This is a read-only field with its value set to `transit`. It identifies the secret engine. - - **Mount Path** — Specify the path to the secret engine within the vault. The default value is `transit/`.
    + - **Mount Path** — Specify the path to the secret engine in the vault. The default value is `transit/`. - ![Create config](/images/yp/security/hashicorp-config.png) + ![Hashicorp KMS configuration](/images/yp/security/hashicorp-config.png) 1. Click **Save**. Your new configuration should appear in the list of configurations. From 340b5fc188b2fd2a8380eee2a61958c861767eb1 Mon Sep 17 00:00:00 2001 From: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> Date: Thu, 2 Mar 2023 14:19:17 -0500 Subject: [PATCH 44/81] [Docs] Port login profiles to stable and preview (#16259) --- .../reference/configuration/yb-tserver.md | 4 +- .../secure/enable-authentication/_index.md | 2 - .../ysql-login-profiles.md | 17 +- .../reference/configuration/yb-tserver.md | 8 + .../secure/enable-authentication/_index.md | 12 + .../ysql-login-profiles.md | 215 ++++++++++++++++++ 6 files changed, 242 insertions(+), 16 deletions(-) create mode 100644 docs/content/stable/secure/enable-authentication/ysql-login-profiles.md diff --git a/docs/content/preview/reference/configuration/yb-tserver.md b/docs/content/preview/reference/configuration/yb-tserver.md index dd15087f4f26..4bdf90ce26e8 100644 --- a/docs/content/preview/reference/configuration/yb-tserver.md +++ b/docs/content/preview/reference/configuration/yb-tserver.md @@ -438,13 +438,13 @@ When YSQL authentication is enabled, you can sign into `ysqlsh` using the defaul Default: `false` - +Default: `false` ##### --pgsql_proxy_bind_address diff --git a/docs/content/preview/secure/enable-authentication/_index.md b/docs/content/preview/secure/enable-authentication/_index.md index 35afd55eaa61..1c466f963d68 100644 --- a/docs/content/preview/secure/enable-authentication/_index.md +++ b/docs/content/preview/secure/enable-authentication/_index.md @@ -34,7 +34,6 @@ In YSQL, further fine-grained control over client authentication is provided by
    -
    diff --git a/docs/content/preview/secure/enable-authentication/ysql-login-profiles.md b/docs/content/preview/secure/enable-authentication/ysql-login-profiles.md index e36ea20bddfd..73f0c3b69e22 100644 --- a/docs/content/preview/secure/enable-authentication/ysql-login-profiles.md +++ b/docs/content/preview/secure/enable-authentication/ysql-login-profiles.md @@ -1,4 +1,4 @@ - -{{< note title="Note" >}} - -This feature is currently only available in [v2.14.6](../../../../v2.14/secure/enable-authentication/ysql-login-profiles/). +--- -{{< /note >}} - - ## View profiles @@ -202,7 +196,7 @@ yugabyte=# \dgP ``` --> - diff --git a/docs/content/stable/reference/configuration/yb-tserver.md b/docs/content/stable/reference/configuration/yb-tserver.md index 3f23db384241..f01bc1364092 100644 --- a/docs/content/stable/reference/configuration/yb-tserver.md +++ b/docs/content/stable/reference/configuration/yb-tserver.md @@ -461,6 +461,14 @@ When YSQL authentication is enabled, you can sign into `ysqlsh` using the defaul Default: `false` +##### --ysql_enable_profile + +Enables YSQL [login profiles](../../../secure/enable-authentication/ysql-login-profiles/). + +When YSQL login profiles are enabled, you can set limits on the number of failed login attempts made by users. + +Default: `false` + ##### --pgsql_proxy_bind_address Specifies the TCP/IP bind addresses for the YSQL API. The default value of `0.0.0.0:5433` allows listening for all IPv4 addresses access to localhost on port `5433`. The `--pgsql_proxy_bind_address` value overwrites `listen_addresses` (default value of `127.0.0.1:5433`) that controls which interfaces accept connection attempts. diff --git a/docs/content/stable/secure/enable-authentication/_index.md b/docs/content/stable/secure/enable-authentication/_index.md index 30665651ccf7..39d4666db9aa 100644 --- a/docs/content/stable/secure/enable-authentication/_index.md +++ b/docs/content/stable/secure/enable-authentication/_index.md @@ -35,6 +35,18 @@ In YSQL, further fine-grained control over client authentication is provided by
    +
    + +
    + +
    Create login profiles
    +
    +
    + Prevent brute force exploits by enabling login profiles in YSQL. +
    +     +
    +
    diff --git a/docs/content/stable/secure/enable-authentication/ysql-login-profiles.md b/docs/content/stable/secure/enable-authentication/ysql-login-profiles.md new file mode 100644 index 000000000000..d344454d475b --- /dev/null +++ b/docs/content/stable/secure/enable-authentication/ysql-login-profiles.md @@ -0,0 +1,215 @@ +--- +title: Create login profiles +headerTitle: Create and configure login profiles in YSQL +linkTitle: Create login profiles +description: Create and configure login profiles in YugabyteDB +headcontent: Prevent brute force cracking with login profiles +menu: + stable: + identifier: ysql-login-profiles + parent: enable-authentication + weight: 725 +type: docs +--- + +To enhance the security of your database, you can enable login profiles to lock accounts after a specified number of login attempts. This prevents brute force exploits. + +When enabled, database administrators with superuser (or in YugabyteDB Managed, `yb_db_admin`) privileges can create login profiles and assign roles to the profiles. + +There is no default profile for roles; you must explicitly assign all roles with login privileges to the profile if you want the policy to apply to all users. Users not associated with a profile continue to have unlimited login attempts. + +When creating a profile, you must specify the number of failed attempts that are allowed before the account with the profile is locked. + +The number of failed attempts increments by one every time authentication fails during login. If the number of failed attempts is equal to the preset limit, then the account is locked. For example, if the limit is 3, a user is locked out on the third failed attempt. + +If authentication is successful, or if an administrator unlocks a locked account, the number of failed attempts resets to 0. + +## Enable login profiles + +### Start local clusters + +To enable login profiles in your local YugabyteDB clusters, include the YB-TServer [--ysql_enable_profile](../../../reference/configuration/yb-tserver/#ysql-enable-profile) flag with the [yugabyted start](../../../reference/configuration/yugabyted/#start) command `--tserver_flags` flag, as follows: + +```sh +./bin/yugabyted start --tserver_flags="ysql_enable_profile=true" +``` + +### Start YB-TServer services + +To enable login profiles in deployable YugabyteDB clusters, you need to start your YB-TServer services using the `--ysql_enable_profile` flag. Your command should look similar to the following: + +```sh +./bin/yb-tserver \ + --tserver_master_addrs \ + --fs_data_dirs \ + --ysql_enable_auth=true \ + --ysql_enable_profile=true \ + >& /home/centos/disk1/yb-tserver.out & +``` + +You can also enable YSQL login profiles by adding the `--ysql_enable_profile=true` to the YB-TServer configuration file (`tserver.conf`). + +For more information, refer to [Start YB-TServers](../../../deploy/manual-deployment/start-tservers/). + +## Manage login profiles + +When profiles are enabled, you can manage login profiles using the following commands: + +- `CREATE PROFILE` +- `DROP PROFILE` +- `ALTER ROLE` + +Only superusers can create or drop profiles, and assign profiles to roles. + +### Create and drop profiles + +To create a profile, do the following: + +```sql +CREATE PROFILE myprofile LIMIT + FAILED_LOGIN_ATTEMPTS ; + [PASSWORD_LOCK_TIME ]; +``` + +Note that `PASSWORD_LOCK_TIME` is optional, and timed locking is not currently supported. + +You can drop a profile as follows: + +```sql +DROP PROFILE myprofile; +``` + +### Assign roles to profiles + +You can assign a role to a profile as follows: + +```sql +ALTER ROLE myuser PROFILE myprofile; +``` + +You can remove a role from a profile as follows: + +```sql +ALTER ROLE myuser NOPROFILE; +``` + +Note that you should remove the association between a role and its profile using `ALTER ROLE ... NOPROFILE` before dropping a role. + +### Lock and unlock roles + +You can unlock a role that has been locked out as follows: + +```sql +ALTER ROLE myuser ACCOUNT UNLOCK; +``` + +You can lock a role so that it can't log in as follows: + +```sql +ALTER ROLE myuser ACCOUNT LOCK; +``` + +### Recover from complete lockout + +If you lock out all roles including administrator roles, you must restart the cluster with the `--ysql_enable_profile` flag disabled. + +While disabling login profiles allows users back in, you won't be able to change any profile information, as profile commands can't be run when the profile flag is disabled. + +To re-enable accounts, do the following: + +1. Restart the cluster without profiles enabled. +1. Create a new superuser. +1. Restart the cluster with profiles enabled. +1. Connect as the new superuser and issue the profile commands to unlock the accounts. + + + +## View profiles + +The `pg_yb_profile` table lists profiles and their attributes. + +To view profiles, execute the following statement: + +```sql +SELECT * FROM pg_yb_profile; +``` + +You should see output similar to the following: + +```output + prfname | prfmaxfailedloginattempts | prfpasswordlocktime +-----------+---------------------------+--------------------- + myprofile | 3 | 0 +(1 row) +``` + +The following table describes the columns and their values: + +| COLUMN | TYPE | DESCRIPTION | +| :---- | :--- | :---------- | +| `prfname` | name | Name of the profile. Must be unique. | +| `prfmaxfailedloginattempts` | int | Maximum number of failed attempts allowed. | +| `prfpasswordlocktime` | int | Interval in seconds to lock the account. NULL implies that the role will be locked indefinitely. | + +### View role profile information + +The `pg_yb_role_profile` table lists role profiles and their attributes. + +To view profiles, execute the following statement: + +```sql +SELECT * FROM pg_yb_role_profile; +``` + +You should see output similar to the following: + +```output + rolprfrole | rolprfprofile | rolprfstatus | rolprffailedloginattempts | rolprflockeduntil +------------+---------------+--------------+---------------------------+------------------- + 13287 | 16384 | o | 0 | +(1 row) +``` + +The following table describes the columns and their values: + +| COLUMN | TYPE | DEFAULT | DESCRIPTION | +| :----- | :--- | :------ | :---------- | +| `rolprfrole` | OID | | OID of the row in PG_ROLE +| `rolprfprofile` | OID | | OID of the row in PROFILE +| `rolprfstatus` | char | o | The status of the account, as follows:
    • `o` (OPEN); allowed to login.
    • `t` (LOCKED(TIMED)); locked for a duration of the timestamp stored in `rolprflockeduntil`. (Note that timed locking is not supported.)
    • `l` (LOCKED); locked indefinitely and can only be unlocked by the admin.
    +| `rolprffailedloginattempts` | int | 0 | Number of failed attempts by this role. +| `rolprflockeduntil` | timestamptz | Null | If `rolprfstatus` is `t`, the duration that the role is locked. Otherwise, the value is NULL and not used. + + + +## Limitations and caveats + +- A profile can't be modified using `ALTER PROFILE`. If a profile needs to be modified, create a new profile. +- Currently a role is locked indefinitely unless an administrator unlocks the role. +- Login profiles are only applicable to challenge-response authentication methods. YugabyteDB also supports authentication methods that are not challenge-response, and login profiles are ignored for these methods as the authentication outcome has already been determined. The authentication methods are as follows: + - Reject + - ImplicitReject + - Trust + - YbTserverKey + - Peer + + For more information on these authentication methods, refer to [Client Authentication](https://www.postgresql.org/docs/11/client-authentication.html) in the PostgreSQL documentation. + +- If the cluster SSL mode is `allow` or `prefer`, a single user login attempt can trigger two failed login attempts. For more information on SSL modes in PostgreSQL, refer to [SSL Support](https://www.postgresql.org/docs/11/libpq-ssl.html) in the PostgreSQL documentation. +- The `\h` and `\dg` meta commands do not currently provide information about PROFILE and ROLE PROFILE catalog objects. + +Enhancements to login profiles are tracked in GitHub issue [#15676](https://github.com/yugabyte/yugabyte-db/issues/15676). From 0fe714bf67b2dcc5efe5ba1e6124bbf71ed7920e Mon Sep 17 00:00:00 2001 From: Aishwarya Chakravarthy Date: Thu, 2 Mar 2023 16:19:34 -0500 Subject: [PATCH 45/81] [docs] [PLAT-7424] changed discovery URL description (#16271) * changed discovery URL desc * fixed image * reworded based on review --- .../administer-yugabyte-platform/oidc-authentication.md | 7 ++----- .../administer-yugabyte-platform/oidc-authentication.md | 7 ++----- .../administer-yugabyte-platform/oidc-authentication.md | 9 +++------ 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md index 8bbbf1b37810..c25382ef2671 100644 --- a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md +++ b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md @@ -7,12 +7,10 @@ menu: preview_yugabyte-platform: identifier: oidc-authentication parent: administer-yugabyte-platform - weight: 20 + weight: 10 type: docs --- - -
    diff --git a/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ycql.md b/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ycql.md index 716771c8ff45..e7aa705dca77 100644 --- a/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ycql.md +++ b/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ycql.md @@ -29,7 +29,11 @@ type: docs -
    You can use YugabyteDB Anywhere to back up your YugabyteDB universe YCQL data. +You can use YugabyteDB Anywhere to back up your YugabyteDB universe YCQL data. This includes actions such as deleting and restoring the backup, as well as restoring and copying the database location. + +If you are using YBA version 2.16 or later to manage universes with YugabyteDB version 2.16 or later, you can additionally create [incremental backups](../ysql/#create-incremental-backups) and [configure backup performance parameters](../ysql/#configure-backup-performance-parameters). + +For information on how to schedule backups for a later time or as a recurring task, see [Schedule universe YCQL data backups](../../schedule-data-backups/ycql/). Note that non-transactional backups are not supported. @@ -39,37 +43,37 @@ To view, [restore](../../restore-universe-data/ycql/), or delete existing backup By default, the list displays all the backups generated for the universe regardless of the time period. You can configure the list to only display the backups created during a specific time period, such as last year, last month, and so on. In addition, you can specify a custom time period. +## Create backups + The **Backups** page allows you to create new backups that start immediately, as follows: -- Click **Backup now** to open the dialog shown in the following illustration:
    +1. Click **Backup now** to open the dialog shown in the following illustration: - ![Backup](/images/yp/create-backup-new-3.png)
    + ![Backup](/images/yp/create-backup-new-3.png) -- In the **Backup Now** dialog, select YCQL as the API type. +1. In the **Backup Now** dialog, select YCQL as the API type. -- Complete the **Select the storage config you want to use for your backup** field whose list depends on your existing backup storage configurations. For more information, see [Configure backup storage](../../configure-backup-storage/). +1. Complete the **Select the storage config you want to use for your backup** field whose list depends on your existing backup storage configurations. For more information, see [Configure backup storage](../../configure-backup-storage/). -- Select the database to back up. +1. Select the database to back up. -- Specify whether you want to back up all tables in the keyspace to which the database belongs or only certain tables. If you choose **Select a subset of tables**, a **Select Tables** dialog opens allowing you to select one or more tables to back up. +1. Specify whether you want to back up all tables in the keyspace to which the database belongs or only certain tables. If you choose **Select a subset of tables**, a **Select Tables** dialog opens allowing you to select one or more tables to back up. -- Specify the period of time during which the backup is to be retained. Note that there's an option to never delete the backup. +1. Specify the period of time during which the backup is to be retained. Note that there's an option to never delete the backup. -- Optionally, specify the number of threads that should be available for the backup process. +1. Optionally, specify the number of threads that should be available for the backup process. -- Click **Backup**. +1. Click **Backup**. If the universe has [encryption at rest enabled](../../../security/enable-encryption-at-rest), data files are backed up as-is (encrypted) to reduce the computation cost of a backup and to keep the files encrypted. A universe key metadata file containing key references is also backed up. -For information on how to schedule backups for a later time or as a recurring task, see [Schedule universe YCQL data backups](../../schedule-data-backups/ycql/). +## View backup details -To view detailed information about an existing backup, click on it to open **Backup Details** shown in the following illustration:
    +To view detailed information about an existing backup, click on it to open **Backup Details** shown in the following illustration: ![Backup details](/images/yp/backup-details-ycql.png) -In addition to actions such as deleting and restoring the backup, as well as restoring tables and copying their locations, you can use **Backup Details** to add an incremental backup for universes that had the YB Controller automatically installed during their creation. To do so, click **Add Incremental Backup**, and then click **Confirm** on the **Add Incremental Backup** dialog. - -YB Controller-enabled universes can have their throttle parameters configured for more efficient backups and restore. For more information, see [Back up universe YSQL data](../../back-up-universe-data/ysql/). +## View all backups To access a list of all backups from all universes, including the deleted universes, navigate to **Backups** on the YugabyteDB Anywhere left-side menu, as per the following illustration: diff --git a/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ysql.md b/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ysql.md index 34edbc8824da..7cc1225c1011 100644 --- a/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ysql.md +++ b/docs/content/preview/yugabyte-platform/back-up-restore-universes/back-up-universe-data/ysql.md @@ -34,7 +34,11 @@ type: docs -
    You can use YugabyteDB Anywhere to back up your YugabyteDB universe YSQL data. +You can use YugabyteDB Anywhere to back up your YugabyteDB universe YSQL data. This includes actions such as deleting and restoring the backup, as well as restoring and copying the database location. + +If you are using YBA version 2.16 or later to manage universes with YugabyteDB version 2.16 or later, you can additionally create [incremental backups](#create-incremental-backups) and [configure backup performance parameters](#configure-backup-performance-parameters). + +For information on how to schedule backups for a later time or as a recurring task, see [Schedule universe YSQL data backups](../../schedule-data-backups/ysql/). To view, [restore](../../restore-universe-data/ysql/), or delete existing backups for your universe, navigate to that universe and select **Backups**, as per the following illustration: @@ -46,15 +50,15 @@ By default, the list displays all the backups generated for the universe regardl The **Backups** page allows you to create new backups that start immediately, as follows: -- Click **Backup now** to open the dialog shown in the following illustration:
    +1. Click **Backup now** to open the dialog shown in the following illustration: - ![Backup](/images/yp/create-backup-new-2.png) + ![Backup](/images/yp/create-backup-new-2.png) -- In the **Backup Now** dialog, select YSQL as the API type and then complete all the other fields. +1. In the **Backup Now** dialog, select YSQL as the API type and then complete all the other fields. - Notice that the contents of the **Select the storage config you want to use for your backup** field list depends on your existing backup storage configurations. For more information, see [Configure backup storage](../../configure-backup-storage/). + Notice that the contents of the **Select the storage config you want to use for your backup** field list depends on your existing backup storage configurations. For more information, see [Configure backup storage](../../configure-backup-storage/). -- Click **Backup**. +1. Click **Backup**. If the universe has [encryption at rest enabled](../../../security/enable-encryption-at-rest), data files are backed up as-is (encrypted) to reduce the computation cost of a backup and to keep the files encrypted. A universe key metadata file, containing key references, is also backed up. To allow YugabyteDB Anywhere to back up your data with the user authentication enabled, follow the instructions provided in [Edit configuration flags](../../../manage-deployments/edit-config-flags) to add the `ysql_enable_auth=true` and `ysql_hba_conf_csv="local all all trust"` YB-TServer flags. @@ -70,15 +74,11 @@ Versions of YugabyteDB Anywhere prior to 2.11.2.0 do not support backups of YSQL -For information on how to schedule backups for a later time or as a recurring task, see [Schedule universe YSQL data backups](../../schedule-data-backups/ysql/). +## View backup details -To view detailed information about an existing backup, click on it to open **Backup Details**. In addition to actions such as deleting and restoring the backup, as well as restoring and copying the database location, you can use **Backup Details** to add an [incremental backup](#create-incremental-backup) for universes that had the YB Controller automatically installed during their creation. The same universes allow you to configure their throttle parameters by clicking **... > Configure Throttle Parameters**, as per the following illustration: +To view detailed information about an existing backup, click on it to open **Backup Details**. -![Throttle parameters](/images/yp/backup-throttle-config-button.png) - -You define throttle parameters to enhance your universe's backups and restore performance using the **Configure Throttle Parameters** dialog shown in the following illustration: - -![Throttle](/images/yp/backup-restore-throttle.png) +## View all backups To access a list of all backups from all universes, including the deleted universes, navigate to **Backups** on the YugabyteDB Anywhere left-side menu, as per the following illustration: @@ -86,19 +86,32 @@ To access a list of all backups from all universes, including the deleted univer ## Create incremental backups +You can use **Backup Details** to add an incremental backup (YBA version 2.16 or later and YugabyteDB version 2.16 or later only). + Incremental backups are taken on top of a complete backup. To reduce the length of time spent on each backup, only SST files that are new to YugabyteDB and not present in the previous backups are incrementally backed up. For example, in most cases, for incremental backups occurring every hour, the 1-hour delta would be significantly smaller compared to the complete backup. The restore happens until the point of the defined increment. You can create an incremental backup on any complete or incremental backup taken using YB-Controller, as follows: -- Navigate to **Backups**, select a backup, and then click on it to open **Backup Details**. +1. Navigate to **Backups**, select a backup, and then click on it to open **Backup Details**. -- In the **Backup Details** view, click **Add Incremental Backup**. +1. In the **Backup Details** view, click **Add Incremental Backup**. -- On the **Add Incremental Backup** dialog, click **Confirm**. +1. On the **Add Incremental Backup** dialog, click **Confirm**. - -A successful incremental backup appears in the list of backups. +A successful incremental backup appears in the list of backups. You can delete only the full backup chain which includes a complete backup and its incremental backups. You cannot delete a subset of successful incremental backups. -A failed incremental backup, which you can delete, is reported similarly to any other failed backup operations. \ No newline at end of file +A failed incremental backup, which you can delete, is reported similarly to any other failed backup operations. + +## Configure backup performance parameters + +If you are using YBA version 2.16 or later to manage universes with YugabyteDB version 2.16 or later, you can tune parallelization and buffer parameters to enhance backup and restore performance. + +To configure throttle parameters, click **... > Configure Throttle Parameters**, as per the following illustration: + +![Throttle parameters](/images/yp/backup-throttle-config-button.png) + +You define throttle parameters using the **Configure Throttle Parameters** dialog shown in the following illustration: + +![Throttle](/images/yp/backup-restore-throttle.png) diff --git a/docs/content/preview/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md b/docs/content/preview/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md index e92abc677605..965c7916b988 100644 --- a/docs/content/preview/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md +++ b/docs/content/preview/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md @@ -18,9 +18,9 @@ You can configure Amazon S3 as your backup target, as follows: 1. Navigate to **Configs** > **Backup** > **Amazon S3**. -2. Click **Create S3 Backup** to access the configuration form shown in the following illustration:
    +2. Click **Create S3 Backup** to access the configuration form shown in the following illustration: - ![S3 Backup](/images/yp/cloud-provider-configuration-backup-aws.png)
    + ![S3 Backup](/images/yp/cloud-provider-configuration-backup-aws.png) 3. Use the **Configuration Name** field to provide a meaningful name for your backup configuration. @@ -30,14 +30,16 @@ You can configure Amazon S3 as your backup target, as follows: 6. Enter values for the **S3 Bucket** and **S3 Bucket Host Base** fields. - For information on how to obtain AWS credentials, see [Understanding and getting your AWS credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html). + For information on how to obtain AWS credentials, see [Understanding and getting your AWS credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html). 7. Click **Save**. You can configure access control for the S3 bucket as follows: -- Provide the required access control list (ACL), and then define **List, Write** permissions to access **Objects**, as well as **Read, Write** permissions for the bucket, as shown in the following illustration:

    - ![S3](/images/yp/backup-aws-access-control.png)
    +- Provide the required access control list (ACL), and then define **List, Write** permissions to access **Objects**, as well as **Read, Write** permissions for the bucket, as shown in the following illustration: + + ![S3](/images/yp/backup-aws-access-control.png) + - Create Bucket policy to enable access to the objects stored in the bucket. ### Required S3 IAM permissions @@ -58,9 +60,9 @@ You can configure Network File System (NFS) as your backup target, as follows: 1. Navigate to **Configs > Backup > Network File System**. -2. Click **Create NFS Backup** to access the configuration form shown in the following illustration:
    +2. Click **Create NFS Backup** to access the configuration form shown in the following illustration: - ![NFS Configuration](/images/yp/cloud-provider-configuration-backup-nfs.png)
    + ![NFS Configuration](/images/yp/cloud-provider-configuration-backup-nfs.png) 3. Use the **Configuration Name** field to provide a meaningful name for your backup configuration. @@ -74,15 +76,15 @@ You can configure Google Cloud Storage (GCS) as your backup target, as follows: 1. Navigate to **Configs > Backup > Google Cloud Storage**. -2. Click **Create GCS Backup** to access the configuration form shown in the following illustration:
    +2. Click **Create GCS Backup** to access the configuration form shown in the following illustration: - ![GCS Configuration](/images/yp/cloud-provider-configuration-backup-gcs.png)
    + ![GCS Configuration](/images/yp/cloud-provider-configuration-backup-gcs.png) 3. Use the **Configuration Name** field to provide a meaningful name for your backup configuration. 4. Complete the **GCS Bucket** and **GCS Credentials** fields. - For information on how to obtain GCS credentials, see [Cloud Storage authentication](https://cloud.google.com/storage/docs/authentication). + For information on how to obtain GCS credentials, see [Cloud Storage authentication](https://cloud.google.com/storage/docs/authentication). 5. Click **Save**. @@ -97,40 +99,36 @@ You can configure Azure as your backup target, as follows: 1. Create a storage account in Azure, as follows: - - - * Navigate to **Portal > Storage Account** and click **Add** (+). - * Complete the mandatory fields, such as **Resource group**, **Storage account name**, and **Location**, as per the following illustration: + - Navigate to **Portal > Storage Account** and click **Add** (+). + - Complete the mandatory fields, such as **Resource group**, **Storage account name**, and **Location**, as per the following illustration: - ![Azure storage account creation](/images/yp/cloud-provider-configuration-backup-azure-account.png)
    + ![Azure storage account creation](/images/yp/cloud-provider-configuration-backup-azure-account.png) 1. Create a blob container, as follows: - + - Open the storage account (for example, **storagetestazure**, as shown in the following illustration). + - Navigate to **Blob service > Containers > + Container** and then click **Create**. - * Open the storage account (for example, **storagetestazure**, as shown in the following illustration). - * Navigate to **Blob service > Containers > + Container** and then click **Create**.
    + ![Azure blob container creation](/images/yp/cloud-provider-configuration-backup-azure-blob-container.png) - ![Azure blob container creation](/images/yp/cloud-provider-configuration-backup-azure-blob-container.png)
    - -1. Obtain the container URL by navigating to **Container > Properties**, as shown in the following illustration:
    +1. Obtain the container URL by navigating to **Container > Properties**, as shown in the following illustration: ![Azure container properties](/images/yp/cloud-provider-configuration-backup-azure-container-properties.png) 1. Generate an SAS Token, as follows: - * Navigate to **Storage account > Shared access signature**, as shown in the following illustration. - * Under **Allowed resource types**, select **Container** and **Object**. - * Click **Generate SAS and connection string** and copy the SAS token. Note that the token should start with `?sv=`. - - ![Azure Shared Access Signature page](/images/yp/cloud-provider-configuration-backup-azure-generate-token.png)
    + - Navigate to **Storage account > Shared access signature**, as shown in the following illustration. + - Under **Allowed resource types**, select **Container** and **Object**. + - Click **Generate SAS and connection string** and copy the SAS token. Note that the token should start with `?sv=`. + + ![Azure Shared Access Signature page](/images/yp/cloud-provider-configuration-backup-azure-generate-token.png) 1. On your YugabyteDB Anywhere instance, provide the container URL and SAS token for creating a backup, as follows: - * Navigate to **Configs** > **Backup** > **Azure Storage**. - * Click **Create AZ Backup** to access the configuration form shown in the following illustration:
    - - ![Azure Configuration](/images/yp/cloud-provider-configuration-backup-azure.png)
    + - Navigate to **Configs** > **Backup** > **Azure Storage**. + - Click **Create AZ Backup** to access the configuration form shown in the following illustration: + + ![Azure Configuration](/images/yp/cloud-provider-configuration-backup-azure.png) - * Use the **Configuration Name** field to provide a meaningful name for your backup configuration. - * Enter values for the **Container URL** and **SAS Token** fields, and then click **Save**. + - Use the **Configuration Name** field to provide a meaningful name for your backup configuration. + - Enter values for the **Container URL** and **SAS Token** fields, and then click **Save**. diff --git a/docs/content/preview/yugabyte-platform/back-up-restore-universes/pitr.md b/docs/content/preview/yugabyte-platform/back-up-restore-universes/pitr.md index 2757a7704227..0a5f40af1db2 100644 --- a/docs/content/preview/yugabyte-platform/back-up-restore-universes/pitr.md +++ b/docs/content/preview/yugabyte-platform/back-up-restore-universes/pitr.md @@ -5,8 +5,8 @@ description: Point-in-time recovery menu: preview_yugabyte-platform: parent: back-up-restore-universes - identifier: pier - weight: 10 + identifier: pitr + weight: 50 type: docs --- @@ -24,7 +24,7 @@ You can create a PITR configuration as follows: ![PITR](/images/yp/pitr-main.png) - If there are currently no databases or keyspaces enabled for PITR, a message is displayed. + If there are currently no databases or keyspaces enabled for PITR, a message is displayed. 1. Click **Enable Point-in-time Recovery** to open the dialog shown in the following illustration: @@ -34,9 +34,9 @@ You can create a PITR configuration as follows: 1. Click **Enable**. -The database or keyspace is now added to the **Databases/Keyspaces with Point-In-Time Recovery Enabled** list. +The database or keyspace is now added to the **Databases/Keyspaces with Point-In-Time Recovery Enabled** list. -## Recover to a point in time +## Recover to a point in time You can recover a snapshot to a specific point in time as follows: @@ -60,4 +60,4 @@ You can disable PITR as follows: 3. Use the dialog shown in the following illustration to confirm that your intention is to disable PITR for the database or keyspace: - ![Disable](/images/yp/pitr-disable.png) \ No newline at end of file + ![Disable](/images/yp/pitr-disable.png) diff --git a/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md b/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md index 742c4bf26a81..1815a57f74e6 100644 --- a/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md +++ b/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md @@ -37,43 +37,43 @@ To back up your universe YCQL data immediately, see [Back up universe YCQL data] Before scheduling a backup of your universe YCQL data, create a policy, as follows: -- Navigate to **Universes**. +1. Navigate to **Universes**. -- Select the name of the universe for which you want to schedule backups. +1. Select the name of the universe for which you want to schedule backups. -- Select the **Tables** tab and click **Actions** to verify that backups are enabled. If disabled, click **Enable Backup**. +1. Select the **Tables** tab and click **Actions** to verify that backups are enabled. If disabled, click **Enable Backup**. -- Select the **Backups** tab and then select **Scheduled Backup Policies**. +1. Select the **Backups** tab and then select **Scheduled Backup Policies**. -- Click **Create Scheduled Backup Policy** to open the dialog shown in the following illustration: +1. Click **Create Scheduled Backup Policy** to open the dialog shown in the following illustration: - ![Create Backup form](/images/yp/scheduled-backup-ycql-1.png)
    + ![Create Backup form](/images/yp/scheduled-backup-ycql-1.png) -- Provide the backup policy name. +1. Provide the backup policy name. -- Ensure that the API type is set as YCQL. +1. Ensure that the API type is set as YCQL. -- Select the database to back up. +1. Select the database to back up. -- Specify whether you want to back up all tables in the keyspace to which the database belongs or only certain tables. If you choose **Select a subset of tables**, a **Select Tables** dialog opens allowing you to select one or more tables to back up. +1. Specify whether you want to back up all tables in the keyspace to which the database belongs or only certain tables. If you choose **Select a subset of tables**, a **Select Tables** dialog opens allowing you to select one or more tables to back up. -- Specify the period of time during which the backup is to be retained. Note that there's an option to never delete the backup. +1. Specify the period of time during which the backup is to be retained. Note that there's an option to never delete the backup. -- Specify the interval between backups or select **Use cron expression (UTC)**. +1. Specify the interval between backups or select **Use cron expression (UTC)**. -- For a YB Controller-powered universe, you can enable **Take incremental backups within full backup intervals** to instruct the schedule policy to take full backups periodically and incremental backups between those full backups. The incremental backups intervals be longer than the full scheduled backup frequency: +1. Enable **Take incremental backups within full backup intervals** to instruct the schedule policy to take full backups periodically and incremental backups between those full backups (YBA version 2.16 or later, and YugabyteDB version 2.16 or later only). The incremental backups intervals must be shorter than the full scheduled backup frequency: - ![Incremental Backup](/images/yp/scheduled-backup-ycql-incremental.png)
    + ![Incremental Backup](/images/yp/scheduled-backup-ycql-incremental.png) - If you disable the full backup, the incremental backup stops. If you enable the full backup again, the incremental backup schedule starts on new full backups. + If you disable the full backup, the incremental backup stops. If you enable the full backup again, the incremental backup schedule starts on new full backups. - If you delete the main full backup schedule, the incremental backup schedule is also deleted. + If you delete the main full backup schedule, the incremental backup schedule is also deleted. - You cannot modify any incremental backup-related property in the schedule; to overwrite any incremental backup property, you have to delete the existing schedule and create a new schedule if needed. + You cannot modify any incremental backup-related property in the schedule; to overwrite any incremental backup property, you have to delete the existing schedule and create a new schedule if needed. -- Optionally, specify the number of threads that should be available for the backup process. +1. Optionally, specify the number of threads that should be available for the backup process. -- Click **Create**. +1. Click **Create**. Subsequent backups are created based on the value you specified for **Set backup intervals** or **Use cron expression**. @@ -91,4 +91,4 @@ You can permanently remove a scheduled backup, as follows: 1. Navigate to the universe's **Backups** tab. 2. Find the scheduled backup in the **Backups** list and click **... > Delete Backup**. -To delete a policy, select **Scheduled Backup Policies**, find the policy and click its **Actions > Delete Policy**. \ No newline at end of file +To delete a policy, select **Scheduled Backup Policies**, find the policy and click its **Actions > Delete Policy**. diff --git a/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md b/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md index 6066532297c6..8628ae2dbf0a 100644 --- a/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md +++ b/docs/content/preview/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md @@ -33,7 +33,7 @@ type: docs -
    You can use YugabyteDB Anywhere to perform regularly scheduled backups of YugabyteDB universe data for all YSQL tables in a namespace. +You can use YugabyteDB Anywhere to perform regularly scheduled backups of YugabyteDB universe data for all YSQL tables in a namespace. To back up your universe YSQL data immediately, see [Back up universe YSQL data](../../back-up-universe-data/ysql/). @@ -50,9 +50,9 @@ Before scheduling a backup of your universe YSQL data, create a policy, as follo 1. Select the **Backups** tab and then select **Scheduled Backup Policies**. 1. Click **Create Scheduled Backup Policy** to open the dialog shown in the following illustration: - - ![Create Backup form](/images/yp/scheduled-backup-ysql-1.png)
    - + + ![Create Backup form](/images/yp/scheduled-backup-ysql-1.png) + 1. Provide the backup policy name. 1. Select the backup storage configuration. Notice that the contents of the **Select the storage config you want to use for your backup** list depends on your existing backup storage configurations. For more information, see [Configure backup storage](../../configure-backup-storage/). @@ -63,15 +63,15 @@ Before scheduling a backup of your universe YSQL data, create a policy, as follo 1. Specify the interval between backups or select **Use cron expression (UTC)**. -1. For a YB Controller-powered universe, you can enable **Take incremental backups within full backup intervals** to instruct the schedule policy to take full backups periodically and incremental backups between those full backups. The incremental backups intervals be longer than the full scheduled backup frequency: +1. Enable **Take incremental backups within full backup intervals** to instruct the schedule policy to take full backups periodically and incremental backups between those full backups (YBA version 2.16 or later, and YugabyteDB version 2.16 or later only). The incremental backups intervals must be shorter than the full scheduled backup frequency: - ![Incremental Backup](/images/yp/scheduled-backup-ycql-incremental.png)
    + ![Incremental Backup](/images/yp/scheduled-backup-ycql-incremental.png) - If you disable the full backup, the incremental backup stops. If you enable the full backup again, the incremental backup schedule starts on new full backups. + If you disable the full backup, the incremental backup stops. If you enable the full backup again, the incremental backup schedule starts on new full backups. - If you delete the main full backup schedule, the incremental backup schedule is also deleted. + If you delete the main full backup schedule, the incremental backup schedule is also deleted. - You cannot modify any incremental backup-related property in the schedule; to overwrite any incremental backup property, you have to delete the existing schedule and create a new schedule if needed. + You cannot modify any incremental backup-related property in the schedule; to overwrite any incremental backup property, you have to delete the existing schedule and create a new schedule if needed. 1. Specify the number of threads that should be available for the backup process. @@ -94,4 +94,3 @@ You can permanently remove a scheduled backup, as follows: 2. Find the scheduled backup in the **Backups** list and click **... > Delete Backup**. To delete a policy, select **Scheduled Backup Policies**, find the policy and click its **Actions > Delete Policy**. - diff --git a/docs/content/preview/yugabyte-platform/create-deployments/_index.md b/docs/content/preview/yugabyte-platform/create-deployments/_index.md index b01ea671ef1e..10078878b2db 100644 --- a/docs/content/preview/yugabyte-platform/create-deployments/_index.md +++ b/docs/content/preview/yugabyte-platform/create-deployments/_index.md @@ -12,6 +12,8 @@ menu: type: indexpage --- +YugabyteDB Anywhere can create a YugabyteDB universe with many instances (virtual machines, pods, and so on, provided by IaaS), logically grouped together to form one distributed database. Each universe includes one primary cluster and, optionally, one or more read replica clusters. All instances belonging to a cluster run on the same type of cloud provider instance. +
    -For information on how to schedule backups for a later time or as a recurring task, see [Schedule universe YSQL data backups](../../schedule-data-backups/ysql/). +## View backup details + +To view detailed information about an existing backup, click on it to open **Backup Details**. + +## View all backups + +To access a list of all backups from all universes, including the deleted universes, navigate to **Backups** on the YugabyteDB Anywhere left-side menu, as per the following illustration: + +![Backups](/images/yp/backups-list.png) + +## Create incremental backups + +You can use **Backup Details** to add an incremental backup (YBA version 2.16 or later for universes with YugabyteDB version 2.16 or later only). + +Incremental backups are taken on top of a complete backup. To reduce the length of time spent on each backup, only SST files that are new to YugabyteDB and not present in the previous backups are incrementally backed up. For example, in most cases, for incremental backups occurring every hour, the 1-hour delta would be significantly smaller compared to the complete backup. The restore happens until the point of the defined increment. + +You can create an incremental backup on any complete or incremental backup taken using YB-Controller, as follows: + +1. Navigate to **Backups**, select a backup, and then click on it to open **Backup Details**. + +1. In the **Backup Details** view, click **Add Incremental Backup**. + +1. On the **Add Incremental Backup** dialog, click **Confirm**. + +A successful incremental backup appears in the list of backups. + +You can delete only the full backup chain which includes a complete backup and its incremental backups. You cannot delete a subset of successful incremental backups. + +A failed incremental backup, which you can delete, is reported similarly to any other failed backup operations. + +## Configure backup performance parameters -To view detailed information about an existing backup, click on it to open **Backup Details**. In addition to actions such as deleting and restoring the backup, as well as restoring and copying the database location, you can use **Backup Details** to add an incremental backup for universes that had the YB Controller automatically installed during their creation. The same universes allow you to configure their throttle parameters by clicking **... > Configure Throttle Parameters**, as per the following illustration: +If you are using YBA version 2.16 or later to manage universes with YugabyteDB version 2.16 or later, you can tune parallelization and buffer parameters to enhance backup and restore performance. -![Throttle parameters](/images/yp/backup-throttle-config-button.png)
    +To configure throttle parameters, click **... > Configure Throttle Parameters**, as per the following illustration: -
    You define throttle parameters to enhance your universe's backups and restore performance using the **Configure Throttle Parameters** dialog shown in the following illustration:

    ![Throttle](/images/yp/backup-restore-throttle.png)


    To access a list of all backups from all universes, including the deleted universes, navigate to **Backups** on the YugabyteDB Anywhere left-side menu, as per the following illustration: +![Throttle parameters](/images/yp/backup-throttle-config-button.png) -![Backups](/images/yp/backups-list.png)
    +You define throttle parameters using the **Configure Throttle Parameters** dialog shown in the following illustration: -You can access the detailed information about a specific backup by clicking it to open the **Backup Details** dialog. \ No newline at end of file +![Throttle](/images/yp/backup-restore-throttle.png) diff --git a/docs/content/stable/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md b/docs/content/stable/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md index 3c1cfb7887a9..98c952e27172 100644 --- a/docs/content/stable/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md +++ b/docs/content/stable/yugabyte-platform/back-up-restore-universes/configure-backup-storage.md @@ -18,9 +18,9 @@ You can configure Amazon S3 as your backup target, as follows: 1. Navigate to **Configs** > **Backup** > **Amazon S3**. -2. Click **Create S3 Backup** to access the configuration form shown in the following illustration:
    +2. Click **Create S3 Backup** to access the configuration form shown in the following illustration: - ![S3 Backup](/images/yp/cloud-provider-configuration-backup-aws.png)
    + ![S3 Backup](/images/yp/cloud-provider-configuration-backup-aws.png) 3. Use the **Configuration Name** field to provide a meaningful name for your backup configuration. @@ -30,14 +30,16 @@ You can configure Amazon S3 as your backup target, as follows: 6. Enter values for the **S3 Bucket** and **S3 Bucket Host Base** fields. - For information on how to obtain AWS credentials, see [Understanding and getting your AWS credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html). + For information on how to obtain AWS credentials, see [Understanding and getting your AWS credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html). 7. Click **Save**. You can configure access control for the S3 bucket as follows: -- Provide the required access control list (ACL), and then define **List, Write** permissions to access **Objects**, as well as **Read, Write** permissions for the bucket, as shown in the following illustration:

    - ![S3](/images/yp/backup-aws-access-control.png)
    +- Provide the required access control list (ACL), and then define **List, Write** permissions to access **Objects**, as well as **Read, Write** permissions for the bucket, as shown in the following illustration: + + ![S3](/images/yp/backup-aws-access-control.png) + - Create Bucket policy to enable access to the objects stored in the bucket. ### Required S3 IAM permissions @@ -58,9 +60,9 @@ You can configure Network File System (NFS) as your backup target, as follows: 1. Navigate to **Configs > Backup > Network File System**. -2. Click **Create NFS Backup** to access the configuration form shown in the following illustration:
    +2. Click **Create NFS Backup** to access the configuration form shown in the following illustration: - ![NFS Configuration](/images/yp/cloud-provider-configuration-backup-nfs.png)
    + ![NFS Configuration](/images/yp/cloud-provider-configuration-backup-nfs.png) 3. Use the **Configuration Name** field to provide a meaningful name for your backup configuration. @@ -74,15 +76,15 @@ You can configure Google Cloud Storage (GCS) as your backup target, as follows: 1. Navigate to **Configs > Backup > Google Cloud Storage**. -2. Click **Create GCS Backup** to access the configuration form shown in the following illustration:
    +2. Click **Create GCS Backup** to access the configuration form shown in the following illustration: - ![GCS Configuration](/images/yp/cloud-provider-configuration-backup-gcs.png)
    + ![GCS Configuration](/images/yp/cloud-provider-configuration-backup-gcs.png) 3. Use the **Configuration Name** field to provide a meaningful name for your backup configuration. 4. Complete the **GCS Bucket** and **GCS Credentials** fields. - For information on how to obtain GCS credentials, see [Cloud Storage authentication](https://cloud.google.com/storage/docs/authentication). + For information on how to obtain GCS credentials, see [Cloud Storage authentication](https://cloud.google.com/storage/docs/authentication). 5. Click **Save**. @@ -97,40 +99,36 @@ You can configure Azure as your backup target, as follows: 1. Create a storage account in Azure, as follows: + - Navigate to **Portal > Storage Account** and click **Add** (+). + - Complete the mandatory fields, such as **Resource group**, **Storage account name**, and **Location**, as per the following illustration: - - * Navigate to **Portal > Storage Account** and click **Add** (+). - * Complete the mandatory fields, such as **Resource group**, **Storage account name**, and **Location**, as per the following illustration: - - ![Azure storage account creation](/images/yp/cloud-provider-configuration-backup-azure-account.png)
    + ![Azure storage account creation](/images/yp/cloud-provider-configuration-backup-azure-account.png) 1. Create a blob container, as follows: + - Open the storage account (for example, **storagetestazure**, as shown in the following illustration). + - Navigate to **Blob service > Containers > + Container** and then click **Create**. + ![Azure blob container creation](/images/yp/cloud-provider-configuration-backup-azure-blob-container.png) - * Open the storage account (for example, **storagetestazure**, as shown in the following illustration). - * Navigate to **Blob service > Containers > + Container** and then click **Create**.
    - - ![Azure blob container creation](/images/yp/cloud-provider-configuration-backup-azure-blob-container.png)
    - -1. Obtain the container URL by navigating to **Container > Properties**, as shown in the following illustration:
    +1. Obtain the container URL by navigating to **Container > Properties**, as shown in the following illustration: ![Azure container properties](/images/yp/cloud-provider-configuration-backup-azure-container-properties.png) 1. Generate an SAS Token, as follows: - * Navigate to **Storage account > Shared access signature**, as shown in the following illustration. - * Under **Allowed resource types**, select **Container** and **Object**. - * Click **Generate SAS and connection string** and copy the SAS token. Note that the token should start with `?sv=`. + - Navigate to **Storage account > Shared access signature**, as shown in the following illustration. + - Under **Allowed resource types**, select **Container** and **Object**. + - Click **Generate SAS and connection string** and copy the SAS token. Note that the token should start with `?sv=`. - ![Azure Shared Access Signature page](/images/yp/cloud-provider-configuration-backup-azure-generate-token.png)
    + ![Azure Shared Access Signature page](/images/yp/cloud-provider-configuration-backup-azure-generate-token.png) 1. On your YugabyteDB Anywhere instance, provide the container URL and SAS token for creating a backup, as follows: - * Navigate to **Configs** > **Backup** > **Azure Storage**. - * Click **Create AZ Backup** to access the configuration form shown in the following illustration:
    + - Navigate to **Configs** > **Backup** > **Azure Storage**. + - Click **Create AZ Backup** to access the configuration form shown in the following illustration: - ![Azure Configuration](/images/yp/cloud-provider-configuration-backup-azure.png)
    + ![Azure Configuration](/images/yp/cloud-provider-configuration-backup-azure.png) - * Use the **Configuration Name** field to provide a meaningful name for your backup configuration. - * Enter values for the **Container URL** and **SAS Token** fields, and then click **Save**. + - Use the **Configuration Name** field to provide a meaningful name for your backup configuration. + - Enter values for the **Container URL** and **SAS Token** fields, and then click **Save**. diff --git a/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md b/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md index e222ca6c78da..ed8dc31c1dc8 100644 --- a/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md +++ b/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ycql.md @@ -37,24 +37,22 @@ To back up your universe YCQL data immediately, see [Back up universe YCQL data] Before scheduling a backup of your universe YCQL data, create a policy, as follows: -- Navigate to **Universes**. -- Select the name of the universe for which you want to schedule backups. -- Select the **Tables** tab and click **Actions** to verify that backups are enabled. If disabled, click **Enable Backup**. -- Select the **Backups** tab and then select **Scheduled Backup Policies**. -- Click **Create Scheduled Backup Policy** to open the dialog shown in the following illustration: - -

    - -![Create Backup form](/images/yp/scheduled-backup-ycql.png)

    - -- Provide the backup policy name. -- Specify the interval between backups or select **Use cron expression (UTC)**. -- Set the API type as YCQL. -- Select the database to back up. -- Specify whether you want to back up all tables in the keyspace to which the database belongs or only certain tables. If you choose **Select a subset of tables**, a **Select Tables** dialog opens allowing you to select one or more tables to back up. -- Specify the period of time during which the backup is to be retained. Note that there's an option to never delete the backup. -- Optionally, specify the number of threads that should be available for the backup process. -- Click **Create**. +1. Navigate to **Universes**. +1. Select the name of the universe for which you want to schedule backups. +1. Select the **Tables** tab and click **Actions** to verify that backups are enabled. If disabled, click **Enable Backup**. +1. Select the **Backups** tab and then select **Scheduled Backup Policies**. +1. Click **Create Scheduled Backup Policy** to open the dialog shown in the following illustration: + + ![Create Backup form](/images/yp/scheduled-backup-ycql.png) + +1. Provide the backup policy name. +1. Specify the interval between backups or select **Use cron expression (UTC)**. +1. Set the API type as YCQL. +1. Select the database to back up. +1. Specify whether you want to back up all tables in the keyspace to which the database belongs or only certain tables. If you choose **Select a subset of tables**, a **Select Tables** dialog opens allowing you to select one or more tables to back up. +1. Specify the period of time during which the backup is to be retained. Note that there's an option to never delete the backup. +1. Optionally, specify the number of threads that should be available for the backup process. +1. Click **Create**. Subsequent backups are created based on the value you specified for **Set backup intervals** or **Use cron expression**. diff --git a/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md b/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md index 760f3e8d2432..9496d81e1dde 100644 --- a/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md +++ b/docs/content/stable/yugabyte-platform/back-up-restore-universes/schedule-data-backups/ysql.md @@ -29,7 +29,7 @@ type: docs -
    You can use YugabyteDB Anywhere to perform regularly scheduled backups of YugabyteDB universe data for all YSQL tables in a namespace. +You can use YugabyteDB Anywhere to perform regularly scheduled backups of YugabyteDB universe data for all YSQL tables in a namespace. To back up your universe YSQL data immediately, see [Back up universe YSQL data](../../back-up-universe-data/ysql/). @@ -45,9 +45,9 @@ Before scheduling a backup of your universe YSQL data, create a policy, as follo 1. Select the **Backups** tab and then select **Scheduled Backup Policies**. -1. Click **Create Scheduled Backup Policy** to open the dialog shown in the following illustration: +1. Click **Create Scheduled Backup Policy**. - ![Create Backup form](/images/yp/scheduled-backup-ysql.png)
    + 1. Provide the backup policy name. diff --git a/docs/content/stable/yugabyte-platform/create-deployments/_index.md b/docs/content/stable/yugabyte-platform/create-deployments/_index.md index 0b96e785e699..3c9c607df01a 100644 --- a/docs/content/stable/yugabyte-platform/create-deployments/_index.md +++ b/docs/content/stable/yugabyte-platform/create-deployments/_index.md @@ -12,6 +12,8 @@ menu: type: indexpage --- +YugabyteDB Anywhere can create a YugabyteDB universe with many instances (virtual machines, pods, and so on, provided by IaaS), logically grouped together to form one distributed database. Each universe includes one primary cluster and, optionally, one or more read replica clusters. All instances belonging to a cluster run on the same type of cloud provider instance. +
    table relation. - ASSERT_EQ(index_info.index_info->indexed_table_id(), table_info.table_id); - // Check table ---> index relation. - ASSERT_EQ(table_info.index_map.size(), 1); - ASSERT_EQ(table_info.index_map.count(index_info.table_id), 1); - ASSERT_EQ(table_info.index_map.begin()->first, index_info.table_id); - ASSERT_EQ(table_info.index_map.begin()->second.table_id(), index_info.table_id); - ASSERT_EQ(table_info.index_map.begin()->second.indexed_table_id(), table_info.table_id); - - ASSERT_OK(client_->DeleteTable(yb_table_name, /* wait */ true)); - ASSERT_EQ(0, ASSERT_RESULT(NumTables(table_name))); -} - -void AdminCliTest::DoTestExportImportIndexSnapshot(Transactional transactional) { - CreateTable(transactional); - CreateIndex(transactional); - - // Default tables that were created. - const string& table_name = table_.name().table_name(); - const string& keyspace = table_.name().namespace_name(); - const string& index_name = index_.name().table_name(); - const YBTableName yb_table_name(YQL_DATABASE_CQL, keyspace, table_name); - const YBTableName yb_index_name(YQL_DATABASE_CQL, keyspace, index_name); - - // Check there are 2 tables. - ASSERT_EQ(2, ASSERT_RESULT(NumTables(table_name))); - - // Create snapshot of default table and the attached index that gets created. - ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); - auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); - - string tmp_dir; - ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); - const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_snapshot.dat"); - ASSERT_OK(RunAdminToolCommand("export_snapshot", snapshot_id, snapshot_file)); - - // Import table and index into the existing table and index. - ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex(keyspace, table_name, index_name, /* same_ids */ true); - - // Import table and index with original names - not providing any names. - // (The table was deleted by the call above.) - ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex(keyspace, table_name, index_name); - - // Import table and index with original names - using the old names. - ASSERT_OK(RunAdminToolCommand( - "import_snapshot", snapshot_file, keyspace, table_name, index_name)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex(keyspace, table_name, index_name); - - // Import table and index with original names - providing only old table name. - ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, keyspace, table_name)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex(keyspace, table_name, index_name); - - // Renaming table and index, but keeping the same keyspace. - ASSERT_OK(RunAdminToolCommand( - "import_snapshot", snapshot_file, keyspace, "new_" + table_name, "new_" + index_name)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex(keyspace, "new_" + table_name, "new_" + index_name); - - // Keeping the same table and index names, but renaming the keyspace. - ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, "new_" + keyspace)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex("new_" + keyspace, table_name, index_name); - - // Repeat previous keyspace renaming case, but pass explicitly the same table name - // (and skip index name). - ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, "new_" + keyspace, table_name)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex("new_" + keyspace, table_name, index_name); - - // Import table and index into a new keyspace with old table and index names. - ASSERT_OK(RunAdminToolCommand( - "import_snapshot", snapshot_file, "new_" + keyspace, table_name, index_name)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex("new_" + keyspace, table_name, index_name); - - // Rename only index and keyspace, but keep the main table name. - ASSERT_OK(RunAdminToolCommand( - "import_snapshot", snapshot_file, "new_" + keyspace, table_name, "new_" + index_name)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex("new_" + keyspace, table_name, "new_" + index_name); - - // Import table and index with renaming into a new keyspace. - ASSERT_OK(RunAdminToolCommand( - "import_snapshot", snapshot_file, "new_" + keyspace, - "new_" + table_name, "new_" + index_name)); - // Wait for the new snapshot completion. - ASSERT_RESULT(WaitForAllSnapshots()); - CheckImportedTableWithIndex("new_" + keyspace, "new_" + table_name, "new_" + index_name); - - // Renaming table only, no new name for the index - expecting error. - ASSERT_NOK(RunAdminToolCommand( - "import_snapshot", snapshot_file, keyspace, "new_" + table_name)); - ASSERT_NOK(RunAdminToolCommand( - "import_snapshot", snapshot_file, "new_" + keyspace, "new_" + table_name)); -} - -TEST_F(AdminCliTest, TestExportImportIndexSnapshot) { - // Test non-transactional table. - DoTestExportImportIndexSnapshot(Transactional::kFalse); - LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); -} - -TEST_F(AdminCliTest, TestExportImportIndexSnapshot_ForTransactional) { - // Test the recreated transactional table. - DoTestExportImportIndexSnapshot(Transactional::kTrue); - LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); -} - -TEST_F(AdminCliTest, TestFailedRestoration) { - CreateTable(Transactional::kTrue); - const string& table_name = table_.name().table_name(); - const string& keyspace = table_.name().namespace_name(); - - // Create snapshot of default table that gets created. - ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); - const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); - LOG(INFO) << "Created snapshot: " << snapshot_id; - - string tmp_dir; - ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); - const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_snapshot.dat"); - ASSERT_OK(RunAdminToolCommand("export_snapshot", snapshot_id, snapshot_file)); - // Import below will not create a new table - reusing the old one. - ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); - - const YBTableName yb_table_name(YQL_DATABASE_CQL, keyspace, table_name); - CheckImportedTable(table_.get(), yb_table_name, /* same_ids */ true); - ASSERT_EQ(1, ASSERT_RESULT(NumTables(table_name))); - - auto new_snapshot_id = ASSERT_RESULT(GetCompletedSnapshot(2)); - if (new_snapshot_id == snapshot_id) { - new_snapshot_id = ASSERT_RESULT(GetCompletedSnapshot(2, 1)); - } - LOG(INFO) << "Imported snapshot: " << new_snapshot_id; - - ASSERT_OK(RunAdminToolCommand("restore_snapshot", new_snapshot_id)); - - const SysSnapshotEntryPB::State state = ASSERT_RESULT(WaitForRestoration()); - LOG(INFO) << "Restoration: " << SysSnapshotEntryPB::State_Name(state); - ASSERT_EQ(state, SysSnapshotEntryPB::FAILED); - - LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); -} - -// Configures two clusters with clients for the producer and consumer side of xcluster replication. -class XClusterAdminCliTest : public AdminCliTest { - public: - virtual int num_tablet_servers() { - return 3; - } - - void SetUp() override { - // Setup the default cluster as the consumer cluster. - AdminCliTest::SetUp(); - // Only create a table on the consumer, producer table may differ in tests. - CreateTable(Transactional::kTrue); - FLAGS_check_bootstrap_required = false; - - // Create the producer cluster. - opts.num_tablet_servers = num_tablet_servers(); - opts.cluster_id = kProducerClusterId; - producer_cluster_ = std::make_unique(opts); - ASSERT_OK(producer_cluster_->StartSync()); - ASSERT_OK(producer_cluster_->WaitForTabletServerCount(num_tablet_servers())); - producer_cluster_client_ = ASSERT_RESULT(producer_cluster_->CreateClient()); - } - - void DoTearDown() override { - if (producer_cluster_) { - producer_cluster_->Shutdown(); - } - AdminCliTest::DoTearDown(); - } - - Status WaitForSetupUniverseReplicationCleanUp(string producer_uuid) { - auto proxy = std::make_shared( - &client_->proxy_cache(), - VERIFY_RESULT(cluster_->GetLeaderMiniMaster())->bound_rpc_addr()); - - master::GetUniverseReplicationRequestPB req; - master::GetUniverseReplicationResponsePB resp; - return WaitFor([proxy, &req, &resp, producer_uuid]() -> Result { - req.set_producer_id(producer_uuid); - RpcController rpc; - Status s = proxy->GetUniverseReplication(req, &resp, &rpc); - - return resp.has_error(); - }, 20s, "Waiting for universe to delete"); - } - - protected: - Status CheckTableIsBeingReplicated( - const std::vector& tables, - SysCDCStreamEntryPB::State target_state = SysCDCStreamEntryPB::ACTIVE) { - string output = VERIFY_RESULT(RunAdminToolCommand(producer_cluster_.get(), "list_cdc_streams")); - string state_search_str = Format( - "value: \"$0\"", - SysCDCStreamEntryPB::State_Name(target_state)); - - for (const auto& table_id : tables) { - // Ensure a stream object with table_id exists. - size_t table_id_pos = output.find(table_id); - if (table_id_pos == string::npos) { - return STATUS_FORMAT( - NotFound, - "Table id '$0' not found in output: $1", - table_id, output); - } - - // Ensure that the strem object has the expected state value. - size_t state_pos = output.find(state_search_str, table_id_pos); - if (state_pos == string::npos) { - return STATUS_FORMAT( - NotFound, - "Table id '$0' has the incorrect state value in output: $1", - table_id, output); - } - - // Ensure that the state value we captured earlier did not belong - // to different stream object. - size_t next_stream_obj_pos = output.find("streams {", table_id_pos); - if (next_stream_obj_pos != string::npos && next_stream_obj_pos <= state_pos) { - return STATUS_FORMAT( - NotFound, - "Table id '$0' has no state value in output: $1", - table_id, output); - } - } - return Status::OK(); - } - - Result ProducerBackupServiceProxy() { - if (!producer_backup_service_proxy_) { - producer_backup_service_proxy_.reset(new MasterBackupProxy( - &producer_cluster_client_->proxy_cache(), - VERIFY_RESULT(producer_cluster_->GetLeaderMasterBoundRpcAddr()))); - } - return producer_backup_service_proxy_.get(); - } - - const string kProducerClusterId = "producer"; - std::unique_ptr producer_cluster_client_; - std::unique_ptr producer_cluster_; - MiniClusterOptions opts; - - private: - std::unique_ptr producer_backup_service_proxy_; -}; - -TEST_F(XClusterAdminCliTest, TestSetupUniverseReplication) { - client::TableHandle producer_cluster_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); - - // Setup universe replication, this should only return once complete. - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id())); - - // Check that the stream was properly created for this table. - ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); - - // Delete this universe so shutdown can proceed. - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); -} - -TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationChecksForColumnIdMismatch) { - client::TableHandle producer_table; - client::TableHandle consumer_table; - const YBTableName table_name(YQL_DATABASE_CQL, - "my_keyspace", - "column_id_mismatch_test_table"); - - client::kv_table_test::CreateTable(Transactional::kTrue, - NumTablets(), - producer_cluster_client_.get(), - &producer_table, - table_name); - client::kv_table_test::CreateTable(Transactional::kTrue, - NumTablets(), - client_.get(), - &consumer_table, - table_name); - - // Drop a column from the consumer table. - { - auto table_alterer = client_.get()->NewTableAlterer(table_name); - ASSERT_OK(table_alterer->DropColumn(kValueColumn)->Alter()); - } - - // Add the same column back into the producer table. This results in a schema mismatch - // between the producer and consumer versions of the table. - { - auto table_alterer = client_.get()->NewTableAlterer(table_name); - table_alterer->AddColumn(kValueColumn)->Type(INT32); - ASSERT_OK(table_alterer->timeout(MonoDelta::FromSeconds(60 * kTimeMultiplier))->Alter()); - } - - // Try setting up replication, this should fail due to the schema mismatch. - ASSERT_NOK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - - producer_table->id())); - - // Make a snapshot of the producer table. - auto timestamp = DateTime::TimestampToString(DateTime::TimestampNow()); - auto producer_backup_proxy = ASSERT_RESULT(ProducerBackupServiceProxy()); - ASSERT_OK(RunAdminToolCommand( - producer_cluster_.get(), "create_snapshot", producer_table.name().namespace_name(), - producer_table.name().table_name())); - - const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot(1, 0, producer_backup_proxy)); - ASSERT_RESULT(WaitForAllSnapshots(producer_backup_proxy)); - - string tmp_dir; - ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); - const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_producer_snapshot.dat"); - ASSERT_OK(RunAdminToolCommand( - producer_cluster_.get(), "export_snapshot", snapshot_id, snapshot_file)); - - // Delete consumer table, then import snapshot of producer table into the existing - // consumer table. This should fix the schema mismatch issue. - ASSERT_OK(client_->DeleteTable(table_name, /* wait */ true)); - ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); - - // Try running SetupUniverseReplication again, this time it should succeed. - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_table->id())); - - // Delete this universe so shutdown can proceed. - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); -} - -TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationFailsWithInvalidSchema) { - client::TableHandle producer_cluster_table; - - // Create a table with a different schema on the producer. - client::kv_table_test::CreateTable(Transactional::kFalse, // Results in different schema! - NumTablets(), - producer_cluster_client_.get(), - &producer_cluster_table); - - // Try to setup universe replication, should return with a useful error. - string error_msg; - // First provide a non-existant table id. - // ASSERT_NOK since this should fail. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id() + "-BAD")); - - // Wait for the universe to be cleaned up - ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); - - // Now try with the correct table id. - // Note that SetupUniverseReplication should call DeleteUniverseReplication to - // clean up the environment on failure, so we don't need to explicitly call - // DeleteUniverseReplication here. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id())); - - // Verify that error message has relevant information. - ASSERT_TRUE(error_msg.find("Source and target schemas don't match") != string::npos); -} - -TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationFailsWithInvalidBootstrapId) { - client::TableHandle producer_cluster_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); - - // Try to setup universe replication with a fake bootstrap id, should return with a useful error. - string error_msg; - // ASSERT_NOK since this should fail. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id(), - "fake-bootstrap-id")); - - // Verify that error message has relevant information. - ASSERT_TRUE(error_msg.find( - "Could not find CDC stream: stream_id: \"fake-bootstrap-id\"") != string::npos); -} - -TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationCleanupOnFailure) { - client::TableHandle producer_cluster_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); - - string error_msg; - // Try to setup universe replication with a fake bootstrap id, should result in failure. - // ASSERT_NOK since this should fail. We should be able to make consecutive calls to - // SetupUniverseReplication without having to call DeleteUniverseReplication first. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id(), - "fake-bootstrap-id")); - - // Wait for the universe to be cleaned up - ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); - - // Try to setup universe replication with fake producer master address. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - "fake-producer-address", - producer_cluster_table->id())); - - // Wait for the universe to be cleaned up - ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); - - // Try to setup universe replication with fake producer table id. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - "fake-producer-table-id")); - - // Wait for the universe to be cleaned up - ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); - - // Test when producer and local table have different schema. - client::TableHandle producer_cluster_table2; - client::TableHandle consumer_table2; - const YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "different_schema_test_table"); - - client::kv_table_test::CreateTable(Transactional::kFalse, // Results in different schema! - NumTablets(), - producer_cluster_client_.get(), - &producer_cluster_table2, - kTableName2); - client::kv_table_test::CreateTable(Transactional::kTrue, - NumTablets(), - client_.get(), - &consumer_table2, - kTableName2); - - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table2->id())); - - // Wait for the universe to be cleaned up - ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); - - // Verify that the environment is cleaned up correctly after failure. - // A valid call to SetupUniverseReplication after the failure should succeed - // without us having to first call DeleteUniverseReplication. - ASSERT_OK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id())); - // Verify table is being replicated. - ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); - - // Try calling SetupUniverseReplication again. This should fail as the producer - // is already present. However, in this case, DeleteUniverseReplication should - // not be called since the error was due to failing a sanity check. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id())); - // Verify the universe replication has not been deleted is still there. - ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); - - // Delete universe. - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); -} - -TEST_F(XClusterAdminCliTest, TestListCdcStreamsWithBootstrappedStreams) { - const int kStreamUuidLength = 32; - client::TableHandle producer_cluster_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); - - string output = ASSERT_RESULT(RunAdminToolCommand(producer_cluster_.get(), "list_cdc_streams")); - // First check that the table and bootstrap status are not present. - ASSERT_EQ(output.find(producer_cluster_table->id()), string::npos); - ASSERT_EQ(output.find(SysCDCStreamEntryPB::State_Name(SysCDCStreamEntryPB::INITIATED)), - string::npos); - - // Bootstrap the producer. - output = ASSERT_RESULT(RunAdminToolCommand( - producer_cluster_.get(), "bootstrap_cdc_producer", producer_cluster_table->id())); - // Get the bootstrap id (output format is "table id: 123, CDC bootstrap id: 123\n"). - string bootstrap_id = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); - - // Check list_cdc_streams again for the table and the status INITIATED. - ASSERT_OK(CheckTableIsBeingReplicated( - {producer_cluster_table->id()}, SysCDCStreamEntryPB::INITIATED)); - - // Setup universe replication using the bootstrap_id - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id(), - bootstrap_id)); - - - // Check list_cdc_streams again for the table and the status ACTIVE. - ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); - - // Try restarting the producer to ensure that the status persists. - ASSERT_OK(producer_cluster_->RestartSync()); - ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); - - // Delete this universe so shutdown can proceed. - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); -} - -TEST_F(XClusterAdminCliTest, TestRenameUniverseReplication) { - client::TableHandle producer_cluster_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); - - // Setup universe replication, this should only return once complete. - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id())); - - // Check that the stream was properly created for this table. - ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); - - // Now rename the replication group and then try to perform operations on it. - std::string new_replication_id = "new_replication_id"; - ASSERT_OK(RunAdminToolCommand("alter_universe_replication", - kProducerClusterId, - "rename_id", - new_replication_id)); - - // Assert that using old universe id fails. - ASSERT_NOK(RunAdminToolCommand("set_universe_replication_enabled", - kProducerClusterId, - 0)); - // But using correct name should succeed. - ASSERT_OK(RunAdminToolCommand("set_universe_replication_enabled", - new_replication_id, - 0)); - - // Also create a second stream so we can verify name collisions. - std::string collision_id = "collision_id"; - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - collision_id, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id())); - ASSERT_NOK(RunAdminToolCommand("alter_universe_replication", - new_replication_id, - "rename_id", - collision_id)); - - // Using correct name should still succeed. - ASSERT_OK(RunAdminToolCommand("set_universe_replication_enabled", - new_replication_id, - 1)); - - // Also test that we can rename again. - std::string new_replication_id2 = "new_replication_id2"; - ASSERT_OK(RunAdminToolCommand("alter_universe_replication", - new_replication_id, - "rename_id", - new_replication_id2)); - - // Assert that using old universe ids fails. - ASSERT_NOK(RunAdminToolCommand("set_universe_replication_enabled", - kProducerClusterId, - 1)); - ASSERT_NOK(RunAdminToolCommand("set_universe_replication_enabled", - new_replication_id, - 1)); - // But using new correct name should succeed. - ASSERT_OK(RunAdminToolCommand("set_universe_replication_enabled", - new_replication_id2, - 1)); - - // Delete this universe so shutdown can proceed. - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", new_replication_id2)); - // Also delete second one too. - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", collision_id)); -} - -class XClusterAlterUniverseAdminCliTest : public XClusterAdminCliTest { - public: - void SetUp() override { - YB_SKIP_TEST_IN_TSAN(); - - // Use more masters so we can test set_master_addresses - opts.num_masters = 3; - - XClusterAdminCliTest::SetUp(); - } -}; - -TEST_F(XClusterAlterUniverseAdminCliTest, TestAlterUniverseReplication) { - client::TableHandle producer_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); - - // Create an additional table to test with as well. - const YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "ql_client_test_table2"); - client::TableHandle consumer_table2; - client::TableHandle producer_table2; - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), client_.get(), &consumer_table2, kTableName2); - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table2, - kTableName2); - - // Setup replication with both tables, this should only return once complete. - // Only use the leader master address initially. - ASSERT_OK(RunAdminToolCommand( - "setup_universe_replication", - kProducerClusterId, - ASSERT_RESULT(producer_cluster_->GetLeaderMiniMaster())->bound_rpc_addr_str(), - producer_table->id() + "," + producer_table2->id())); - - // Test set_master_addresses, use all the master addresses now. - ASSERT_OK(RunAdminToolCommand("alter_universe_replication", - kProducerClusterId, - "set_master_addresses", - producer_cluster_->GetMasterAddresses())); - ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id(), producer_table2->id()})); - - // Test removing a table. - ASSERT_OK(RunAdminToolCommand("alter_universe_replication", - kProducerClusterId, - "remove_table", - producer_table->id())); - ASSERT_OK(CheckTableIsBeingReplicated({producer_table2->id()})); - ASSERT_NOK(CheckTableIsBeingReplicated({producer_table->id()})); - - // Test adding a table. - ASSERT_OK(RunAdminToolCommand("alter_universe_replication", - kProducerClusterId, - "add_table", - producer_table->id())); - ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id(), producer_table2->id()})); - - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); -} - -TEST_F(XClusterAlterUniverseAdminCliTest, TestAlterUniverseReplicationWithBootstrapId) { - const int kStreamUuidLength = 32; - client::TableHandle producer_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); - - // Create an additional table to test with as well. - const YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "ql_client_test_table2"); - client::TableHandle consumer_table2; - client::TableHandle producer_table2; - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), client_.get(), &consumer_table2, kTableName2); - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table2, - kTableName2); - - // Get bootstrap ids for both producer tables and get bootstrap ids. - string output = ASSERT_RESULT(RunAdminToolCommand( - producer_cluster_.get(), "bootstrap_cdc_producer", producer_table->id())); - string bootstrap_id1 = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); - ASSERT_OK(CheckTableIsBeingReplicated( - {producer_table->id()}, - master::SysCDCStreamEntryPB_State_INITIATED)); - - output = ASSERT_RESULT(RunAdminToolCommand( - producer_cluster_.get(), "bootstrap_cdc_producer", producer_table2->id())); - string bootstrap_id2 = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); - ASSERT_OK(CheckTableIsBeingReplicated( - {producer_table2->id()}, - master::SysCDCStreamEntryPB_State_INITIATED)); - - // Setup replication with first table, this should only return once complete. - // Only use the leader master address initially. - ASSERT_OK(RunAdminToolCommand( - "setup_universe_replication", - kProducerClusterId, - ASSERT_RESULT(producer_cluster_->GetLeaderMiniMaster())->bound_rpc_addr_str(), - producer_table->id(), - bootstrap_id1)); - - // Test adding the second table with bootstrap id - ASSERT_OK(RunAdminToolCommand("alter_universe_replication", - kProducerClusterId, - "add_table", - producer_table2->id(), - bootstrap_id2)); - ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id(), producer_table2->id()})); - - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); -} - -// delete_cdc_stream tests -TEST_F(XClusterAdminCliTest, TestDeleteCDCStreamWithConsumerSetup) { - client::TableHandle producer_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); - - // Setup universe replication, this should only return once complete. - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_table->id())); - ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id()})); - - string stream_id = ASSERT_RESULT(GetRecentStreamId(producer_cluster_.get())); - - // Should fail as it should meet the conditions to be stopped. - ASSERT_NOK(RunAdminToolCommand(producer_cluster_.get(), "delete_cdc_stream", stream_id)); - // Should pass as we force it. - ASSERT_OK(RunAdminToolCommand(producer_cluster_.get(), "delete_cdc_stream", stream_id, - "force_delete")); - // Delete universe should fail as we've force deleted the stream. - ASSERT_NOK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", - kProducerClusterId, - "ignore-errors")); -} - -TEST_F(XClusterAdminCliTest, TestDeleteCDCStreamWithAlterUniverse) { - client::TableHandle producer_table; - constexpr int kNumTables = 3; - const client::YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "test_table2"); - - // Create identical table on producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); - - // Create some additional tables. The reason is that the number of tables removed using - // alter_universe_replication remove_table must be less than the total number of tables. - string producer_table_ids_str = producer_table->id(); - std::vector producer_table_ids = {producer_table->id()}; - for (int i = 1; i < kNumTables; i++) { - client::TableHandle tmp_producer_table, tmp_consumer_table; - const client::YBTableName kTestTableName( - YQL_DATABASE_CQL, "my_keyspace", Format("test_table_$0", i)); - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), client_.get(), &tmp_consumer_table, - kTestTableName); - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &tmp_producer_table, - kTestTableName); - producer_table_ids_str += Format(",$0", tmp_producer_table->id()); - producer_table_ids.push_back(tmp_producer_table->id()); - } - - // Setup universe replication. - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_table_ids_str)); - ASSERT_OK(CheckTableIsBeingReplicated(producer_table_ids)); - - // Obtain the stream ID for the first table. - string stream_id = ASSERT_RESULT(GetRecentStreamId( - producer_cluster_.get(), producer_table->id())); - ASSERT_FALSE(stream_id.empty()); - - // Mark one stream as deleted. - ASSERT_OK(RunAdminToolCommand(producer_cluster_.get(), - "delete_cdc_stream", - stream_id, - "force_delete")); - - // Remove table should fail as its stream is marked as deleting on producer. - ASSERT_NOK(RunAdminToolCommand("alter_universe_replication", - kProducerClusterId, - "remove_table", - producer_table->id())); - ASSERT_OK(RunAdminToolCommand("alter_universe_replication", - kProducerClusterId, - "remove_table", - producer_table->id(), - "ignore-errors")); -} - -TEST_F(XClusterAdminCliTest, TestWaitForReplicationDrain) { - client::TableHandle producer_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); - - // Setup universe replication. - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_table->id())); - ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id()})); - string stream_id = ASSERT_RESULT(GetRecentStreamId(producer_cluster_.get())); - - // API should succeed with correctly formatted arguments. - ASSERT_OK(RunAdminToolCommand(producer_cluster_.get(), - "wait_for_replication_drain", stream_id)); - ASSERT_OK(RunAdminToolCommand(producer_cluster_.get(), - "wait_for_replication_drain", stream_id, GetCurrentTimeMicros())); - ASSERT_OK(RunAdminToolCommand(producer_cluster_.get(), - "wait_for_replication_drain", stream_id, "minus", "3s")); - - // API should fail with an invalid stream ID. - ASSERT_NOK(RunAdminToolCommand(producer_cluster_.get(), - "wait_for_replication_drain", "abc")); - - // API should fail with an invalid target_time format. - ASSERT_NOK(RunAdminToolCommand(producer_cluster_.get(), - "wait_for_replication_drain", stream_id, 123)); - ASSERT_NOK(RunAdminToolCommand(producer_cluster_.get(), - "wait_for_replication_drain", stream_id, "minus", "hello")); -} - -TEST_F(XClusterAdminCliTest, TestDeleteCDCStreamWithBootstrap) { - const int kStreamUuidLength = 32; - client::TableHandle producer_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); - - string output = ASSERT_RESULT(RunAdminToolCommand( - producer_cluster_.get(), "bootstrap_cdc_producer", producer_table->id())); - // Get the bootstrap id (output format is "table id: 123, CDC bootstrap id: 123\n"). - string bootstrap_id = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); - - // Setup universe replication, this should only return once complete. - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_table->id(), - bootstrap_id)); - ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id()})); - - // Should fail as it should meet the conditions to be stopped. - ASSERT_NOK(RunAdminToolCommand(producer_cluster_.get(), "delete_cdc_stream", bootstrap_id)); - // Delete should work fine from deleting from universe. - ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); -} - -TEST_F(AdminCliTest, TestDeleteCDCStreamWithCreateCDCStream) { - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), client_.get(), &table_); - - // Create CDC stream - ASSERT_OK(RunAdminToolCommand(cluster_.get(), - "create_cdc_stream", - table_->id())); - - string stream_id = ASSERT_RESULT(GetRecentStreamId(cluster_.get())); - - // Should be deleted. - ASSERT_OK(RunAdminToolCommand(cluster_.get(), "delete_cdc_stream", stream_id)); -} - -TEST_F(XClusterAdminCliTest, TestFailedSetupUniverseWithDeletion) { - client::TableHandle producer_cluster_table; - - // Create an identical table on the producer. - client::kv_table_test::CreateTable( - Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); - - string error_msg; - // Setup universe replication, this should only return once complete. - // First provide a non-existant table id. - // ASSERT_NOK since this should fail. - ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput(&error_msg, - "setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id() + "-BAD")); - - ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); - - // Universe should be deleted by BG cleanup - ASSERT_NOK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); - - ASSERT_OK(RunAdminToolCommand("setup_universe_replication", - kProducerClusterId, - producer_cluster_->GetMasterAddresses(), - producer_cluster_table->id())); - std::this_thread::sleep_for(5s); -} - -TEST_F(AdminCliTest, TestSetPreferredZone) { - const std::string c1z1 = "c1.r1.z1"; - const std::string c1z2 = "c1.r1.z2"; - const std::string c2z1 = "c2.r1.z1"; - const std::string c1z1_json = - "{\"placementCloud\":\"c1\",\"placementRegion\":\"r1\",\"placementZone\":\"z1\"}"; - const std::string c1z2_json = - "{\"placementCloud\":\"c1\",\"placementRegion\":\"r1\",\"placementZone\":\"z2\"}"; - const std::string c2z1_json = - "{\"placementCloud\":\"c2\",\"placementRegion\":\"r1\",\"placementZone\":\"z1\"}"; - const std::string affinitized_leaders_json_Start = "\"affinitizedLeaders\""; - const std::string multi_affinitized_leaders_json_start = - "\"multiAffinitizedLeaders\":[{\"zones\":["; - const std::string json_end = "]}]"; - - ASSERT_OK(RunAdminToolCommand( - "modify_placement_info", strings::Substitute("$0,$1,$2", c1z1, c1z2, c2z1), 5, "")); - - ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", "")); - auto output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); - ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); - ASSERT_EQ(output.find(multi_affinitized_leaders_json_start), string::npos); - - ASSERT_OK(RunAdminToolCommand("set_preferred_zones", c1z1)); - output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); - ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); - ASSERT_NE(output.find(multi_affinitized_leaders_json_start + c1z1_json + json_end), string::npos); - - ASSERT_OK(RunAdminToolCommand("set_preferred_zones", c1z1, c1z2, c2z1)); - output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); - ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); - ASSERT_NE( - output.find( - multi_affinitized_leaders_json_start + c1z1_json + "," + c1z2_json + "," + c2z1_json + - json_end), - string::npos); - - ASSERT_OK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:1", c1z1))); - output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); - ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); - ASSERT_NE(output.find(multi_affinitized_leaders_json_start + c1z1_json + json_end), string::npos); - - ASSERT_OK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:1", c1z1), c1z2)); - output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); - ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); - ASSERT_NE( - output.find(multi_affinitized_leaders_json_start + c1z1_json + "," + c1z2_json + json_end), - string::npos); - - ASSERT_OK(RunAdminToolCommand( - "set_preferred_zones", strings::Substitute("$0:1", c1z1), strings::Substitute("$0:2", c1z2), - strings::Substitute("$0:3", c2z1))); - output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); - ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); - ASSERT_NE( - output.find( - multi_affinitized_leaders_json_start + c1z1_json + "]},{\"zones\":[" + c1z2_json + - "]},{\"zones\":[" + c2z1_json + json_end), - string::npos); - - ASSERT_OK(RunAdminToolCommand( - "set_preferred_zones", strings::Substitute("$0:1", c1z1), strings::Substitute("$0:1", c1z2), - strings::Substitute("$0:2", c2z1))); - output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); - ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); - ASSERT_NE( - output.find( - multi_affinitized_leaders_json_start + c1z1_json + "," + c1z2_json + "]},{\"zones\":[" + - c2z1_json + json_end), - string::npos); - - ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:", c1z1))); - ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:0", c1z1))); - ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:-13", c1z1))); - ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:2", c1z1))); - ASSERT_NOK(RunAdminToolCommand( - "set_preferred_zones", strings::Substitute("$0:1", c1z1), strings::Substitute("$0:3", c1z2))); - ASSERT_NOK(RunAdminToolCommand( - "set_preferred_zones", strings::Substitute("$0:2", c1z1), strings::Substitute("$0:2", c1z2), - strings::Substitute("$0:3", c2z1))); -} - -class XClusterAdminCliTest_Large : public XClusterAdminCliTest { - public: - void SetUp() override { - // Skip this test in TSAN since the test will time out waiting - // for table creation to finish. - YB_SKIP_TEST_IN_TSAN(); - - XClusterAdminCliTest::SetUp(); - } - int num_tablet_servers() override { - return 5; - } -}; - -TEST_F(XClusterAdminCliTest_Large, TestBootstrapProducerPerformance) { - const int table_count = 10; - const int tablet_count = 5; - const int expected_runtime_seconds = 15 * kTimeMultiplier; - const std::string keyspace = "my_keyspace"; - std::vector tables; - - for (int i = 0; i < table_count; i++) { - client::TableHandle th; - tables.push_back(th); - - // Build the table. - client::YBSchemaBuilder builder; - builder.AddColumn(kKeyColumn)->Type(INT32)->HashPrimaryKey()->NotNull(); - builder.AddColumn(kValueColumn)->Type(INT32); - - TableProperties table_properties; - table_properties.SetTransactional(true); - builder.SetTableProperties(table_properties); - - const YBTableName table_name(YQL_DATABASE_CQL, keyspace, - Format("bootstrap_producer_performance_test_table_$0", i)); - ASSERT_OK(producer_cluster_client_.get()->CreateNamespaceIfNotExists( - table_name.namespace_name(), - table_name.namespace_type())); - - ASSERT_OK(tables.at(i).Create( - table_name, tablet_count, producer_cluster_client_.get(), &builder)); - } - - std::string table_ids = tables.at(0)->id(); - for (size_t i = 1; i < tables.size(); ++i) { - table_ids += "," + tables.at(i)->id(); - } - - // Wait for load balancer to be idle until we call bootstrap_cdc_producer. - // This prevents TABLET_DATA_TOMBSTONED errors when we make rpc calls. - // Todo: need to improve bootstrap behaviour with load balancer. - ASSERT_OK(WaitFor( - [this, table_ids]() -> Result { - return producer_cluster_client_->IsLoadBalancerIdle(); - }, - MonoDelta::FromSeconds(120 * kTimeMultiplier), - "Waiting for load balancer to be idle")); - - // Add delays to all rpc calls to simulate live environment. - FLAGS_TEST_yb_inbound_big_calls_parse_delay_ms = 5; - FLAGS_rpc_throttle_threshold_bytes = 0; - // Enable parallelized version of BootstrapProducer. - FLAGS_parallelize_bootstrap_producer = true; - - // Check that bootstrap_cdc_producer returns within time limit. - ASSERT_OK(WaitFor( - [this, table_ids]() -> Result { - auto res = RunAdminToolCommand(producer_cluster_.get(), - "bootstrap_cdc_producer", table_ids); - return res.ok(); - }, - MonoDelta::FromSeconds(expected_runtime_seconds), - "Waiting for bootstrap_cdc_producer to complete")); -} - -} // namespace tools -} // namespace yb diff --git a/ent/src/yb/tools/yb-admin_cli.h b/ent/src/yb/tools/yb-admin_cli.h deleted file mode 100644 index 48594cebf7f0..000000000000 --- a/ent/src/yb/tools/yb-admin_cli.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) YugaByte, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations -// under the License. - -#pragma once - -namespace yb { -namespace tools { -namespace enterprise { - -class ClusterAdminClient; - -} // namespace enterprise -} // namespace tools -} // namespace yb - -#include "../../../../src/yb/tools/yb-admin_cli.h" - -namespace yb { -namespace tools { -namespace enterprise { - -class ClusterAdminCli : public yb::tools::ClusterAdminCli { - typedef yb::tools::ClusterAdminCli super; - - private: - void RegisterCommandHandlers(ClusterAdminClientClass* client) override; -}; - -} // namespace enterprise -} // namespace tools -} // namespace yb diff --git a/ent/src/yb/tools/yb-admin_cli_ent.cc b/ent/src/yb/tools/yb-admin_cli_ent.cc deleted file mode 100644 index 49b270eb0cc0..000000000000 --- a/ent/src/yb/tools/yb-admin_cli_ent.cc +++ /dev/null @@ -1,959 +0,0 @@ -// Copyright (c) YugaByte, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations -// under the License. - -#include "yb/tools/yb-admin_cli.h" - -#include -#include - -#include - -#include "yb/common/hybrid_time.h" -#include "yb/common/json_util.h" -#include "yb/common/snapshot.h" -#include "yb/gutil/strings/util.h" -#include "yb/tools/yb-admin_client.h" -#include "yb/tools/yb-admin_util.h" -#include "yb/util/date_time.h" -#include "yb/util/format.h" -#include "yb/util/jsonwriter.h" -#include "yb/util/pb_util.h" -#include "yb/util/result.h" -#include "yb/util/status_format.h" -#include "yb/util/stol_utils.h" -#include "yb/util/string_case.h" - -namespace yb { -namespace tools { -namespace enterprise { - -using std::string; -using std::vector; - -using client::YBTableName; -using strings::Substitute; - -namespace { - -const string kMinus = "minus"; - -template -Result GetOptionalArg(const Args& args, size_t idx) { - if (args.size() <= idx) { - return T::Nil(); - } - if (args.size() > idx + 1) { - return STATUS_FORMAT(InvalidArgument, - "Too many arguments for command, at most $0 expected, but $1 found", - idx + 1, args.size()); - } - return VERIFY_RESULT(T::FromString(args[idx])); -} - -Status ListSnapshots(ClusterAdminClientClass* client, const EnumBitSet& flags) { - auto snapshot_response = VERIFY_RESULT(client->ListSnapshots(flags)); - - rapidjson::Document document(rapidjson::kObjectType); - bool json = flags.Test(ListSnapshotsFlag::JSON); - - if (snapshot_response.has_current_snapshot_id()) { - if (json) { - AddStringField( - "current_snapshot_id", SnapshotIdToString(snapshot_response.current_snapshot_id()), - &document, &document.GetAllocator()); - } else { - std::cout << "Current snapshot id: " - << SnapshotIdToString(snapshot_response.current_snapshot_id()) << std::endl; - } - } - - rapidjson::Value json_snapshots(rapidjson::kArrayType); - if (!json) { - if (snapshot_response.snapshots_size()) { - // Using 2 tabs so that the header can be aligned to the time. - std::cout << RightPadToUuidWidth("Snapshot UUID") << kColumnSep << "State" << kColumnSep - << kColumnSep << "Creation Time" << std::endl; - } else { - std::cout << "No snapshots" << std::endl; - } - } - - for (master::SnapshotInfoPB& snapshot : *snapshot_response.mutable_snapshots()) { - rapidjson::Value json_snapshot(rapidjson::kObjectType); - if (json) { - AddStringField( - "id", SnapshotIdToString(snapshot.id()), &json_snapshot, &document.GetAllocator()); - const auto& entry = snapshot.entry(); - AddStringField( - "state", master::SysSnapshotEntryPB::State_Name(entry.state()), &json_snapshot, - &document.GetAllocator()); - AddStringField( - "snapshot_time", HybridTimeToString(HybridTime::FromPB(entry.snapshot_hybrid_time())), - &json_snapshot, &document.GetAllocator()); - AddStringField( - "previous_snapshot_time", - HybridTimeToString(HybridTime::FromPB(entry.previous_snapshot_hybrid_time())), - &json_snapshot, &document.GetAllocator()); - } else { - std::cout << SnapshotIdToString(snapshot.id()) << kColumnSep - << master::SysSnapshotEntryPB::State_Name(snapshot.entry().state()) << kColumnSep - << HybridTimeToString(HybridTime::FromPB(snapshot.entry().snapshot_hybrid_time())) - << std::endl; - } - - // Not implemented in json mode. - if (flags.Test(ListSnapshotsFlag::SHOW_DETAILS)) { - for (master::SysRowEntry& entry : *snapshot.mutable_entry()->mutable_entries()) { - string decoded_data; - switch (entry.type()) { - case master::SysRowEntryType::NAMESPACE: { - auto meta = - VERIFY_RESULT(pb_util::ParseFromSlice(entry.data())); - meta.clear_transaction(); - decoded_data = JsonWriter::ToJson(meta, JsonWriter::COMPACT); - break; - } - case master::SysRowEntryType::UDTYPE: { - auto meta = - VERIFY_RESULT(pb_util::ParseFromSlice(entry.data())); - decoded_data = JsonWriter::ToJson(meta, JsonWriter::COMPACT); - break; - } - case master::SysRowEntryType::TABLE: { - auto meta = - VERIFY_RESULT(pb_util::ParseFromSlice(entry.data())); - meta.clear_schema(); - meta.clear_partition_schema(); - meta.clear_index_info(); - meta.clear_indexes(); - meta.clear_transaction(); - decoded_data = JsonWriter::ToJson(meta, JsonWriter::COMPACT); - break; - } - default: - break; - } - - if (!decoded_data.empty()) { - entry.set_data("DATA"); - std::cout << kColumnSep - << StringReplace( - JsonWriter::ToJson(entry, JsonWriter::COMPACT), "\"DATA\"", decoded_data, - false) - << std::endl; - } - } - } - if (json) { - json_snapshots.PushBack(json_snapshot, document.GetAllocator()); - } - } - - if (json) { - document.AddMember("snapshots", json_snapshots, document.GetAllocator()); - std::cout << common::PrettyWriteRapidJsonToString(document) << std::endl; - return Status::OK(); - } - - auto restorations_result = - VERIFY_RESULT(client->ListSnapshotRestorations(TxnSnapshotRestorationId::Nil())); - if (restorations_result.restorations_size() == 0) { - std::cout << "No snapshot restorations" << std::endl; - } else if (flags.Test(ListSnapshotsFlag::NOT_SHOW_RESTORED)) { - std::cout << "Not show fully RESTORED entries" << std::endl; - } - - bool title_printed = false; - for (const auto& restoration : restorations_result.restorations()) { - if (!flags.Test(ListSnapshotsFlag::NOT_SHOW_RESTORED) || - restoration.entry().state() != master::SysSnapshotEntryPB::RESTORED) { - if (!title_printed) { - std::cout << RightPadToUuidWidth("Restoration UUID") << kColumnSep << "State" << std::endl; - title_printed = true; - } - std::cout << TryFullyDecodeTxnSnapshotRestorationId(restoration.id()) << kColumnSep - << master::SysSnapshotEntryPB::State_Name(restoration.entry().state()) << std::endl; - } - } - - return Status::OK(); -} - -Result ListSnapshotRestorations( - ClusterAdminClientClass* client, const TxnSnapshotRestorationId& restoration_id) { - auto resp = VERIFY_RESULT(client->ListSnapshotRestorations(restoration_id)); - rapidjson::Document result; - result.SetObject(); - rapidjson::Value json_restorations(rapidjson::kArrayType); - for (const auto& restoration : resp.restorations()) { - rapidjson::Value json_restoration(rapidjson::kObjectType); - AddStringField( - "id", VERIFY_RESULT(FullyDecodeTxnSnapshotRestorationId(restoration.id())).ToString(), - &json_restoration, &result.GetAllocator()); - AddStringField( - "snapshot_id", - VERIFY_RESULT(FullyDecodeTxnSnapshotId(restoration.entry().snapshot_id())).ToString(), - &json_restoration, &result.GetAllocator()); - AddStringField( - "state", master::SysSnapshotEntryPB_State_Name(restoration.entry().state()), - &json_restoration, &result.GetAllocator()); - json_restorations.PushBack(json_restoration, result.GetAllocator()); - } - result.AddMember("restorations", json_restorations, result.GetAllocator()); - return result; -} - -} // namespace - -void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClientClass* client) { - super::RegisterCommandHandlers(client); - - std::string options = ""; - for (auto flag : ListSnapshotsFlagList()) { - options += Format(" [$0]", flag); - } - Register( - "list_snapshots", std::move(options), - [client](const CLIArguments& args) -> Status { - EnumBitSet flags; - - for (size_t i = 0; i < args.size(); ++i) { - std::string uppercase_flag; - ToUpperCase(args[i], &uppercase_flag); - - bool found = false; - for (auto flag : ListSnapshotsFlagList()) { - if (uppercase_flag == ToString(flag)) { - flags.Set(flag); - found = true; - break; - } - } - if (!found) { - return STATUS_FORMAT(InvalidArgument, "Wrong flag: $0", args[i]); - } - } - - RETURN_NOT_OK_PREPEND(ListSnapshots(client, flags), "Unable to list snapshots"); - return Status::OK(); - }); - - Register( - "create_snapshot", - " " - " [
    ]..." - " [] (default 60, set 0 to skip flushing)", - [client](const CLIArguments& args) -> Status { - int timeout_secs = 60; - const auto tables = VERIFY_RESULT(ResolveTableNames( - client, args.begin(), args.end(), - [&timeout_secs](auto i, const auto& end) -> Status { - if (std::next(i) == end) { - timeout_secs = VERIFY_RESULT(CheckedStoi(*i)); - return Status::OK(); - } - return ClusterAdminCli::kInvalidArguments; - })); - - for (auto table : tables) { - if (table.is_cql_namespace() && table.is_system()) { - return STATUS(InvalidArgument, - "Cannot create snapshot of YCQL system table", - table.table_name()); - } - } - - RETURN_NOT_OK_PREPEND(client->CreateSnapshot(tables, true, timeout_secs), - Substitute("Unable to create snapshot of tables: $0", - yb::ToString(tables))); - return Status::OK(); - }); - - RegisterJson( - "list_snapshot_restorations", - " []", - [client](const CLIArguments& args) -> Result { - auto restoration_id = VERIFY_RESULT(GetOptionalArg(args, 0)); - return ListSnapshotRestorations(client, restoration_id); - }); - - RegisterJson( - "create_snapshot_schedule", - " " - " " - " ", - [client](const CLIArguments& args) -> Result { - RETURN_NOT_OK(CheckArgumentsCount(args.size(), 3, 3)); - auto interval = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[0]))); - auto retention = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[1]))); - const auto tables = VERIFY_RESULT(ResolveTableNames( - client, args.begin() + 2, args.end(), TailArgumentsProcessor(), true)); - // This is just a paranoid check, should never happen. - if (tables.size() != 1 || !tables[0].has_namespace()) { - return STATUS(InvalidArgument, "Expecting exactly one keyspace argument"); - } - if (tables[0].namespace_type() != YQL_DATABASE_CQL && - tables[0].namespace_type() != YQL_DATABASE_PGSQL) { - return STATUS( - InvalidArgument, "Snapshot schedule can only be setup on YCQL or YSQL namespace"); - } - return client->CreateSnapshotSchedule(tables[0], interval, retention); - }); - - RegisterJson( - "list_snapshot_schedules", - " []", - [client](const CLIArguments& args) -> Result { - RETURN_NOT_OK(CheckArgumentsCount(args.size(), 0, 1)); - auto schedule_id = VERIFY_RESULT(GetOptionalArg(args, 0)); - return client->ListSnapshotSchedules(schedule_id); - }); - - RegisterJson( - "delete_snapshot_schedule", - " ", - [client](const CLIArguments& args) -> Result { - RETURN_NOT_OK(CheckArgumentsCount(args.size(), 1, 1)); - auto schedule_id = VERIFY_RESULT(SnapshotScheduleId::FromString(args[0])); - return client->DeleteSnapshotSchedule(schedule_id); - }); - - RegisterJson( - "restore_snapshot_schedule", - Format(" ( | $0 )", kMinus), - [client](const CLIArguments& args) -> Result { - RETURN_NOT_OK(CheckArgumentsCount(args.size(), 2, 3)); - auto schedule_id = VERIFY_RESULT(SnapshotScheduleId::FromString(args[0])); - HybridTime restore_at; - if (args.size() == 2) { - restore_at = VERIFY_RESULT(HybridTime::ParseHybridTime(args[1])); - } else { - if (args[1] != kMinus) { - return ClusterAdminCli::kInvalidArguments; - } - restore_at = VERIFY_RESULT(HybridTime::ParseHybridTime("-" + args[2])); - } - - return client->RestoreSnapshotSchedule(schedule_id, restore_at); - }); - - RegisterJson( - "edit_snapshot_schedule", - " (interval | retention " - "){1,2}", - [client](const CLIArguments& args) -> Result { - if (args.size() != 3 && args.size() != 5) { - return STATUS(InvalidArgument, - Format("Expected 3 or 5 arguments, received $0", args.size())); - } - auto schedule_id = VERIFY_RESULT(SnapshotScheduleId::FromString(args[0])); - std::optional new_interval; - std::optional new_retention; - for (size_t i = 1; i + 1 < args.size(); i += 2) { - if (args[i] == "interval") { - if (new_interval) { - return STATUS(InvalidArgument, "Repeated interval"); - } - new_interval = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[i + 1]))); - } else if (args[i] == "retention") { - if (new_retention) { - return STATUS(InvalidArgument, "Repeated retention"); - } - new_retention = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[i + 1]))); - } else { - return STATUS( - InvalidArgument, - Format("Expected either \"retention\" or \"interval\", got: $0", args[i])); - } - } - return client->EditSnapshotSchedule(schedule_id, new_interval, new_retention); - }); - - Register( - "create_keyspace_snapshot", " [ycql.]", - [client](const CLIArguments& args) -> Status { - if (args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - - const TypedNamespaceName keyspace = VERIFY_RESULT(ParseNamespaceName(args[0])); - SCHECK_NE( - keyspace.db_type, YQL_DATABASE_PGSQL, InvalidArgument, - Format("Wrong keyspace type: $0", YQLDatabase_Name(keyspace.db_type))); - - RETURN_NOT_OK_PREPEND(client->CreateNamespaceSnapshot(keyspace), - Substitute("Unable to create snapshot of keyspace: $0", - keyspace.name)); - return Status::OK(); - }); - - Register( - "create_database_snapshot", " [ysql.]", - [client](const CLIArguments& args) -> Status { - if (args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - - const TypedNamespaceName database = - VERIFY_RESULT(ParseNamespaceName(args[0], YQL_DATABASE_PGSQL)); - SCHECK_EQ( - database.db_type, YQL_DATABASE_PGSQL, InvalidArgument, - Format("Wrong database type: $0", YQLDatabase_Name(database.db_type))); - - RETURN_NOT_OK_PREPEND(client->CreateNamespaceSnapshot(database), - Substitute("Unable to create snapshot of database: $0", - database.name)); - return Status::OK(); - }); - - Register( - "restore_snapshot", Format(" [ | $0 ]", kMinus), - [client](const CLIArguments& args) -> Status { - if (args.size() < 1 || 3 < args.size()) { - return ClusterAdminCli::kInvalidArguments; - } else if (args.size() == 3 && args[1] != kMinus) { - return ClusterAdminCli::kInvalidArguments; - } - const string snapshot_id = args[0]; - HybridTime timestamp; - if (args.size() == 2) { - timestamp = VERIFY_RESULT(HybridTime::ParseHybridTime(args[1])); - } else if (args.size() == 3) { - timestamp = VERIFY_RESULT(HybridTime::ParseHybridTime("-" + args[2])); - } - - RETURN_NOT_OK_PREPEND(client->RestoreSnapshot(snapshot_id, timestamp), - Substitute("Unable to restore snapshot $0", snapshot_id)); - return Status::OK(); - }); - - Register( - "export_snapshot", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 2) { - return ClusterAdminCli::kInvalidArguments; - } - - const string snapshot_id = args[0]; - const string file_name = args[1]; - RETURN_NOT_OK_PREPEND(client->CreateSnapshotMetaFile(snapshot_id, file_name), - Substitute("Unable to export snapshot $0 to file $1", - snapshot_id, - file_name)); - return Status::OK(); - }); - - Register( - "import_snapshot", " [ []...]", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - - const string file_name = args[0]; - TypedNamespaceName keyspace; - size_t num_tables = 0; - vector tables; - - if (args.size() >= 2) { - keyspace = VERIFY_RESULT(ParseNamespaceName(args[1])); - num_tables = args.size() - 2; - - if (num_tables > 0) { - LOG_IF(DFATAL, keyspace.name.empty()) << "Uninitialized keyspace: " << keyspace.name; - tables.reserve(num_tables); - - for (size_t i = 0; i < num_tables; ++i) { - tables.push_back(YBTableName(keyspace.db_type, keyspace.name, args[2 + i])); - } - } - } - - const string msg = num_tables > 0 ? - Substitute("Unable to import tables $0 from snapshot meta file $1", - yb::ToString(tables), file_name) : - Substitute("Unable to import snapshot meta file $0", file_name); - - RETURN_NOT_OK_PREPEND(client->ImportSnapshotMetaFile(file_name, keyspace, tables), msg); - return Status::OK(); - }); - - Register( - "delete_snapshot", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - - const string snapshot_id = args[0]; - RETURN_NOT_OK_PREPEND(client->DeleteSnapshot(snapshot_id), - Substitute("Unable to delete snapshot $0", snapshot_id)); - return Status::OK(); - }); - - Register( - "list_replica_type_counts", - "
    ", - [client](const CLIArguments& args) -> Status { - const auto table_name = VERIFY_RESULT( - ResolveSingleTableName(client, args.begin(), args.end())); - RETURN_NOT_OK_PREPEND(client->ListReplicaTypeCounts(table_name), - "Unable to list live and read-only replica counts"); - return Status::OK(); - }); - - Register( - "set_preferred_zones", - " [:] [[:]]...", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - RETURN_NOT_OK_PREPEND(client->SetPreferredZones(args), "Unable to set preferred zones"); - return Status::OK(); - }); - - Register( - "rotate_universe_key", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - RETURN_NOT_OK_PREPEND( - client->RotateUniverseKey(args[0]), "Unable to rotate universe key."); - return Status::OK(); - }); - - Register( - "disable_encryption", "", - [client](const CLIArguments& args) -> Status { - RETURN_NOT_OK_PREPEND(client->DisableEncryption(), "Unable to disable encryption."); - return Status::OK(); - }); - - Register( - "is_encryption_enabled", "", - [client](const CLIArguments& args) -> Status { - RETURN_NOT_OK_PREPEND(client->IsEncryptionEnabled(), "Unable to get encryption status."); - return Status::OK(); - }); - - Register( - "add_universe_key_to_all_masters", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 2) { - return ClusterAdminCli::kInvalidArguments; - } - string key_id = args[0]; - faststring contents; - RETURN_NOT_OK(ReadFileToString(Env::Default(), args[1], &contents)); - string universe_key = contents.ToString(); - - RETURN_NOT_OK_PREPEND(client->AddUniverseKeyToAllMasters(key_id, universe_key), - "Unable to add universe key to all masters."); - return Status::OK(); - }); - - Register( - "all_masters_have_universe_key_in_memory", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - RETURN_NOT_OK_PREPEND(client->AllMastersHaveUniverseKeyInMemory(args[0]), - "Unable to check whether master has universe key in memory."); - return Status::OK(); - }); - - Register( - "rotate_universe_key_in_memory", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - string key_id = args[0]; - - RETURN_NOT_OK_PREPEND(client->RotateUniverseKeyInMemory(key_id), - "Unable rotate universe key in memory."); - return Status::OK(); - }); - - Register( - "disable_encryption_in_memory", "", - [client](const CLIArguments& args) -> Status { - if (args.size() != 0) { - return ClusterAdminCli::kInvalidArguments; - } - RETURN_NOT_OK_PREPEND(client->DisableEncryptionInMemory(), "Unable to disable encryption."); - return Status::OK(); - }); - - Register( - "write_universe_key_to_file", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 2) { - return ClusterAdminCli::kInvalidArguments; - } - RETURN_NOT_OK_PREPEND(client->WriteUniverseKeyToFile(args[0], args[1]), - "Unable to write key to file"); - return Status::OK(); - }); - - Register( - "create_cdc_stream", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - const string table_id = args[0]; - RETURN_NOT_OK_PREPEND(client->CreateCDCStream(table_id), - Substitute("Unable to create CDC stream for table $0", table_id)); - return Status::OK(); - }); - - Register( - "create_change_data_stream", " [] []", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - - std::string checkpoint_type = yb::ToString("IMPLICIT"); - std::string record_type = yb::ToString("CHANGE"); - std::string uppercase_checkpoint_type; - std::string uppercase_record_type; - - if (args.size() > 1) { - ToUpperCase(args[1], &uppercase_checkpoint_type); - if (uppercase_checkpoint_type != yb::ToString("EXPLICIT") && - uppercase_checkpoint_type != yb::ToString("IMPLICIT")) { - return ClusterAdminCli::kInvalidArguments; - } - checkpoint_type = uppercase_checkpoint_type; - } - - if (args.size() > 2) { - ToUpperCase(args[2], &uppercase_record_type); - if (uppercase_record_type != yb::ToString("ALL") && - uppercase_record_type != yb::ToString("CHANGE")) { - return ClusterAdminCli::kInvalidArguments; - } - record_type = uppercase_record_type; - } - - const string namespace_name = args[0]; - - const TypedNamespaceName database = - VERIFY_RESULT(ParseNamespaceName(args[0], YQL_DATABASE_PGSQL)); - SCHECK_EQ( - database.db_type, YQL_DATABASE_PGSQL, InvalidArgument, - Format("Wrong database type: $0", YQLDatabase_Name(database.db_type))); - - RETURN_NOT_OK_PREPEND( - client->CreateCDCSDKDBStream(database, checkpoint_type, record_type), - Substitute("Unable to create CDC stream for database $0", namespace_name)); - return Status::OK(); - }); - - Register( - "delete_cdc_stream", " [force_delete]", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - const string stream_id = args[0]; - bool force_delete = false; - if (args.size() >= 2 && args[1] == "force_delete") { - force_delete = true; - } - RETURN_NOT_OK_PREPEND(client->DeleteCDCStream(stream_id, force_delete), - Substitute("Unable to delete CDC stream id $0", stream_id)); - return Status::OK(); - }); - - Register( - "delete_change_data_stream", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - - const std::string db_stream_id = args[0]; - RETURN_NOT_OK_PREPEND(client->DeleteCDCSDKDBStream(db_stream_id), - Substitute("Unable to delete CDC database stream id $0", - db_stream_id)); - return Status::OK(); - }); - - Register( - "list_cdc_streams", " []", - [client](const CLIArguments& args) -> Status { - if (args.size() != 0 && args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - const string table_id = (args.size() == 1 ? args[0] : ""); - RETURN_NOT_OK_PREPEND(client->ListCDCStreams(table_id), - Substitute("Unable to list CDC streams for table $0", table_id)); - return Status::OK(); - }); - - Register( - "list_change_data_streams", " []", - [client](const CLIArguments& args) -> Status { - if (args.size() != 0 && args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - const string namespace_name = args.size() == 1 ? args[0] : ""; - string msg = (args.size() == 1) - ? Substitute("Unable to list CDC streams for namespace $0", namespace_name) - : "Unable to list CDC streams"; - - RETURN_NOT_OK_PREPEND(client->ListCDCSDKStreams(namespace_name), msg); - return Status::OK(); - }); - - Register( - "get_change_data_stream_info", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 0 && args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - const string db_stream_id = args.size() == 1 ? args[0] : ""; - RETURN_NOT_OK_PREPEND(client->GetCDCDBStreamInfo(db_stream_id), - Substitute("Unable to list CDC stream info for database stream $0", - db_stream_id)); - return Status::OK(); - }); - - Register( - "setup_universe_replication", - " " - " []" , - [client](const CLIArguments& args) -> Status { - if (args.size() < 3) { - return ClusterAdminCli::kInvalidArguments; - } - const string producer_uuid = args[0]; - - vector producer_addresses; - boost::split(producer_addresses, args[1], boost::is_any_of(",")); - - vector table_uuids; - boost::split(table_uuids, args[2], boost::is_any_of(",")); - - vector producer_bootstrap_ids; - if (args.size() == 4) { - boost::split(producer_bootstrap_ids, args[3], boost::is_any_of(",")); - } - - RETURN_NOT_OK_PREPEND(client->SetupUniverseReplication(producer_uuid, - producer_addresses, - table_uuids, - producer_bootstrap_ids), - Substitute("Unable to setup replication from universe $0", - producer_uuid)); - return Status::OK(); - }); - - Register( - "delete_universe_replication", " [ignore-errors]", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - const string producer_id = args[0]; - bool ignore_errors = false; - if (args.size() >= 2 && args[1] == "ignore-errors") { - ignore_errors = true; - } - RETURN_NOT_OK_PREPEND(client->DeleteUniverseReplication(producer_id, ignore_errors), - Substitute("Unable to delete replication for universe $0", - producer_id)); - return Status::OK(); - }); - - Register( - "alter_universe_replication", - " " - " (set_master_addresses [] |" - " add_table []" - " [] |" - " remove_table [] [ignore-errors] |" - " rename_id )", - [client](const CLIArguments& args) -> Status { - if (args.size() < 3 || args.size() > 4) { - return ClusterAdminCli::kInvalidArguments; - } - if (args.size() == 4 && args[1] != "add_table" && args[1] != "remove_table") { - return ClusterAdminCli::kInvalidArguments; - } - - const string producer_uuid = args[0]; - vector master_addresses; - vector add_tables; - vector remove_tables; - vector bootstrap_ids_to_add; - string new_producer_universe_id = ""; - bool remove_table_ignore_errors = false; - - vector newElem, *lst; - if (args[1] == "set_master_addresses") { - lst = &master_addresses; - } else if (args[1] == "add_table") { - lst = &add_tables; - } else if (args[1] == "remove_table") { - lst = &remove_tables; - if (args.size() == 4 && args[3] == "ignore-errors") { - remove_table_ignore_errors = true; - } - } else if (args[1] == "rename_id") { - lst = nullptr; - new_producer_universe_id = args[2]; - } else { - return ClusterAdminCli::kInvalidArguments; - } - - if (lst) { - boost::split(newElem, args[2], boost::is_any_of(",")); - lst->insert(lst->end(), newElem.begin(), newElem.end()); - - if (args[1] == "add_table" && args.size() == 4) { - boost::split(bootstrap_ids_to_add, args[3], boost::is_any_of(",")); - } - } - - RETURN_NOT_OK_PREPEND(client->AlterUniverseReplication(producer_uuid, - master_addresses, - add_tables, - remove_tables, - bootstrap_ids_to_add, - new_producer_universe_id, - remove_table_ignore_errors), - Substitute("Unable to alter replication for universe $0", producer_uuid)); - - return Status::OK(); - }); - - Register( - "change_xcluster_role", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - auto xcluster_role = args[0]; - if (xcluster_role == "STANDBY") { - return client->ChangeXClusterRole(cdc::XClusterRole::STANDBY); - } - if (xcluster_role == "ACTIVE") { - return client->ChangeXClusterRole(cdc::XClusterRole::ACTIVE); - } - return STATUS(InvalidArgument, - Format("Expected one of STANDBY OR ACTIVE, found $0", args[0])); - }); - - - Register( - "set_universe_replication_enabled", " (0|1)", - [client](const CLIArguments& args) -> Status { - if (args.size() < 2) { - return ClusterAdminCli::kInvalidArguments; - } - const string producer_id = args[0]; - const bool is_enabled = VERIFY_RESULT(CheckedStoi(args[1])) != 0; - RETURN_NOT_OK_PREPEND(client->SetUniverseReplicationEnabled(producer_id, is_enabled), - Substitute("Unable to $0 replication for universe $1", - is_enabled ? "enable" : "disable", - producer_id)); - return Status::OK(); - }); - - Register( - "bootstrap_cdc_producer", " ", - [client](const CLIArguments& args) -> Status { - if (args.size() < 1) { - return ClusterAdminCli::kInvalidArguments; - } - - vector table_ids; - boost::split(table_ids, args[0], boost::is_any_of(",")); - - RETURN_NOT_OK_PREPEND(client->BootstrapProducer(table_ids), - "Unable to bootstrap CDC producer"); - return Status::OK(); - }); - - Register( - "wait_for_replication_drain", - Format(" " - " [ | $0 ]", kMinus), - [client](const CLIArguments& args) -> Status { - RETURN_NOT_OK(CheckArgumentsCount(args.size(), 1, 3)); - vector stream_ids; - boost::split(stream_ids, args[0], boost::is_any_of(",")); - string target_time; - if (args.size() == 2) { - target_time = args[1]; - } else if (args.size() == 3) { - if (args[1] != kMinus) { - return ClusterAdminCli::kInvalidArguments; - } - target_time = "-" + args[2]; - } - - return client->WaitForReplicationDrain(stream_ids, target_time); - }); - - Register( - "setup_namespace_universe_replication", - " ", - [client](const CLIArguments& args) -> Status { - RETURN_NOT_OK(CheckArgumentsCount(args.size(), 3, 3)); - const string producer_uuid = args[0]; - vector producer_addresses; - boost::split(producer_addresses, args[1], boost::is_any_of(",")); - TypedNamespaceName producer_namespace = VERIFY_RESULT(ParseNamespaceName(args[2])); - - RETURN_NOT_OK_PREPEND( - client->SetupNSUniverseReplication( - producer_uuid, producer_addresses, producer_namespace), - Substitute("Unable to setup namespace replication from universe $0", producer_uuid)); - return Status::OK(); - }); - - Register( - "get_replication_status", " []", - [client](const CLIArguments& args) -> Status { - if (args.size() != 0 && args.size() != 1) { - return ClusterAdminCli::kInvalidArguments; - } - const string producer_universe_uuid = args.size() == 1 ? args[0] : ""; - RETURN_NOT_OK_PREPEND(client->GetReplicationInfo(producer_universe_uuid), - "Unable to get replication status"); - return Status::OK(); - }); - - RegisterJson( - "get_xcluster_estimated_data_loss", "", - [client](const CLIArguments& args) -> Result { - return client->GetXClusterEstimatedDataLoss(); - }); - - RegisterJson( - "get_xcluster_safe_time", "", - [client](const CLIArguments& args) -> Result { - return client->GetXClusterSafeTime(); - }); -} // NOLINT -- a long function but that is OK - -} // namespace enterprise -} // namespace tools -} // namespace yb diff --git a/ent/src/yb/tools/yb-admin_client.h b/ent/src/yb/tools/yb-admin_client.h deleted file mode 100644 index 4709a1dbd6bd..000000000000 --- a/ent/src/yb/tools/yb-admin_client.h +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) YugaByte, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "../../../../src/yb/tools/yb-admin_client.h" -#include "yb/cdc/cdc_service.pb.h" -#include "yb/common/snapshot.h" -#include "yb/master/master_backup.pb.h" -#include "yb/rpc/secure_stream.h" -#include "yb/server/secure.h" -#include "yb/util/env_util.h" -#include "yb/util/path_util.h" - -namespace yb { -namespace tools { -namespace enterprise { - -// Flags for list_snapshot command. -YB_DEFINE_ENUM(ListSnapshotsFlag, (SHOW_DETAILS)(NOT_SHOW_RESTORED)(SHOW_DELETED)(JSON)); -using ListSnapshotsFlags = EnumBitSet; - -// Constants for disabling tablet splitting during PITR restores. -static constexpr double kPitrSplitDisableDurationSecs = 600; -static constexpr double kPitrSplitDisableCheckFreqMs = 500; - -class ClusterAdminClient : public yb::tools::ClusterAdminClient { - typedef yb::tools::ClusterAdminClient super; - public: - ClusterAdminClient(std::string addrs, MonoDelta timeout) - : super(std::move(addrs), timeout) {} - - ClusterAdminClient(const HostPort& init_master_addrs, MonoDelta timeout) - : super(init_master_addrs, timeout) {} - - // Snapshot operations. - Result ListSnapshots(const ListSnapshotsFlags& flags); - Status CreateSnapshot(const std::vector& tables, - const bool add_indexes = true, - const int flush_timeout_secs = 0); - Status CreateNamespaceSnapshot(const TypedNamespaceName& ns); - Result ListSnapshotRestorations( - const TxnSnapshotRestorationId& restoration_id); - Result CreateSnapshotSchedule(const client::YBTableName& keyspace, - MonoDelta interval, MonoDelta retention); - Result ListSnapshotSchedules(const SnapshotScheduleId& schedule_id); - Result DeleteSnapshotSchedule(const SnapshotScheduleId& schedule_id); - Result RestoreSnapshotSchedule( - const SnapshotScheduleId& schedule_id, HybridTime restore_at); - Status RestoreSnapshot(const std::string& snapshot_id, HybridTime timestamp); - - Result EditSnapshotSchedule( - const SnapshotScheduleId& schedule_id, - std::optional new_interval, - std::optional new_retention); - - Status DeleteSnapshot(const std::string& snapshot_id); - - Status CreateSnapshotMetaFile(const std::string& snapshot_id, - const std::string& file_name); - Status ImportSnapshotMetaFile(const std::string& file_name, - const TypedNamespaceName& keyspace, - const std::vector& tables); - Status ListReplicaTypeCounts(const client::YBTableName& table_name); - - Status SetPreferredZones(const std::vector& preferred_zones); - - Status RotateUniverseKey(const std::string& key_path); - - Status DisableEncryption(); - - Status IsEncryptionEnabled(); - - Status AddUniverseKeyToAllMasters( - const std::string& key_id, const std::string& universe_key); - - Status AllMastersHaveUniverseKeyInMemory(const std::string& key_id); - - Status RotateUniverseKeyInMemory(const std::string& key_id); - - Status DisableEncryptionInMemory(); - - Status WriteUniverseKeyToFile(const std::string& key_id, const std::string& file_name); - - Status CreateCDCStream(const TableId& table_id); - - Status CreateCDCSDKDBStream( - const TypedNamespaceName& ns, const std::string& CheckPointType, - const std::string& RecordType); - - Status DeleteCDCStream(const std::string& stream_id, bool force_delete = false); - - Status DeleteCDCSDKDBStream(const std::string& db_stream_id); - - Status ListCDCStreams(const TableId& table_id); - - Status ListCDCSDKStreams(const std::string& namespace_name); - - Status GetCDCDBStreamInfo(const std::string& db_stream_id); - - Status SetupUniverseReplication(const std::string& producer_uuid, - const std::vector& producer_addresses, - const std::vector& tables, - const std::vector& producer_bootstrap_ids); - - Status DeleteUniverseReplication(const std::string& producer_id, - bool ignore_errors = false); - - Status AlterUniverseReplication( - const std::string& producer_uuid, - const std::vector& producer_addresses, - const std::vector& add_tables, - const std::vector& remove_tables, - const std::vector& producer_bootstrap_ids_to_add, - const std::string& new_producer_universe_id, - bool remove_table_ignore_errors = false); - - Status RenameUniverseReplication(const std::string& old_universe_name, - const std::string& new_universe_name); - - Status WaitForSetupUniverseReplicationToFinish(const std::string& producer_uuid); - - Status ChangeXClusterRole(cdc::XClusterRole role); - - Status SetUniverseReplicationEnabled(const std::string& producer_id, - bool is_enabled); - - Status BootstrapProducer(const std::vector& table_id); - - Status WaitForReplicationDrain(const std::vector& stream_ids, - const std::string& target_time); - - Status SetupNSUniverseReplication(const std::string& producer_uuid, - const std::vector& producer_addresses, - const TypedNamespaceName& producer_namespace); - - Status GetReplicationInfo(const std::string& universe_uuid); - - Result GetXClusterEstimatedDataLoss(); - - Result GetXClusterSafeTime(); - - private: - Result SuitableSnapshotId( - const SnapshotScheduleId& schedule_id, HybridTime restore_at, CoarseTimePoint deadline); - - Status SendEncryptionRequest(const std::string& key_path, bool enable_encryption); - - Result GetFirstRpcAddressForTS(); - - void CleanupEnvironmentOnSetupUniverseReplicationFailure( - const std::string& producer_uuid, const Status& failure_status); - - Status DisableTabletSplitsDuringRestore(CoarseTimePoint deadline); - - Result RestoreSnapshotScheduleDeprecated( - const SnapshotScheduleId& schedule_id, HybridTime restore_at); - - std::string GetDBTypeName(const master::SysNamespaceEntryPB& pb); - // Map: Old name -> New name. - typedef std::unordered_map NSNameToNameMap; - Status UpdateUDTypes( - QLTypePB* pb_type, bool* update_meta, const NSNameToNameMap& ns_name_to_name); - - DISALLOW_COPY_AND_ASSIGN(ClusterAdminClient); -}; - -} // namespace enterprise -} // namespace tools -} // namespace yb diff --git a/ent/src/yb/tools/yb-admin_client_ent.cc b/ent/src/yb/tools/yb-admin_client_ent.cc deleted file mode 100644 index d5d3a78a5d63..000000000000 --- a/ent/src/yb/tools/yb-admin_client_ent.cc +++ /dev/null @@ -1,1859 +0,0 @@ -// Copyright (c) YugaByte, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations -// under the License. - -#include - -#include - -#include - -#include "yb/cdc/cdc_service.h" -#include "yb/cdc/cdc_service.proxy.h" -#include "yb/client/client.h" - -#include "yb/common/constants.h" -#include "yb/common/entity_ids.h" -#include "yb/common/json_util.h" -#include "yb/common/ql_type_util.h" -#include "yb/common/wire_protocol.h" - -#include "yb/encryption/encryption_util.h" - -#include "yb/gutil/casts.h" -#include "yb/gutil/strings/util.h" -#include "yb/gutil/strings/split.h" - -#include "yb/master/master_defaults.h" -#include "yb/master/master_error.h" -#include "yb/master/master_backup.proxy.h" -#include "yb/master/master_client.proxy.h" -#include "yb/master/master_cluster.proxy.h" -#include "yb/master/master_ddl.proxy.h" -#include "yb/master/master_encryption.proxy.h" -#include "yb/master/master_replication.proxy.h" -#include "yb/master/master_admin.pb.h" -#include "yb/master/master_admin.proxy.h" -#include "yb/master/master_util.h" - -#include "yb/tools/yb-admin_cli.h" -#include "yb/tools/yb-admin_client.h" - -#include "yb/rpc/messenger.h" -#include "yb/rpc/rpc_controller.h" -#include "yb/tools/yb-admin_util.h" -#include "yb/util/cast.h" -#include "yb/util/env.h" -#include "yb/util/flags.h" -#include "yb/util/jsonwriter.h" -#include "yb/util/monotime.h" -#include "yb/util/pb_util.h" -#include "yb/util/physical_time.h" -#include "yb/util/protobuf_util.h" -#include "yb/util/stol_utils.h" -#include "yb/util/string_case.h" -#include "yb/util/string_trim.h" -#include "yb/util/string_util.h" -#include "yb/util/timestamp.h" -#include "yb/util/format.h" -#include "yb/util/status_format.h" -#include "yb/util/test_util.h" - -DEFINE_test_flag(int32, metadata_file_format_version, 0, - "Used in 'export_snapshot' metadata file format (0 means using latest format)."); - -DECLARE_bool(use_client_to_server_encryption); -DECLARE_int32(yb_client_admin_operation_timeout_sec); - -namespace yb { -namespace tools { -namespace enterprise { - -using namespace std::literals; - -using std::cout; -using std::endl; -using std::pair; -using std::string; -using std::vector; -using std::unordered_map; - -using google::protobuf::RepeatedPtrField; - -using client::YBTableName; -using pb_util::ParseFromSlice; -using rpc::RpcController; - -using master::ChangeEncryptionInfoRequestPB; -using master::ChangeEncryptionInfoResponsePB; -using master::CreateSnapshotRequestPB; -using master::CreateSnapshotResponsePB; -using master::DeleteSnapshotRequestPB; -using master::DeleteSnapshotResponsePB; -using master::IdPairPB; -using master::ImportSnapshotMetaRequestPB; -using master::ImportSnapshotMetaResponsePB; -using master::ImportSnapshotMetaResponsePB_TableMetaPB; -using master::ListSnapshotRestorationsRequestPB; -using master::ListSnapshotRestorationsResponsePB; -using master::ListSnapshotsRequestPB; -using master::ListSnapshotsResponsePB; -using master::ListTablesRequestPB; -using master::ListTablesResponsePB; -using master::ListTabletServersResponsePB; -using master::RestoreSnapshotRequestPB; -using master::RestoreSnapshotResponsePB; -using master::SnapshotInfoPB; -using master::SysNamespaceEntryPB; -using master::SysRowEntry; -using master::SysRowEntryType; -using master::BackupRowEntryPB; -using master::SysTablesEntryPB; -using master::SysSnapshotEntryPB; -using master::SysUDTypeEntryPB; - -PB_ENUM_FORMATTERS(yb::master::SysSnapshotEntryPB::State); - -namespace { - -template -Result SnapshotScheduleInfoToJson( - const master::SnapshotScheduleInfoPB& schedule, Allocator* allocator) { - rapidjson::Value json_schedule(rapidjson::kObjectType); - AddStringField( - "id", VERIFY_RESULT(FullyDecodeSnapshotScheduleId(schedule.id())).ToString(), &json_schedule, - allocator); - - const auto& filter = schedule.options().filter(); - string filter_output; - // The user input should only have 1 entry, at namespace level. - if (filter.tables().tables_size() == 1) { - const auto& table_id = filter.tables().tables(0); - if (table_id.has_namespace_()) { - string database_type; - if (table_id.namespace_().database_type() == YQL_DATABASE_PGSQL) { - database_type = "ysql"; - } else if (table_id.namespace_().database_type() == YQL_DATABASE_CQL) { - database_type = "ycql"; - } - if (!database_type.empty()) { - filter_output = Format("$0.$1", database_type, table_id.namespace_().name()); - } - } - } - // If the user input was non standard, just display the whole debug PB. - if (filter_output.empty()) { - filter_output = filter.ShortDebugString(); - DCHECK(false) << "Non standard filter " << filter_output; - } - rapidjson::Value options(rapidjson::kObjectType); - AddStringField("filter", filter_output, &options, allocator); - auto interval_min = schedule.options().interval_sec() / MonoTime::kSecondsPerMinute; - AddStringField("interval", Format("$0 min", interval_min), &options, allocator); - auto retention_min = schedule.options().retention_duration_sec() / MonoTime::kSecondsPerMinute; - AddStringField("retention", Format("$0 min", retention_min), &options, allocator); - auto delete_time = HybridTime::FromPB(schedule.options().delete_time()); - if (delete_time) { - AddStringField("delete_time", HybridTimeToString(delete_time), &options, allocator); - } - - json_schedule.AddMember("options", options, *allocator); - rapidjson::Value json_snapshots(rapidjson::kArrayType); - for (const auto& snapshot : schedule.snapshots()) { - rapidjson::Value json_snapshot(rapidjson::kObjectType); - AddStringField( - "id", VERIFY_RESULT(FullyDecodeTxnSnapshotId(snapshot.id())).ToString(), &json_snapshot, - allocator); - auto snapshot_ht = HybridTime::FromPB(snapshot.entry().snapshot_hybrid_time()); - AddStringField("snapshot_time", HybridTimeToString(snapshot_ht), &json_snapshot, allocator); - auto previous_snapshot_ht = - HybridTime::FromPB(snapshot.entry().previous_snapshot_hybrid_time()); - if (previous_snapshot_ht) { - AddStringField( - "previous_snapshot_time", HybridTimeToString(previous_snapshot_ht), &json_snapshot, - allocator); - } - json_snapshots.PushBack(json_snapshot, *allocator); - } - json_schedule.AddMember("snapshots", json_snapshots, *allocator); - return json_schedule; -} - -} // namespace - -Result ClusterAdminClient::ListSnapshots(const ListSnapshotsFlags& flags) { - ListSnapshotsResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - ListSnapshotsRequestPB req; - req.set_list_deleted_snapshots(flags.Test(ListSnapshotsFlag::SHOW_DELETED)); - auto* flags_pb = req.mutable_detail_options(); - // Explicitly set all boolean fields as the defaults of this proto are inconsistent. - if (flags.Test(ListSnapshotsFlag::SHOW_DETAILS)) { - flags_pb->set_show_namespace_details(true); - flags_pb->set_show_udtype_details(true); - flags_pb->set_show_table_details(true); - flags_pb->set_show_tablet_details(false); - } else { - flags_pb->set_show_namespace_details(false); - flags_pb->set_show_udtype_details(false); - flags_pb->set_show_table_details(false); - flags_pb->set_show_tablet_details(false); - } - return master_backup_proxy_->ListSnapshots(req, &resp, rpc); - })); - return resp; -} - -Status ClusterAdminClient::CreateSnapshot( - const vector& tables, - const bool add_indexes, - const int flush_timeout_secs) { - if (flush_timeout_secs > 0) { - const auto status = FlushTables(tables, add_indexes, flush_timeout_secs, false); - if (status.IsTimedOut()) { - cout << status.ToString(false) << " (ignored)" << endl; - } else if (!status.ok() && !status.IsNotFound()) { - return status; - } - } - - CreateSnapshotResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - CreateSnapshotRequestPB req; - for (const YBTableName& table_name : tables) { - table_name.SetIntoTableIdentifierPB(req.add_tables()); - } - - req.set_add_indexes(add_indexes); - req.set_add_ud_types(true); // No-op for YSQL. - req.set_transaction_aware(true); - return master_backup_proxy_->CreateSnapshot(req, &resp, rpc); - })); - - cout << "Started snapshot creation: " << SnapshotIdToString(resp.snapshot_id()) << endl; - return Status::OK(); -} - -Status ClusterAdminClient::CreateNamespaceSnapshot(const TypedNamespaceName& ns) { - ListTablesResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - ListTablesRequestPB req; - - req.mutable_namespace_()->set_name(ns.name); - req.mutable_namespace_()->set_database_type(ns.db_type); - req.set_exclude_system_tables(true); - req.add_relation_type_filter(master::USER_TABLE_RELATION); - req.add_relation_type_filter(master::INDEX_TABLE_RELATION); - req.add_relation_type_filter(master::MATVIEW_TABLE_RELATION); - return master_ddl_proxy_->ListTables(req, &resp, rpc); - })); - - if (resp.tables_size() == 0) { - return STATUS_FORMAT(InvalidArgument, "No tables found in namespace: $0", ns.name); - } - - vector tables(resp.tables_size()); - for (int i = 0; i < resp.tables_size(); ++i) { - const auto& table = resp.tables(i); - tables[i].set_table_id(table.id()); - tables[i].set_namespace_id(table.namespace_().id()); - tables[i].set_pgschema_name(table.pgschema_name()); - - RSTATUS_DCHECK(table.relation_type() == master::USER_TABLE_RELATION || - table.relation_type() == master::INDEX_TABLE_RELATION || - table.relation_type() == master::MATVIEW_TABLE_RELATION, InternalError, - Format("Invalid relation type: $0", table.relation_type())); - RSTATUS_DCHECK_EQ(table.namespace_().name(), ns.name, InternalError, - Format("Invalid namespace name: $0", table.namespace_().name())); - RSTATUS_DCHECK_EQ(table.namespace_().database_type(), ns.db_type, InternalError, - Format("Invalid namespace type: $0", - YQLDatabase_Name(table.namespace_().database_type()))); - } - - return CreateSnapshot(tables, /* add_indexes */ false); -} - -Result ClusterAdminClient::ListSnapshotRestorations( - const TxnSnapshotRestorationId& restoration_id) { - master::ListSnapshotRestorationsResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - master::ListSnapshotRestorationsRequestPB req; - if (restoration_id) { - req.set_restoration_id(restoration_id.data(), restoration_id.size()); - } - return master_backup_proxy_->ListSnapshotRestorations(req, &resp, rpc); - })); - return resp; -} - -Result ClusterAdminClient::CreateSnapshotSchedule( - const client::YBTableName& keyspace, MonoDelta interval, MonoDelta retention) { - master::CreateSnapshotScheduleResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - master::CreateSnapshotScheduleRequestPB req; - - auto& options = *req.mutable_options(); - auto& filter_tables = *options.mutable_filter()->mutable_tables()->mutable_tables(); - keyspace.SetIntoTableIdentifierPB(filter_tables.Add()); - - options.set_interval_sec(interval.ToSeconds()); - options.set_retention_duration_sec(retention.ToSeconds()); - return master_backup_proxy_->CreateSnapshotSchedule(req, &resp, rpc); - })); - - rapidjson::Document document; - document.SetObject(); - - AddStringField( - "schedule_id", - VERIFY_RESULT(FullyDecodeSnapshotScheduleId(resp.snapshot_schedule_id())).ToString(), - &document, &document.GetAllocator()); - return document; -} - -Result ClusterAdminClient::ListSnapshotSchedules( - const SnapshotScheduleId& schedule_id) { - master::ListSnapshotSchedulesResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [this, &resp, &schedule_id](RpcController* rpc) { - master::ListSnapshotSchedulesRequestPB req; - if (schedule_id) { - req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); - } - return master_backup_proxy_->ListSnapshotSchedules(req, &resp, rpc); - })); - - rapidjson::Document result; - result.SetObject(); - rapidjson::Value json_schedules(rapidjson::kArrayType); - for (const auto& schedule : resp.schedules()) { - json_schedules.PushBack( - VERIFY_RESULT(SnapshotScheduleInfoToJson(schedule, &result.GetAllocator())), - result.GetAllocator()); - } - result.AddMember("schedules", json_schedules, result.GetAllocator()); - return result; -} - -Result ClusterAdminClient::DeleteSnapshotSchedule( - const SnapshotScheduleId& schedule_id) { - master::DeleteSnapshotScheduleResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [this, &resp, &schedule_id](RpcController* rpc) { - master::DeleteSnapshotScheduleRequestPB req; - req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); - - return master_backup_proxy_->DeleteSnapshotSchedule(req, &resp, rpc); - })); - - rapidjson::Document document; - document.SetObject(); - AddStringField("schedule_id", schedule_id.ToString(), &document, &document.GetAllocator()); - return document; -} - -bool SnapshotSuitableForRestoreAt(const SysSnapshotEntryPB& entry, HybridTime restore_at) { - return (entry.state() == master::SysSnapshotEntryPB::COMPLETE || - entry.state() == master::SysSnapshotEntryPB::CREATING) && - HybridTime::FromPB(entry.snapshot_hybrid_time()) >= restore_at && - HybridTime::FromPB(entry.previous_snapshot_hybrid_time()) < restore_at; -} - -Result ClusterAdminClient::SuitableSnapshotId( - const SnapshotScheduleId& schedule_id, HybridTime restore_at, CoarseTimePoint deadline) { - for (;;) { - auto last_snapshot_time = HybridTime::kMin; - { - RpcController rpc; - rpc.set_deadline(deadline); - master::ListSnapshotSchedulesRequestPB req; - master::ListSnapshotSchedulesResponsePB resp; - if (schedule_id) { - req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); - } - - RETURN_NOT_OK_PREPEND(master_backup_proxy_->ListSnapshotSchedules(req, &resp, &rpc), - "Failed to list snapshot schedules"); - - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - - if (resp.schedules().size() < 1) { - return STATUS_FORMAT(InvalidArgument, "Unknown schedule: $0", schedule_id); - } - - for (const auto& snapshot : resp.schedules()[0].snapshots()) { - auto snapshot_hybrid_time = HybridTime::FromPB(snapshot.entry().snapshot_hybrid_time()); - last_snapshot_time = std::max(last_snapshot_time, snapshot_hybrid_time); - if (SnapshotSuitableForRestoreAt(snapshot.entry(), restore_at)) { - return VERIFY_RESULT(FullyDecodeTxnSnapshotId(snapshot.id())); - } - } - if (last_snapshot_time > restore_at) { - return STATUS_FORMAT( - IllegalState, "Cannot restore at $0, last snapshot: $1, snapshots: $2", - restore_at, last_snapshot_time, resp.schedules()[0].snapshots()); - } - } - RpcController rpc; - rpc.set_deadline(deadline); - master::CreateSnapshotRequestPB req; - master::CreateSnapshotResponsePB resp; - req.set_schedule_id(schedule_id.data(), schedule_id.size()); - RETURN_NOT_OK_PREPEND(master_backup_proxy_->CreateSnapshot(req, &resp, &rpc), - "Failed to create snapshot"); - if (resp.has_error()) { - auto status = StatusFromPB(resp.error().status()); - if (master::MasterError(status) == master::MasterErrorPB::PARALLEL_SNAPSHOT_OPERATION) { - std::this_thread::sleep_until(std::min(deadline, CoarseMonoClock::now() + 1s)); - continue; - } - return status; - } - return FullyDecodeTxnSnapshotId(resp.snapshot_id()); - } -} - -Status ClusterAdminClient::DisableTabletSplitsDuringRestore(CoarseTimePoint deadline) { - // TODO(Sanket): Eventually all of this logic needs to be moved - // to the master and exposed as APIs for the clients to consume. - const auto splitting_disabled_until = - CoarseMonoClock::Now() + MonoDelta::FromSeconds(kPitrSplitDisableDurationSecs); - // Disable splitting and then wait for all pending splits to complete before - // starting restoration. - VERIFY_RESULT_PREPEND( - DisableTabletSplitsInternal(kPitrSplitDisableDurationSecs * 1000, kPitrFeatureName), - "Failed to disable tablet split before restore."); - - while (CoarseMonoClock::Now() < std::min(splitting_disabled_until, deadline)) { - // Wait for existing split operations to complete. - const auto resp = VERIFY_RESULT_PREPEND( - IsTabletSplittingCompleteInternal(true /* wait_for_parent_deletion */, - deadline - CoarseMonoClock::now() /* timeout */), - "Tablet splitting did not complete. Cannot restore."); - if (resp.is_tablet_splitting_complete()) { - break; - } - SleepFor(MonoDelta::FromMilliseconds(kPitrSplitDisableCheckFreqMs)); - } - - if (CoarseMonoClock::now() >= deadline) { - return STATUS(TimedOut, "Timed out waiting for tablet splitting to complete."); - } - - // Return if we have used almost all of our time in waiting for splitting to complete, - // since we can't guarantee that another split does not start. - if (CoarseMonoClock::now() + MonoDelta::FromSeconds(3) >= splitting_disabled_until) { - return STATUS(TimedOut, "Not enough time after disabling splitting to disable ", - "splitting again."); - } - - // Disable for kPitrSplitDisableDurationSecs again so the restore has the full amount of time with - // splitting disables. This overwrites the previous value since the feature_name is the same so - // overall the time is still kPitrSplitDisableDurationSecs. - VERIFY_RESULT_PREPEND( - DisableTabletSplitsInternal(kPitrSplitDisableDurationSecs * 1000, kPitrFeatureName), - "Failed to disable tablet split before restore."); - - return Status::OK(); -} - -Result ClusterAdminClient::RestoreSnapshotScheduleDeprecated( - const SnapshotScheduleId& schedule_id, HybridTime restore_at) { - auto deadline = CoarseMonoClock::now() + timeout_; - - // Disable splitting for the entire run of restore. - RETURN_NOT_OK(DisableTabletSplitsDuringRestore(deadline)); - - // Get the suitable snapshot to restore from. - auto snapshot_id = VERIFY_RESULT(SuitableSnapshotId(schedule_id, restore_at, deadline)); - - for (;;) { - RpcController rpc; - rpc.set_deadline(deadline); - master::ListSnapshotsRequestPB req; - req.set_snapshot_id(snapshot_id.data(), snapshot_id.size()); - master::ListSnapshotsResponsePB resp; - RETURN_NOT_OK_PREPEND(master_backup_proxy_->ListSnapshots(req, &resp, &rpc), - "Failed to list snapshots"); - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - if (resp.snapshots().size() != 1) { - return STATUS_FORMAT( - IllegalState, "Wrong number of snapshots received $0", resp.snapshots().size()); - } - if (resp.snapshots()[0].entry().state() == master::SysSnapshotEntryPB::COMPLETE) { - if (SnapshotSuitableForRestoreAt(resp.snapshots()[0].entry(), restore_at)) { - break; - } - return STATUS_FORMAT( - IllegalState, "Snapshot is not suitable for restore at $0", restore_at); - } - auto now = CoarseMonoClock::now(); - if (now >= deadline) { - return STATUS_FORMAT( - TimedOut, "Timed out to complete a snapshot $0", snapshot_id); - } - std::this_thread::sleep_until(std::min(deadline, now + 100ms)); - } - - RpcController rpc; - rpc.set_deadline(deadline); - RestoreSnapshotRequestPB req; - RestoreSnapshotResponsePB resp; - req.set_snapshot_id(snapshot_id.data(), snapshot_id.size()); - req.set_restore_ht(restore_at.ToUint64()); - RETURN_NOT_OK_PREPEND(master_backup_proxy_->RestoreSnapshot(req, &resp, &rpc), - "Failed to restore snapshot"); - - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - - auto restoration_id = VERIFY_RESULT(FullyDecodeTxnSnapshotRestorationId(resp.restoration_id())); - - rapidjson::Document document; - document.SetObject(); - - AddStringField("snapshot_id", snapshot_id.ToString(), &document, &document.GetAllocator()); - AddStringField("restoration_id", restoration_id.ToString(), &document, &document.GetAllocator()); - - return document; -} - -Result ClusterAdminClient::RestoreSnapshotSchedule( - const SnapshotScheduleId& schedule_id, HybridTime restore_at) { - auto deadline = CoarseMonoClock::now() + timeout_; - - RpcController rpc; - rpc.set_deadline(deadline); - master::RestoreSnapshotScheduleRequestPB req; - master::RestoreSnapshotScheduleResponsePB resp; - req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); - req.set_restore_ht(restore_at.ToUint64()); - - Status s = master_backup_proxy_->RestoreSnapshotSchedule(req, &resp, &rpc); - if (!s.ok()) { - if (s.IsRemoteError() && - rpc.error_response()->code() == rpc::ErrorStatusPB::ERROR_NO_SUCH_METHOD) { - cout << "WARNING: fallback to RestoreSnapshotScheduleDeprecated." << endl; - return RestoreSnapshotScheduleDeprecated(schedule_id, restore_at); - } - RETURN_NOT_OK_PREPEND(s, Format("Failed to restore snapshot from schedule: $0", - schedule_id.ToString())); - } - - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - - auto snapshot_id = VERIFY_RESULT(FullyDecodeTxnSnapshotId(resp.snapshot_id())); - auto restoration_id = VERIFY_RESULT(FullyDecodeTxnSnapshotRestorationId(resp.restoration_id())); - - rapidjson::Document document; - document.SetObject(); - - AddStringField("snapshot_id", snapshot_id.ToString(), &document, &document.GetAllocator()); - AddStringField("restoration_id", restoration_id.ToString(), &document, &document.GetAllocator()); - - return document; -} - -Result ClusterAdminClient::EditSnapshotSchedule( - const SnapshotScheduleId& schedule_id, std::optional new_interval, - std::optional new_retention) { - - master::EditSnapshotScheduleResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - master::EditSnapshotScheduleRequestPB req; - req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); - if (new_interval) { - req.set_interval_sec(new_interval->ToSeconds()); - } - if (new_retention) { - req.set_retention_duration_sec(new_retention->ToSeconds()); - } - return master_backup_proxy_->EditSnapshotSchedule(req, &resp, rpc); - })); - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - rapidjson::Document document; - document.SetObject(); - rapidjson::Value json_schedule = - VERIFY_RESULT(SnapshotScheduleInfoToJson(resp.schedule(), &document.GetAllocator())); - document.AddMember("schedule", json_schedule, document.GetAllocator()); - return document; -} - -Status ClusterAdminClient::RestoreSnapshot(const string& snapshot_id, - HybridTime timestamp) { - RestoreSnapshotResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - RestoreSnapshotRequestPB req; - req.set_snapshot_id(StringToSnapshotId(snapshot_id)); - if (timestamp) { - req.set_restore_ht(timestamp.ToUint64()); - } - return master_backup_proxy_->RestoreSnapshot(req, &resp, rpc); - })); - - cout << "Started restoring snapshot: " << snapshot_id << endl - << "Restoration id: " << FullyDecodeTxnSnapshotRestorationId(resp.restoration_id()) << endl; - if (timestamp) { - cout << "Restore at: " << timestamp << endl; - } - return Status::OK(); -} - -Status ClusterAdminClient::DeleteSnapshot(const std::string& snapshot_id) { - DeleteSnapshotResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - DeleteSnapshotRequestPB req; - req.set_snapshot_id(StringToSnapshotId(snapshot_id)); - return master_backup_proxy_->DeleteSnapshot(req, &resp, rpc); - })); - - cout << "Deleted snapshot: " << snapshot_id << endl; - return Status::OK(); -} - -Status ClusterAdminClient::CreateSnapshotMetaFile(const string& snapshot_id, - const string& file_name) { - ListSnapshotsResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - ListSnapshotsRequestPB req; - req.set_snapshot_id(StringToSnapshotId(snapshot_id)); - - // Format 0 - latest format (== Format 2 at the moment). - // Format -1 - old format (no 'namespace_name' in the Table entry). - // Format 1 - old format. - // Format 2 - new format. - if (FLAGS_TEST_metadata_file_format_version == 0 || - FLAGS_TEST_metadata_file_format_version >= 2) { - req.set_prepare_for_backup(true); - } - return master_backup_proxy_->ListSnapshots(req, &resp, rpc); - })); - - if (resp.snapshots_size() > 1) { - LOG(WARNING) << "Requested snapshot metadata for snapshot '" << snapshot_id << "', but got " - << resp.snapshots_size() << " snapshots in the response"; - } - - SnapshotInfoPB* snapshot = nullptr; - for (SnapshotInfoPB& snapshot_entry : *resp.mutable_snapshots()) { - if (SnapshotIdToString(snapshot_entry.id()) == snapshot_id) { - snapshot = &snapshot_entry; - break; - } - } - if (!snapshot) { - return STATUS_FORMAT( - InternalError, "Response contained $0 entries but no entry for snapshot '$1'", - resp.snapshots_size(), snapshot_id); - } - - if (FLAGS_TEST_metadata_file_format_version == -1) { - // Remove 'namespace_name' from SysTablesEntryPB. - SysSnapshotEntryPB& sys_entry = *snapshot->mutable_entry(); - for (SysRowEntry& entry : *sys_entry.mutable_entries()) { - if (entry.type() == SysRowEntryType::TABLE) { - auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); - meta.clear_namespace_name(); - entry.set_data(meta.SerializeAsString()); - } - } - } - - cout << "Exporting snapshot " << snapshot_id << " (" - << snapshot->entry().state() << ") to file " << file_name << endl; - - // Serialize snapshot protobuf to given path. - RETURN_NOT_OK(pb_util::WritePBContainerToPath( - Env::Default(), file_name, *snapshot, pb_util::OVERWRITE, pb_util::SYNC)); - - cout << "Snapshot metadata was saved into file: " << file_name << endl; - return Status::OK(); -} - -string ClusterAdminClient::GetDBTypeName(const SysNamespaceEntryPB& pb) { - const YQLDatabase db_type = GetDatabaseType(pb); - switch (db_type) { - case YQL_DATABASE_UNKNOWN: return "UNKNOWN DB TYPE"; - case YQL_DATABASE_CQL: return "YCQL keyspace"; - case YQL_DATABASE_PGSQL: return "YSQL database"; - case YQL_DATABASE_REDIS: return "YEDIS namespace"; - } - FATAL_INVALID_ENUM_VALUE(YQLDatabase, db_type); -} - -Status ClusterAdminClient::UpdateUDTypes( - QLTypePB* pb_type, bool* update_meta, const NSNameToNameMap& ns_name_to_name) { - return IterateAndDoForUDT( - pb_type, - [update_meta, &ns_name_to_name](QLTypePB::UDTypeInfo* udtype_info) -> Status { - auto ns_it = ns_name_to_name.find(udtype_info->keyspace_name()); - if (ns_it == ns_name_to_name.end()) { - return STATUS_FORMAT( - InvalidArgument, "No metadata for keyspace '$0' referenced in UDType '$1'", - udtype_info->keyspace_name(), udtype_info->name()); - } - - if (udtype_info->keyspace_name() != ns_it->second) { - udtype_info->set_keyspace_name(ns_it->second); - *DCHECK_NOTNULL(update_meta) = true; - } - return Status::OK(); - }); -} - -Status ClusterAdminClient::ImportSnapshotMetaFile(const string& file_name, - const TypedNamespaceName& keyspace, - const vector& tables) { - cout << "Read snapshot meta file " << file_name << endl; - - ImportSnapshotMetaRequestPB req; - - SnapshotInfoPB* const snapshot_info = req.mutable_snapshot(); - - // Read snapshot protobuf from given path. - RETURN_NOT_OK(pb_util::ReadPBContainerFromPath(Env::Default(), file_name, snapshot_info)); - - if (!snapshot_info->has_format_version()) { - SCHECK_EQ( - 0, snapshot_info->backup_entries_size(), InvalidArgument, - Format("Metadata file in Format 1 has backup entries from Format 2: $0", - snapshot_info->backup_entries_size())); - - // Repack PB data loaded in the old format. - // Init BackupSnapshotPB based on loaded SnapshotInfoPB. - SysSnapshotEntryPB& sys_entry = *snapshot_info->mutable_entry(); - snapshot_info->mutable_backup_entries()->Reserve(sys_entry.entries_size()); - for (SysRowEntry& entry : *sys_entry.mutable_entries()) { - snapshot_info->add_backup_entries()->mutable_entry()->Swap(&entry); - } - - sys_entry.clear_entries(); - snapshot_info->set_format_version(2); - } - - cout << "Importing snapshot " << SnapshotIdToString(snapshot_info->id()) - << " (" << snapshot_info->entry().state() << ")" << endl; - - // Map: Old namespace ID -> [Old name, New name] pair. - typedef pair NSNamePair; - typedef unordered_map NSIdToNameMap; - NSIdToNameMap ns_id_to_name; - NSNameToNameMap ns_name_to_name; - - size_t table_index = 0; - bool was_table_renamed = false; - for (BackupRowEntryPB& backup_entry : *snapshot_info->mutable_backup_entries()) { - SysRowEntry& entry = *backup_entry.mutable_entry(); - const YBTableName table_name = table_index < tables.size() - ? tables[table_index] : YBTableName(); - - switch (entry.type()) { - case SysRowEntryType::NAMESPACE: { - auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); - const string db_type = GetDBTypeName(meta); - cout << "Target imported " << db_type << " name: " - << (keyspace.name.empty() ? meta.name() : keyspace.name) << endl - << db_type << " being imported: " << meta.name() << endl; - - if (!keyspace.name.empty() && keyspace.name != meta.name()) { - ns_id_to_name[entry.id()] = NSNamePair(meta.name(), keyspace.name); - ns_name_to_name[meta.name()] = keyspace.name; - - meta.set_name(keyspace.name); - entry.set_data(meta.SerializeAsString()); - } else { - ns_id_to_name[entry.id()] = NSNamePair(meta.name(), meta.name()); - ns_name_to_name[meta.name()] = meta.name(); - } - break; - } - case SysRowEntryType::UDTYPE: { - // Note: UDT renaming is not supported. - auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); - const auto ns_it = ns_id_to_name.find(meta.namespace_id()); - cout << "Target imported user-defined type name: " - << (ns_it == ns_id_to_name.end() ? "[" + meta.namespace_id() + "]" : - ns_it->second.second) << "." << meta.name() << endl - << "User-defined type being imported: " - << (ns_it == ns_id_to_name.end() ? "[" + meta.namespace_id() + "]" : - ns_it->second.first) << "." << meta.name() << endl; - - bool update_meta = false; - // Update QLTypePB::UDTypeInfo::keyspace_name in the UDT params. - for (int i = 0; i < meta.field_names_size(); ++i) { - RETURN_NOT_OK(UpdateUDTypes(meta.mutable_field_types(i), &update_meta, ns_name_to_name)); - } - - if (update_meta) { - entry.set_data(meta.SerializeAsString()); - } - break; - } - case SysRowEntryType::TABLE: { - if (was_table_renamed && table_name.empty()) { - // Renaming is allowed for all tables OR for no one table. - return STATUS_FORMAT(InvalidArgument, - "There is no name for table (including indexes) number: $0", - table_index); - } - - auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); - const auto colocated_prefix = meta.colocated() ? "colocated " : ""; - - if (meta.indexed_table_id().empty()) { - cout << "Table type: " << colocated_prefix << "table" << endl; - } else { - cout << "Table type: " << colocated_prefix << "index (attaching to the old table id " - << meta.indexed_table_id() << ")" << endl; - } - - if (!table_name.empty()) { - DCHECK(table_name.has_namespace()); - DCHECK(table_name.has_table()); - cout << "Target imported " << colocated_prefix << "table name: " - << table_name.ToString() << endl; - } else if (!keyspace.name.empty()) { - cout << "Target imported " << colocated_prefix << "table name: " - << keyspace.name << "." << meta.name() << endl; - } - - // Print old table name before the table renaming in the code below. - cout << (meta.colocated() ? "Colocated t" : "T") << "able being imported: " - << (meta.namespace_name().empty() ? - "[" + meta.namespace_id() + "]" : meta.namespace_name()) - << "." << meta.name() << endl; - - bool update_meta = false; - if (!table_name.empty() && table_name.table_name() != meta.name()) { - meta.set_name(table_name.table_name()); - update_meta = true; - was_table_renamed = true; - } - if (!keyspace.name.empty() && keyspace.name != meta.namespace_name()) { - meta.set_namespace_name(keyspace.name); - update_meta = true; - } - - if (meta.name().empty()) { - return STATUS(IllegalState, "Could not find table name from snapshot metadata"); - } - - // Update QLTypePB::UDTypeInfo::keyspace_name in all UDT params in the table Schema. - SchemaPB* const schema = meta.mutable_schema(); - // Recursively update ids in used user-defined types. - for (int i = 0; i < schema->columns_size(); ++i) { - QLTypePB* const pb_type = schema->mutable_columns(i)->mutable_type(); - RETURN_NOT_OK(UpdateUDTypes(pb_type, &update_meta, ns_name_to_name)); - } - - // Update the table name if needed. - if (update_meta) { - entry.set_data(meta.SerializeAsString()); - } - - ++table_index; - break; - } - default: - break; - } - } - - // RPC timeout is a function of the number of tables that needs to be imported. - ImportSnapshotMetaResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - return master_backup_proxy_->ImportSnapshotMeta(req, &resp, rpc); - }, timeout_ * std::max(static_cast(1), table_index))); - - const int kObjectColumnWidth = 16; - const auto pad_object_type = [](const string& s) { - return RightPadToWidth(s, kObjectColumnWidth); - }; - - cout << "Successfully applied snapshot." << endl - << pad_object_type("Object") << kColumnSep - << RightPadToUuidWidth("Old ID") << kColumnSep - << RightPadToUuidWidth("New ID") << endl; - - const RepeatedPtrField& tables_meta = - resp.tables_meta(); - CreateSnapshotRequestPB snapshot_req; - CreateSnapshotResponsePB snapshot_resp; - - for (int i = 0; i < tables_meta.size(); ++i) { - const ImportSnapshotMetaResponsePB_TableMetaPB& table_meta = tables_meta.Get(i); - const string& new_table_id = table_meta.table_ids().new_id(); - - cout << pad_object_type("Keyspace") << kColumnSep - << table_meta.namespace_ids().old_id() << kColumnSep - << table_meta.namespace_ids().new_id() << endl; - - if (!ImportSnapshotMetaResponsePB_TableType_IsValid(table_meta.table_type())) { - return STATUS_FORMAT(InternalError, "Found unknown table type: ", table_meta.table_type()); - } - - const string table_type = - AllCapsToCamelCase(ImportSnapshotMetaResponsePB_TableType_Name(table_meta.table_type())); - cout << pad_object_type(table_type) << kColumnSep - << table_meta.table_ids().old_id() << kColumnSep - << new_table_id << endl; - - const RepeatedPtrField& udts_map = table_meta.ud_types_ids(); - for (int j = 0; j < udts_map.size(); ++j) { - const IdPairPB& pair = udts_map.Get(j); - cout << pad_object_type("UDType") << kColumnSep - << pair.old_id() << kColumnSep - << pair.new_id() << endl; - } - - const RepeatedPtrField& tablets_map = table_meta.tablets_ids(); - for (int j = 0; j < tablets_map.size(); ++j) { - const IdPairPB& pair = tablets_map.Get(j); - cout << pad_object_type(Format("Tablet $0", j)) << kColumnSep - << pair.old_id() << kColumnSep - << pair.new_id() << endl; - } - - RETURN_NOT_OK(yb_client_->WaitForCreateTableToFinish( - new_table_id, - CoarseMonoClock::Now() + MonoDelta::FromSeconds(FLAGS_yb_client_admin_operation_timeout_sec) - )); - - snapshot_req.mutable_tables()->Add()->set_table_id(new_table_id); - } - - // All indexes already are in the request. Do not add them twice. - snapshot_req.set_add_indexes(false); - snapshot_req.set_transaction_aware(true); - snapshot_req.set_imported(true); - // Create new snapshot. - RETURN_NOT_OK(RequestMasterLeader(&snapshot_resp, [&](RpcController* rpc) { - return master_backup_proxy_->CreateSnapshot(snapshot_req, &snapshot_resp, rpc); - })); - - cout << pad_object_type("Snapshot") << kColumnSep - << SnapshotIdToString(snapshot_info->id()) << kColumnSep - << SnapshotIdToString(snapshot_resp.snapshot_id()) << endl; - - return Status::OK(); -} - -Status ClusterAdminClient::ListReplicaTypeCounts(const YBTableName& table_name) { - vector tablet_ids, ranges; - RETURN_NOT_OK(yb_client_->GetTablets(table_name, 0, &tablet_ids, &ranges)); - master::GetTabletLocationsResponsePB resp; - RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { - master::GetTabletLocationsRequestPB req; - for (const auto& tablet_id : tablet_ids) { - req.add_tablet_ids(tablet_id); - } - return master_client_proxy_->GetTabletLocations(req, &resp, rpc); - })); - - struct ReplicaCounts { - int live_count; - int read_only_count; - string placement_uuid; - }; - std::map replica_map; - - std::cout << "Tserver ID\t\tPlacement ID\t\tLive count\t\tRead only count\n"; - - for (int tablet_idx = 0; tablet_idx < resp.tablet_locations_size(); tablet_idx++) { - const master::TabletLocationsPB& locs = resp.tablet_locations(tablet_idx); - for (int replica_idx = 0; replica_idx < locs.replicas_size(); replica_idx++) { - const auto& replica = locs.replicas(replica_idx); - const string& ts_uuid = replica.ts_info().permanent_uuid(); - const string& placement_uuid = - replica.ts_info().has_placement_uuid() ? replica.ts_info().placement_uuid() : ""; - bool is_replica_read_only = - replica.member_type() == consensus::PeerMemberType::PRE_OBSERVER || - replica.member_type() == consensus::PeerMemberType::OBSERVER; - int live_count = is_replica_read_only ? 0 : 1; - int read_only_count = 1 - live_count; - if (replica_map.count(ts_uuid) == 0) { - replica_map[ts_uuid].live_count = live_count; - replica_map[ts_uuid].read_only_count = read_only_count; - replica_map[ts_uuid].placement_uuid = placement_uuid; - } else { - ReplicaCounts* counts = &replica_map[ts_uuid]; - counts->live_count += live_count; - counts->read_only_count += read_only_count; - } - } - } - - for (auto const& tserver : replica_map) { - std::cout << tserver.first << "\t\t" << tserver.second.placement_uuid << "\t\t" - << tserver.second.live_count << "\t\t" << tserver.second.read_only_count << std::endl; - } - - return Status::OK(); -} - -Status ClusterAdminClient::SetPreferredZones(const std::vector& preferred_zones) { - rpc::RpcController rpc; - master::SetPreferredZonesRequestPB req; - master::SetPreferredZonesResponsePB resp; - rpc.set_timeout(timeout_); - - std::set zones; - std::set visited_priorities; - - for (const string& zone : preferred_zones) { - if (zones.find(zone) != zones.end()) { - return STATUS_SUBSTITUTE( - InvalidArgument, "Invalid argument for preferred zone $0, values should not repeat", - zone); - } - - std::vector zone_priority_split = strings::Split(zone, ":", strings::AllowEmpty()); - if (zone_priority_split.size() == 0 || zone_priority_split.size() > 2) { - return STATUS_SUBSTITUTE( - InvalidArgument, - "Invalid argument for preferred zone $0, should have format cloud.region.zone[:priority]", - zone); - } - - std::vector tokens = strings::Split(zone_priority_split[0], ".", strings::AllowEmpty()); - if (tokens.size() != 3) { - return STATUS_SUBSTITUTE( - InvalidArgument, - "Invalid argument for preferred zone $0, should have format cloud.region.zone:[priority]", - zone); - } - - uint priority = 1; - - if (zone_priority_split.size() == 2) { - auto result = CheckedStoi(zone_priority_split[1]); - if (!result.ok() || result.get() < 1) { - return STATUS_SUBSTITUTE( - InvalidArgument, - "Invalid argument for preferred zone $0, priority should be non-zero positive integer", - zone); - } - priority = static_cast(result.get()); - } - - // Max priority if each zone has a unique priority value can only be the size of the input array - if (priority > preferred_zones.size()) { - return STATUS( - InvalidArgument, - "Priority value cannot be more than the number of zones in the preferred list since each " - "priority should be associated with at least one zone from the list"); - } - - CloudInfoPB* cloud_info = nullptr; - visited_priorities.insert(priority); - - while (req.multi_preferred_zones_size() < static_cast(priority)) { - req.add_multi_preferred_zones(); - } - - auto current_list = req.mutable_multi_preferred_zones(priority - 1); - cloud_info = current_list->add_zones(); - - cloud_info->set_placement_cloud(tokens[0]); - cloud_info->set_placement_region(tokens[1]); - cloud_info->set_placement_zone(tokens[2]); - - zones.emplace(zone); - - if (priority == 1) { - // Handle old clusters which can only handle a single priority. New clusters will ignore this - // member as multi_preferred_zones is already set. - cloud_info = req.add_preferred_zones(); - cloud_info->set_placement_cloud(tokens[0]); - cloud_info->set_placement_region(tokens[1]); - cloud_info->set_placement_zone(tokens[2]); - } - } - - int size = static_cast(visited_priorities.size()); - if (size > 0 && (*(visited_priorities.rbegin()) != size)) { - return STATUS_SUBSTITUTE( - InvalidArgument, "Invalid argument, each priority should have at least one zone"); - } - - RETURN_NOT_OK(master_cluster_proxy_->SetPreferredZones(req, &resp, &rpc)); - - if (resp.has_error()) { - return STATUS(ServiceUnavailable, resp.error().status().message()); - } - - return Status::OK(); -} - -Status ClusterAdminClient::RotateUniverseKey(const std::string& key_path) { - return SendEncryptionRequest(key_path, true); -} - -Status ClusterAdminClient::DisableEncryption() { - return SendEncryptionRequest("", false); -} - -Status ClusterAdminClient::SendEncryptionRequest( - const std::string& key_path, bool enable_encryption) { - RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); - rpc::RpcController rpc; - rpc.set_timeout(timeout_); - - // Get the cluster config from the master leader. - master::ChangeEncryptionInfoRequestPB encryption_info_req; - master::ChangeEncryptionInfoResponsePB encryption_info_resp; - encryption_info_req.set_encryption_enabled(enable_encryption); - if (key_path != "") { - encryption_info_req.set_key_path(key_path); - } - RETURN_NOT_OK_PREPEND(master_encryption_proxy_-> - ChangeEncryptionInfo(encryption_info_req, &encryption_info_resp, &rpc), - "MasterServiceImpl::ChangeEncryptionInfo call fails.") - - if (encryption_info_resp.has_error()) { - return StatusFromPB(encryption_info_resp.error().status()); - } - return Status::OK(); -} - -Status ClusterAdminClient::IsEncryptionEnabled() { - RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); - rpc::RpcController rpc; - rpc.set_timeout(timeout_); - - master::IsEncryptionEnabledRequestPB req; - master::IsEncryptionEnabledResponsePB resp; - RETURN_NOT_OK_PREPEND(master_encryption_proxy_-> - IsEncryptionEnabled(req, &resp, &rpc), - "MasterServiceImpl::IsEncryptionEnabled call fails."); - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - - std::cout << "Encryption status: " << (resp.encryption_enabled() ? - Format("ENABLED with key id $0", resp.key_id()) : "DISABLED" ) << std::endl; - return Status::OK(); -} - -Status ClusterAdminClient::AddUniverseKeyToAllMasters( - const std::string& key_id, const std::string& universe_key) { - - RETURN_NOT_OK(encryption::EncryptionParams::IsValidKeySize( - universe_key.size() - encryption::EncryptionParams::kBlockSize)); - - master::AddUniverseKeysRequestPB req; - master::AddUniverseKeysResponsePB resp; - auto* universe_keys = req.mutable_universe_keys(); - (*universe_keys->mutable_map())[key_id] = universe_key; - - for (auto hp : VERIFY_RESULT(HostPort::ParseStrings(master_addr_list_, 7100))) { - rpc::RpcController rpc; - rpc.set_timeout(timeout_); - master::MasterEncryptionProxy proxy(proxy_cache_.get(), hp); - RETURN_NOT_OK_PREPEND(proxy.AddUniverseKeys(req, &resp, &rpc), - Format("MasterServiceImpl::AddUniverseKeys call fails on host $0.", - hp.ToString())); - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - std::cout << Format("Successfully added key to the node $0.\n", hp.ToString()); - } - - return Status::OK(); -} - -Status ClusterAdminClient::AllMastersHaveUniverseKeyInMemory(const std::string& key_id) { - master::HasUniverseKeyInMemoryRequestPB req; - master::HasUniverseKeyInMemoryResponsePB resp; - req.set_version_id(key_id); - - for (auto hp : VERIFY_RESULT(HostPort::ParseStrings(master_addr_list_, 7100))) { - rpc::RpcController rpc; - rpc.set_timeout(timeout_); - master::MasterEncryptionProxy proxy(proxy_cache_.get(), hp); - RETURN_NOT_OK_PREPEND(proxy.HasUniverseKeyInMemory(req, &resp, &rpc), - "MasterServiceImpl::ChangeEncryptionInfo call fails."); - - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - if (!resp.has_key()) { - return STATUS_FORMAT(TryAgain, "Node $0 does not have universe key in memory", hp); - } - - std::cout << Format("Node $0 has universe key in memory: $1\n", hp.ToString(), resp.has_key()); - } - - return Status::OK(); -} - -Status ClusterAdminClient::RotateUniverseKeyInMemory(const std::string& key_id) { - RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); - rpc::RpcController rpc; - rpc.set_timeout(timeout_); - - master::ChangeEncryptionInfoRequestPB req; - master::ChangeEncryptionInfoResponsePB resp; - req.set_encryption_enabled(true); - req.set_in_memory(true); - req.set_version_id(key_id); - RETURN_NOT_OK_PREPEND(master_encryption_proxy_->ChangeEncryptionInfo(req, &resp, &rpc), - "MasterServiceImpl::ChangeEncryptionInfo call fails."); - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - - std::cout << "Rotated universe key in memory\n"; - return Status::OK(); -} - -Status ClusterAdminClient::DisableEncryptionInMemory() { - RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); - rpc::RpcController rpc; - rpc.set_timeout(timeout_); - - master::ChangeEncryptionInfoRequestPB req; - master::ChangeEncryptionInfoResponsePB resp; - req.set_encryption_enabled(false); - RETURN_NOT_OK_PREPEND(master_encryption_proxy_->ChangeEncryptionInfo(req, &resp, &rpc), - "MasterServiceImpl::ChangeEncryptionInfo call fails."); - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - - std::cout << "Encryption disabled\n"; - return Status::OK(); -} - -Status ClusterAdminClient::WriteUniverseKeyToFile( - const std::string& key_id, const std::string& file_name) { - RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); - rpc::RpcController rpc; - rpc.set_timeout(timeout_); - - master::GetUniverseKeyRegistryRequestPB req; - master::GetUniverseKeyRegistryResponsePB resp; - RETURN_NOT_OK_PREPEND(master_encryption_proxy_->GetUniverseKeyRegistry(req, &resp, &rpc), - "MasterServiceImpl::ChangeEncryptionInfo call fails."); - if (resp.has_error()) { - return StatusFromPB(resp.error().status()); - } - - auto universe_keys = resp.universe_keys(); - const auto& it = universe_keys.map().find(key_id); - if (it == universe_keys.map().end()) { - return STATUS_FORMAT(NotFound, "Could not find key with id $0", key_id); - } - - RETURN_NOT_OK(WriteStringToFile(Env::Default(), Slice(it->second), file_name)); - - std::cout << "Finished writing to file\n"; - return Status::OK(); -} - -Status ClusterAdminClient::CreateCDCSDKDBStream( - const TypedNamespaceName& ns, const std::string& checkpoint_type, - const std::string& record_type) { - HostPort ts_addr = VERIFY_RESULT(GetFirstRpcAddressForTS()); - auto cdc_proxy = std::make_unique(proxy_cache_.get(), ts_addr); - - cdc::CreateCDCStreamRequestPB req; - cdc::CreateCDCStreamResponsePB resp; - - req.set_namespace_name(ns.name); - - if (record_type == yb::ToString("ALL")) { - req.set_record_type(cdc::CDCRecordType::ALL); - } else { - req.set_record_type(cdc::CDCRecordType::CHANGE); - } - - req.set_record_format(cdc::CDCRecordFormat::PROTO); - req.set_source_type(cdc::CDCRequestSource::CDCSDK); - if (checkpoint_type == yb::ToString("EXPLICIT")) { - req.set_checkpoint_type(cdc::CDCCheckpointType::EXPLICIT); - } else { - req.set_checkpoint_type(cdc::CDCCheckpointType::IMPLICIT); - } - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(cdc_proxy->CreateCDCStream(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error creating stream: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "CDC Stream ID: " << resp.db_stream_id() << endl; - return Status::OK(); -} - -Status ClusterAdminClient::CreateCDCStream(const TableId& table_id) { - master::CreateCDCStreamRequestPB req; - master::CreateCDCStreamResponsePB resp; - req.set_table_id(table_id); - req.mutable_options()->Reserve(3); - - auto record_type_option = req.add_options(); - record_type_option->set_key(cdc::kRecordType); - record_type_option->set_value(CDCRecordType_Name(cdc::CDCRecordType::CHANGE)); - - auto record_format_option = req.add_options(); - record_format_option->set_key(cdc::kRecordFormat); - record_format_option->set_value(CDCRecordFormat_Name(cdc::CDCRecordFormat::JSON)); - - auto source_type_option = req.add_options(); - source_type_option->set_key(cdc::kSourceType); - source_type_option->set_value(CDCRequestSource_Name(cdc::CDCRequestSource::XCLUSTER)); - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->CreateCDCStream(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error creating stream: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "CDC Stream ID: " << resp.stream_id() << endl; - return Status::OK(); -} - -Status ClusterAdminClient::DeleteCDCSDKDBStream(const std::string& db_stream_id) { - master::DeleteCDCStreamRequestPB req; - master::DeleteCDCStreamResponsePB resp; - req.add_stream_id(db_stream_id); - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->DeleteCDCStream(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error deleting stream: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "Successfully deleted Change Data Stream ID: " << db_stream_id << endl; - return Status::OK(); -} - -Status ClusterAdminClient::DeleteCDCStream(const std::string& stream_id, bool force_delete) { - master::DeleteCDCStreamRequestPB req; - master::DeleteCDCStreamResponsePB resp; - req.add_stream_id(stream_id); - req.set_force_delete(force_delete); - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->DeleteCDCStream(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error deleting stream: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "Successfully deleted CDC Stream ID: " << stream_id << endl; - return Status::OK(); -} - -Status ClusterAdminClient::ListCDCStreams(const TableId& table_id) { - master::ListCDCStreamsRequestPB req; - master::ListCDCStreamsResponsePB resp; - req.set_id_type(yb::master::IdTypePB::TABLE_ID); - if (!table_id.empty()) { - req.set_table_id(table_id); - } - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->ListCDCStreams(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error getting CDC stream list: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "CDC Streams: \r\n" << resp.DebugString(); - return Status::OK(); -} - -Status ClusterAdminClient::ListCDCSDKStreams(const std::string& namespace_name) { - master::ListCDCStreamsRequestPB req; - master::ListCDCStreamsResponsePB resp; - req.set_id_type(yb::master::IdTypePB::NAMESPACE_ID); - - if (!namespace_name.empty()) { - cout << "Filtering out DB streams for the namespace: " << namespace_name << "\n\n"; - master::GetNamespaceInfoResponsePB namespace_info_resp; - RETURN_NOT_OK(yb_client_->GetNamespaceInfo("", namespace_name, YQL_DATABASE_PGSQL, - &namespace_info_resp)); - req.set_namespace_id(namespace_info_resp.namespace_().id()); - } - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->ListCDCStreams(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error getting CDC stream list: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "CDC Streams: \r\n" << resp.DebugString(); - return Status::OK(); -} - -Status ClusterAdminClient::GetCDCDBStreamInfo(const std::string& db_stream_id) { - master::GetCDCDBStreamInfoRequestPB req; - master::GetCDCDBStreamInfoResponsePB resp; - req.set_db_stream_id(db_stream_id); - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->GetCDCDBStreamInfo(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error getting info corresponding to CDC db stream : " << - resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "CDC DB Stream Info: \r\n" << resp.DebugString(); - return Status::OK(); -} - -Status ClusterAdminClient::WaitForSetupUniverseReplicationToFinish(const string& producer_uuid) { - master::IsSetupUniverseReplicationDoneRequestPB req; - req.set_producer_id(producer_uuid); - for (;;) { - master::IsSetupUniverseReplicationDoneResponsePB resp; - RpcController rpc; - rpc.set_timeout(timeout_); - Status s = master_replication_proxy_->IsSetupUniverseReplicationDone(req, &resp, &rpc); - - if (!s.ok() || resp.has_error()) { - LOG(WARNING) << "Encountered error while waiting for setup_universe_replication to complete" - << " : " << (!s.ok() ? s.ToString() : resp.error().status().message()); - } - if (resp.has_done() && resp.done()) { - return StatusFromPB(resp.replication_error()); - } - - // Still processing, wait and then loop again. - std::this_thread::sleep_for(100ms); - } -} - -Status ClusterAdminClient::SetupUniverseReplication( - const string& producer_uuid, const vector& producer_addresses, - const vector& tables, - const vector& producer_bootstrap_ids) { - master::SetupUniverseReplicationRequestPB req; - master::SetupUniverseReplicationResponsePB resp; - req.set_producer_id(producer_uuid); - - req.mutable_producer_master_addresses()->Reserve(narrow_cast(producer_addresses.size())); - for (const auto& addr : producer_addresses) { - // HostPort::FromString() expects a default port. - auto hp = VERIFY_RESULT(HostPort::FromString(addr, master::kMasterDefaultPort)); - HostPortToPB(hp, req.add_producer_master_addresses()); - } - - req.mutable_producer_table_ids()->Reserve(narrow_cast(tables.size())); - for (const auto& table : tables) { - req.add_producer_table_ids(table); - } - - for (const auto& producer_bootstrap_id : producer_bootstrap_ids) { - req.add_producer_bootstrap_ids(producer_bootstrap_id); - } - - RpcController rpc; - rpc.set_timeout(timeout_); - auto setup_result_status = master_replication_proxy_->SetupUniverseReplication(req, &resp, &rpc); - - setup_result_status = WaitForSetupUniverseReplicationToFinish(producer_uuid); - - if (resp.has_error()) { - cout << "Error setting up universe replication: " << resp.error().status().message() << endl; - Status status_from_error = StatusFromPB(resp.error().status()); - - return status_from_error; - } - - // Clean up config files if setup fails to complete. - if (!setup_result_status.ok()) { - cout << "Error waiting for universe replication setup to complete: " - << setup_result_status.message().ToBuffer() << endl; - return setup_result_status; - } - - cout << "Replication setup successfully" << endl; - return Status::OK(); -} - -Status ClusterAdminClient::DeleteUniverseReplication(const std::string& producer_id, - bool ignore_errors) { - master::DeleteUniverseReplicationRequestPB req; - master::DeleteUniverseReplicationResponsePB resp; - req.set_producer_id(producer_id); - req.set_ignore_errors(ignore_errors); - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->DeleteUniverseReplication(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error deleting universe replication: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - if (resp.warnings().size() > 0) { - cout << "Encountered the following warnings while running delete_universe_replication:" << endl; - for (const auto& warning : resp.warnings()) { - cout << " - " << warning.message() << endl; - } - } - - cout << "Replication deleted successfully" << endl; - return Status::OK(); -} - -Status ClusterAdminClient::AlterUniverseReplication(const std::string& producer_uuid, - const std::vector& producer_addresses, - const std::vector& add_tables, - const std::vector& remove_tables, - const std::vector& producer_bootstrap_ids_to_add, - const std::string& new_producer_universe_id, - bool remove_table_ignore_errors) { - master::AlterUniverseReplicationRequestPB req; - master::AlterUniverseReplicationResponsePB resp; - req.set_producer_id(producer_uuid); - req.set_remove_table_ignore_errors(remove_table_ignore_errors); - - if (!producer_addresses.empty()) { - req.mutable_producer_master_addresses()->Reserve(narrow_cast(producer_addresses.size())); - for (const auto& addr : producer_addresses) { - // HostPort::FromString() expects a default port. - auto hp = VERIFY_RESULT(HostPort::FromString(addr, master::kMasterDefaultPort)); - HostPortToPB(hp, req.add_producer_master_addresses()); - } - } - - if (!add_tables.empty()) { - req.mutable_producer_table_ids_to_add()->Reserve(narrow_cast(add_tables.size())); - for (const auto& table : add_tables) { - req.add_producer_table_ids_to_add(table); - } - - if (!producer_bootstrap_ids_to_add.empty()) { - // There msut be a bootstrap id for every table id. - if (producer_bootstrap_ids_to_add.size() != add_tables.size()) { - cout << "The number of bootstrap ids must equal the number of table ids. " - << "Use separate alter commands if only some tables are being bootstrapped." << endl; - return STATUS(InternalError, "Invalid number of bootstrap ids"); - } - - req.mutable_producer_bootstrap_ids_to_add()->Reserve( - narrow_cast(producer_bootstrap_ids_to_add.size())); - for (const auto& bootstrap_id : producer_bootstrap_ids_to_add) { - req.add_producer_bootstrap_ids_to_add(bootstrap_id); - } - } - } - - if (!remove_tables.empty()) { - req.mutable_producer_table_ids_to_remove()->Reserve(narrow_cast(remove_tables.size())); - for (const auto& table : remove_tables) { - req.add_producer_table_ids_to_remove(table); - } - } - - if (!new_producer_universe_id.empty()) { - req.set_new_producer_universe_id(new_producer_universe_id); - } - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->AlterUniverseReplication(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error altering universe replication: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - if (!add_tables.empty()) { - // If we are adding tables, then wait for the altered producer to be deleted (this happens once - // it is merged with the original). - RETURN_NOT_OK(WaitForSetupUniverseReplicationToFinish(producer_uuid + ".ALTER")); - } - - cout << "Replication altered successfully" << endl; - return Status::OK(); -} - -Status ClusterAdminClient::ChangeXClusterRole(cdc::XClusterRole role) { - master::ChangeXClusterRoleRequestPB req; - master::ChangeXClusterRoleResponsePB resp; - req.set_role(role); - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->ChangeXClusterRole(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error changing role: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "Changed role successfully" << endl; - return Status::OK(); -} - -Status ClusterAdminClient::SetUniverseReplicationEnabled(const std::string& producer_id, - bool is_enabled) { - master::SetUniverseReplicationEnabledRequestPB req; - master::SetUniverseReplicationEnabledResponsePB resp; - req.set_producer_id(producer_id); - req.set_is_enabled(is_enabled); - const string toggle = (is_enabled ? "enabl" : "disabl"); - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->SetUniverseReplicationEnabled(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error " << toggle << "ing " - << "universe replication: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "Replication " << toggle << "ed successfully" << endl; - return Status::OK(); -} - -Result ClusterAdminClient::GetFirstRpcAddressForTS() { - RepeatedPtrField servers; - RETURN_NOT_OK(ListTabletServers(&servers)); - for (const ListTabletServersResponsePB::Entry& server : servers) { - if (server.has_registration() && - !server.registration().common().private_rpc_addresses().empty()) { - return HostPortFromPB(server.registration().common().private_rpc_addresses(0)); - } - } - - return STATUS(NotFound, "Didn't find a server registered with the Master"); -} - -Status ClusterAdminClient::BootstrapProducer(const vector& table_ids) { - - HostPort ts_addr = VERIFY_RESULT(GetFirstRpcAddressForTS()); - auto cdc_proxy = std::make_unique(proxy_cache_.get(), ts_addr); - - cdc::BootstrapProducerRequestPB bootstrap_req; - cdc::BootstrapProducerResponsePB bootstrap_resp; - for (const auto& table_id : table_ids) { - bootstrap_req.add_table_ids(table_id); - } - RpcController rpc; - rpc.set_timeout(MonoDelta::FromSeconds(std::max(timeout_.ToSeconds(), 120.0))); - RETURN_NOT_OK(cdc_proxy->BootstrapProducer(bootstrap_req, &bootstrap_resp, &rpc)); - - if (bootstrap_resp.has_error()) { - cout << "Error bootstrapping consumer: " << bootstrap_resp.error().status().message() << endl; - return StatusFromPB(bootstrap_resp.error().status()); - } - - if (implicit_cast(bootstrap_resp.cdc_bootstrap_ids().size()) != table_ids.size()) { - cout << "Received invalid number of bootstrap ids: " << bootstrap_resp.ShortDebugString(); - return STATUS(InternalError, "Invalid number of bootstrap ids"); - } - - int i = 0; - for (const auto& bootstrap_id : bootstrap_resp.cdc_bootstrap_ids()) { - cout << "table id: " << table_ids[i++] << ", CDC bootstrap id: " << bootstrap_id << endl; - } - return Status::OK(); -} - -Status ClusterAdminClient::WaitForReplicationDrain(const std::vector &stream_ids, - const string& target_time) { - master::WaitForReplicationDrainRequestPB req; - master::WaitForReplicationDrainResponsePB resp; - for (const auto& stream_id : stream_ids) { - req.add_stream_ids(stream_id); - } - // If target_time is not provided, it will be set to current time in the master API. - if (!target_time.empty()) { - auto result = HybridTime::ParseHybridTime(target_time); - if (!result.ok()) { - return STATUS(InvalidArgument, "Error parsing target_time: " + result.ToString()); - } - req.set_target_time(result->GetPhysicalValueMicros()); - } - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->WaitForReplicationDrain(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error waiting for replication drain: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - std::unordered_map> undrained_streams; - for (const auto& stream_info : resp.undrained_stream_info()) { - undrained_streams[stream_info.stream_id()].push_back(stream_info.tablet_id()); - } - if (!undrained_streams.empty()) { - cout << "Found undrained replications:" << endl; - for (const auto& stream_to_tablets : undrained_streams) { - cout << "- Under Stream " << stream_to_tablets.first << ":" << endl; - for (const auto& tablet_id : stream_to_tablets.second) { - cout << " - Tablet: " << tablet_id << endl; - } - } - } else { - cout << "All replications are caught-up." << endl; - } - return Status::OK(); -} - -Status ClusterAdminClient::SetupNSUniverseReplication( - const std::string& producer_uuid, - const std::vector& producer_addresses, - const TypedNamespaceName& producer_namespace) { - switch (producer_namespace.db_type) { - case YQL_DATABASE_CQL: - break; - case YQL_DATABASE_PGSQL: - return STATUS(InvalidArgument, - "YSQL not currently supported for namespace-level replication setup"); - default: - return STATUS(InvalidArgument, "Unsupported namespace type"); - } - - master::SetupNSUniverseReplicationRequestPB req; - master::SetupNSUniverseReplicationResponsePB resp; - req.set_producer_id(producer_uuid); - req.set_producer_ns_name(producer_namespace.name); - req.set_producer_ns_type(producer_namespace.db_type); - - req.mutable_producer_master_addresses()->Reserve(narrow_cast(producer_addresses.size())); - for (const auto& addr : producer_addresses) { - auto hp = VERIFY_RESULT(HostPort::FromString(addr, master::kMasterDefaultPort)); - HostPortToPB(hp, req.add_producer_master_addresses()); - } - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->SetupNSUniverseReplication(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error setting up namespace-level universe replication: " - << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << "Namespace-level replication setup successfully" << endl; - return Status::OK(); -} - -Status ClusterAdminClient::GetReplicationInfo( - const std::string& universe_uuid) { - - master::GetReplicationStatusRequestPB req; - master::GetReplicationStatusResponsePB resp; - - if (!universe_uuid.empty()) { - req.set_universe_id(universe_uuid); - } - - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->GetReplicationStatus(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error getting replication status: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - cout << resp.DebugString(); - return Status::OK(); -} - -Result ClusterAdminClient::GetXClusterEstimatedDataLoss() { - master::GetXClusterEstimatedDataLossRequestPB req; - master::GetXClusterEstimatedDataLossResponsePB resp; - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->GetXClusterEstimatedDataLoss(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error getting xCluster estimated data loss values: " << resp.error().status().message() - << endl; - return StatusFromPB(resp.error().status()); - } - - rapidjson::Document document; - document.SetArray(); - for (const auto& data_loss : resp.namespace_data_loss()) { - rapidjson::Value json_entry(rapidjson::kObjectType); - AddStringField("namespace_id", data_loss.namespace_id(), &json_entry, &document.GetAllocator()); - - // Use 1 second granularity. - int64_t data_loss_s = MonoDelta::FromMicroseconds(data_loss.data_loss_us()).ToSeconds(); - AddStringField( - "data_loss_sec", std::to_string(data_loss_s), &json_entry, &document.GetAllocator()); - document.PushBack(json_entry, document.GetAllocator()); - } - - return document; -} - -Result ClusterAdminClient::GetXClusterSafeTime() { - master::GetXClusterSafeTimeRequestPB req; - master::GetXClusterSafeTimeResponsePB resp; - RpcController rpc; - rpc.set_timeout(timeout_); - RETURN_NOT_OK(master_replication_proxy_->GetXClusterSafeTime(req, &resp, &rpc)); - - if (resp.has_error()) { - cout << "Error getting xCluster safe time values: " << resp.error().status().message() << endl; - return StatusFromPB(resp.error().status()); - } - - rapidjson::Document document; - document.SetArray(); - for (const auto& safe_time : resp.namespace_safe_times()) { - rapidjson::Value json_entry(rapidjson::kObjectType); - AddStringField("namespace_id", safe_time.namespace_id(), &json_entry, &document.GetAllocator()); - const auto& st = HybridTime::FromPB(safe_time.safe_time_ht()); - AddStringField("safe_time", HybridTimeToString(st), &json_entry, &document.GetAllocator()); - AddStringField( - "safe_time_epoch", std::to_string(st.GetPhysicalValueMicros()), &json_entry, - &document.GetAllocator()); - document.PushBack(json_entry, document.GetAllocator()); - } - - return document; -} - -} // namespace enterprise -} // namespace tools -} // namespace yb diff --git a/src/yb/integration-tests/cassandra_cpp_driver-test.cc b/src/yb/integration-tests/cassandra_cpp_driver-test.cc index 75a4ee8304da..50430fd29ee1 100644 --- a/src/yb/integration-tests/cassandra_cpp_driver-test.cc +++ b/src/yb/integration-tests/cassandra_cpp_driver-test.cc @@ -1088,7 +1088,7 @@ TEST_F_EX(CppCassandraDriverTest, TestDeferredIndexBackfillsAfterWait, // Launch Backfill through yb-admin constexpr auto kAdminRpcTimeout = 5; - auto yb_admin_client = std::make_unique( + auto yb_admin_client = std::make_unique( cluster_->GetMasterAddresses(), MonoDelta::FromSeconds(kAdminRpcTimeout)); ASSERT_OK(yb_admin_client->Init()); ASSERT_OK(yb_admin_client->LaunchBackfillIndexForTable(table_name)); diff --git a/src/yb/integration-tests/load_balancer_placement_policy-test.cc b/src/yb/integration-tests/load_balancer_placement_policy-test.cc index e5148f0abf41..3c59a3407027 100644 --- a/src/yb/integration-tests/load_balancer_placement_policy-test.cc +++ b/src/yb/integration-tests/load_balancer_placement_policy-test.cc @@ -52,7 +52,7 @@ class LoadBalancerPlacementPolicyTest : public YBTableTestBase { void SetUp() override { YBTableTestBase::SetUp(); - yb_admin_client_ = std::make_unique( + yb_admin_client_ = std::make_unique( external_mini_cluster()->GetMasterAddresses(), kDefaultTimeout); ASSERT_OK(yb_admin_client_->Init()); @@ -169,7 +169,7 @@ class LoadBalancerPlacementPolicyTest : public YBTableTestBase { kDefaultTimeout)); } - std::unique_ptr yb_admin_client_; + std::unique_ptr yb_admin_client_; }; TEST_F(LoadBalancerPlacementPolicyTest, CreateTableWithPlacementPolicyTest) { diff --git a/src/yb/integration-tests/master_failover-itest.cc b/src/yb/integration-tests/master_failover-itest.cc index 25d476fbdc45..74431ee8bf0a 100644 --- a/src/yb/integration-tests/master_failover-itest.cc +++ b/src/yb/integration-tests/master_failover-itest.cc @@ -612,7 +612,7 @@ class MasterFailoverTestWithPlacement : public MasterFailoverTest { opts_.extra_tserver_flags.push_back("--placement_uuid=" + kLivePlacementUuid); opts_.extra_master_flags.push_back("--enable_register_ts_from_raft=true"); MasterFailoverTest::SetUp(); - yb_admin_client_ = std::make_unique( + yb_admin_client_ = std::make_unique( cluster_->GetMasterAddresses(), MonoDelta::FromSeconds(30)); ASSERT_OK(yb_admin_client_->Init()); ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("c.r.z0,c.r.z1,c.r.z2", 3, kLivePlacementUuid)); @@ -636,7 +636,7 @@ class MasterFailoverTestWithPlacement : public MasterFailoverTest { protected: const string kReadReplicaPlacementUuid = "read_replica"; const string kLivePlacementUuid = "live"; - std::unique_ptr yb_admin_client_; + std::unique_ptr yb_admin_client_; }; TEST_F_EX(MasterFailoverTest, TestFailoverWithReadReplicas, MasterFailoverTestWithPlacement) { diff --git a/src/yb/integration-tests/master_path_handlers-itest.cc b/src/yb/integration-tests/master_path_handlers-itest.cc index 129091c14fa2..4316f1abafd2 100644 --- a/src/yb/integration-tests/master_path_handlers-itest.cc +++ b/src/yb/integration-tests/master_path_handlers-itest.cc @@ -366,8 +366,8 @@ TEST_F_EX(MasterPathHandlersItest, TestTablePlacementInfo, MasterPathHandlersExt ASSERT_EQ(result_str.find("live_replicas", pos + 1), string::npos); // Verify cluster level replication info. - auto yb_admin_client_ = std::make_unique( - cluster_->GetMasterAddresses(), MonoDelta::FromSeconds(30)); + auto yb_admin_client_ = std::make_unique( + cluster_->GetMasterAddresses(), MonoDelta::FromSeconds(30)); ASSERT_OK(yb_admin_client_->Init()); ASSERT_OK(yb_admin_client_->ModifyPlacementInfo("cloud.region.zone", 3, "table_uuid")); TestUrl(url, &result); diff --git a/src/yb/integration-tests/yb_table_test_base.cc b/src/yb/integration-tests/yb_table_test_base.cc index 64564b537204..55f187305e28 100644 --- a/src/yb/integration-tests/yb_table_test_base.cc +++ b/src/yb/integration-tests/yb_table_test_base.cc @@ -216,7 +216,7 @@ void YBTableTestBase::CreateAdminClient() { } else { addrs = mini_cluster_->GetMasterAddresses(); } - yb_admin_client_ = std::make_unique( + yb_admin_client_ = std::make_unique( addrs, MonoDelta::FromMilliseconds(client_rpc_timeout_ms())); ASSERT_OK(yb_admin_client_->Init()); diff --git a/src/yb/integration-tests/yb_table_test_base.h b/src/yb/integration-tests/yb_table_test_base.h index fc6d5c2d48a4..c081484013e9 100644 --- a/src/yb/integration-tests/yb_table_test_base.h +++ b/src/yb/integration-tests/yb_table_test_base.h @@ -94,7 +94,7 @@ class YBTableTestBase : public YBTest { client::TableHandle table_; std::unique_ptr client_; - std::unique_ptr yb_admin_client_; + std::unique_ptr yb_admin_client_; bool table_exists_ = false; yb::MiniCluster* mini_cluster() { diff --git a/src/yb/tools/CMakeLists.txt b/src/yb/tools/CMakeLists.txt index f3c40e2b7654..e75a8e2574b1 100644 --- a/src/yb/tools/CMakeLists.txt +++ b/src/yb/tools/CMakeLists.txt @@ -33,8 +33,6 @@ set(YB_PCH_PREFIX tools) set(YB_PCH_DEP_LIBS opid_proto yb_test_util boost_program_options) -YB_INCLUDE_EXTENSIONS() - set(LINK_LIBS yb_client log @@ -59,8 +57,7 @@ target_link_libraries(insert-generated-rows ${LINK_LIBS}) add_executable(yb-admin - yb-admin_cli.cc - ${YB_ADMIN_SRCS_EXTENSIONS}) + yb-admin_cli.cc) target_link_libraries(yb-admin yb-admin_lib master_error @@ -68,8 +65,7 @@ target_link_libraries(yb-admin add_library(yb-admin_lib yb-admin_client.cc - yb-admin_util.cc - ${YB_ADMIN_LIB_SRCS_EXTENSIONS}) + yb-admin_util.cc) target_link_libraries(yb-admin_lib ${LINK_LIBS}) add_executable(yb-ts-cli ts-cli.cc) @@ -174,6 +170,9 @@ target_link_libraries(tools_test_utils yb_util yb_test_util) add_library(test_admin_client test_admin_client.cc) target_link_libraries(test_admin_client master_proto integration-tests) +add_library(yb-admin-test-base yb-admin-test-base.cc) +target_link_libraries(yb-admin-test-base integration-tests ql-dml-test-base) + set(YB_TEST_LINK_LIBS admin-test-base ysck @@ -189,6 +188,7 @@ set(YB_TEST_LINK_LIBS tools_test_utils yb-backup-test_base pg_wrapper_test_base + yb-admin-test-base ${YB_MIN_TEST_LIBS}) ADD_YB_TEST(ysck-test) ADD_YB_TEST(yb-bulk_load-test) @@ -215,17 +215,9 @@ ADD_YB_TEST(data-patcher-test) ADD_YB_TEST_DEPENDENCIES(data-patcher-test data-patcher) ADD_YB_TEST(yb-backup/yb-backup-test) ADD_YB_TEST(yb-backup/yb-backup-cross-feature-test) - - -# Additional tests -if(YB_ENT_CURRENT_SOURCE_DIR) - # Set the test source file folder. - set(CMAKE_CURRENT_LIST_DIR ${YB_ENT_CURRENT_SOURCE_DIR}) - - foreach(test ${TOOLS_EXTENSIONS_TESTS}) - ADD_YB_TEST(${test}) - endforeach() -endif() +ADD_YB_TEST(yb-admin-snapshot-test) +ADD_YB_TEST(yb-admin-xcluster-test) +ADD_YB_TEST(yb-admin_client-test) set(YB_PCH_PREFIX util) add_executable(run-with-timeout run-with-timeout.cc) diff --git a/ent/src/yb/tools/tools_fwd.h b/src/yb/tools/tools_fwd.h similarity index 93% rename from ent/src/yb/tools/tools_fwd.h rename to src/yb/tools/tools_fwd.h index 558372a403c4..917148eb0a8e 100644 --- a/ent/src/yb/tools/tools_fwd.h +++ b/src/yb/tools/tools_fwd.h @@ -15,10 +15,8 @@ namespace yb { namespace tools { -namespace enterprise { class ClusterAdminClient; -} // namespace enterprise } // namespace tools } // namespace yb diff --git a/src/yb/tools/yb-admin-snapshot-test.cc b/src/yb/tools/yb-admin-snapshot-test.cc new file mode 100644 index 000000000000..c9cf7ee000d5 --- /dev/null +++ b/src/yb/tools/yb-admin-snapshot-test.cc @@ -0,0 +1,670 @@ +// Copyright (c) YugaByte, Inc. +// +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. + +#include "yb/tools/yb-admin-test-base.h" +#include "yb/util/flags.h" + +#include "yb/client/client.h" +#include "yb/client/ql-dml-test-base.h" +#include "yb/client/schema.h" +#include "yb/client/table.h" +#include "yb/client/table_info.h" + +#include "yb/integration-tests/external_mini_cluster_ent.h" + +#include "yb/master/master_backup.proxy.h" +#include "yb/master/master_replication.proxy.h" + +#include "yb/rpc/secure_stream.h" + +#include "yb/tools/admin-test-base.h" +#include "yb/tools/yb-admin_util.h" + +#include "yb/tserver/mini_tablet_server.h" +#include "yb/client/table_alterer.h" +#include "yb/tserver/tablet_server.h" + +#include "yb/util/backoff_waiter.h" +#include "yb/util/date_time.h" +#include "yb/util/env_util.h" +#include "yb/util/monotime.h" +#include "yb/util/path_util.h" +#include "yb/util/subprocess.h" +#include "yb/util/tsan_util.h" + +DECLARE_string(certs_dir); +DECLARE_bool(check_bootstrap_required); + +namespace yb { +namespace tools { + +using namespace std::literals; + +using std::shared_ptr; +using std::string; + +using client::Transactional; +using client::YBTable; +using client::YBTableInfo; +using client::YBTableName; +using master::ListSnapshotRestorationsRequestPB; +using master::ListSnapshotRestorationsResponsePB; +using master::ListSnapshotsRequestPB; +using master::ListSnapshotsResponsePB; +using master::MasterBackupProxy; +using master::MasterReplicationProxy; +using master::SysCDCStreamEntryPB; +using master::SysSnapshotEntryPB; +using rpc::RpcController; + +class AdminCliTest : public AdminCliTestBase { + protected: + Result BackupServiceProxy() { + if (!backup_service_proxy_) { + backup_service_proxy_.reset(new MasterBackupProxy( + &client_->proxy_cache(), VERIFY_RESULT(cluster_->GetLeaderMasterBoundRpcAddr()))); + } + return backup_service_proxy_.get(); + } + + Status WaitForRestoreSnapshot() { + return WaitFor( + [this]() -> Result { + const auto document = + VERIFY_RESULT(RunAdminToolCommandJson("list_snapshot_restorations")); + auto it = document.FindMember("restorations"); + if (it == document.MemberEnd()) { + LOG(INFO) << "No restorations"; + return false; + } + auto value = it->value.GetArray(); + for (const auto& restoration : value) { + auto state_it = restoration.FindMember("state"); + if (state_it == restoration.MemberEnd()) { + return STATUS(NotFound, "'state' not found"); + } + if (state_it->value.GetString() != "RESTORED"s) { + return false; + } + } + return true; + }, + 30s, "Waiting for snapshot restore to complete"); + } + + Result WaitForAllSnapshots( + master::MasterBackupProxy* const alternate_proxy = nullptr) { + auto proxy = alternate_proxy; + if (!proxy) { + proxy = VERIFY_RESULT(BackupServiceProxy()); + } + return tools::WaitForAllSnapshots(proxy); + } + + Result GetCompletedSnapshot(int num_snapshots = 1, int idx = 0) { + auto* proxy = VERIFY_RESULT(BackupServiceProxy()); + return tools::GetCompletedSnapshot(proxy, num_snapshots, idx); + } + + Result WaitForRestoration() { + auto* proxy = VERIFY_RESULT(BackupServiceProxy()); + + ListSnapshotRestorationsRequestPB req; + ListSnapshotRestorationsResponsePB resp; + RETURN_NOT_OK(WaitFor( + [proxy, &req, &resp]() -> Result { + RpcController rpc; + RETURN_NOT_OK(proxy->ListSnapshotRestorations(req, &resp, &rpc)); + for (auto const& restoration : resp.restorations()) { + if (restoration.entry().state() == SysSnapshotEntryPB::RESTORING) { + return false; + } + } + return true; + }, + 30s, "Waiting for all restorations to complete")); + + SCHECK_EQ(resp.restorations_size(), 1, IllegalState, "Expected only one restoration"); + return resp.restorations(0).entry().state(); + } + + Result NumTables(const string& table_name) const; + void ImportTableAs(const string& snapshot_file, const string& keyspace, const string& table_name); + void CheckImportedTable( + const YBTable* src_table, const YBTableName& yb_table_name, bool same_ids = false); + void CheckAndDeleteImportedTable( + const string& keyspace, const string& table_name, bool same_ids = false); + void CheckImportedTableWithIndex( + const string& keyspace, const string& table_name, const string& index_name, + bool same_ids = false); + + void DoTestImportSnapshot(const string& format = ""); + void DoTestExportImportIndexSnapshot(Transactional transactional); + + private: + std::unique_ptr backup_service_proxy_; +}; + +TEST_F(AdminCliTest, TestNonTLS) { ASSERT_OK(RunAdminToolCommand("list_all_masters")); } + +// TODO: Enabled once ENG-4900 is resolved. +TEST_F(AdminCliTest, DISABLED_TestTLS) { + ASSERT_OK(RunAdminToolCommand("--certs_dir_name", GetCertsDir(), "list_all_masters")); +} + +TEST_F(AdminCliTest, TestCreateSnapshot) { + CreateTable(Transactional::kFalse); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + // There is custom table. + const auto tables = ASSERT_RESULT(client_->ListTables(table_name, /* exclude_ysql */ true)); + ASSERT_EQ(1, tables.size()); + + ListSnapshotsRequestPB req; + ListSnapshotsResponsePB resp; + RpcController rpc; + ASSERT_OK(ASSERT_RESULT(BackupServiceProxy())->ListSnapshots(req, &resp, &rpc)); + ASSERT_EQ(resp.snapshots_size(), 0); + + // Create snapshot of default table that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + + rpc.Reset(); + ASSERT_OK(ASSERT_RESULT(BackupServiceProxy())->ListSnapshots(req, &resp, &rpc)); + ASSERT_EQ(resp.snapshots_size(), 1); + + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +Result AdminCliTest::NumTables(const string& table_name) const { + auto tables = + VERIFY_RESULT(client_->ListTables(/* filter */ table_name, /* exclude_ysql */ true)); + return tables.size(); +} + +void AdminCliTest::CheckImportedTable( + const YBTable* src_table, const YBTableName& yb_table_name, bool same_ids) { + shared_ptr table; + ASSERT_OK(client_->OpenTable(yb_table_name, &table)); + + ASSERT_EQ(same_ids, table->id() == src_table->id()); + ASSERT_EQ(table->table_type(), src_table->table_type()); + ASSERT_EQ(table->GetPartitionsCopy(), src_table->GetPartitionsCopy()); + ASSERT_TRUE(table->partition_schema().Equals(src_table->partition_schema())); + ASSERT_TRUE(table->schema().Equals(src_table->schema())); + ASSERT_EQ( + table->schema().table_properties().is_transactional(), + src_table->schema().table_properties().is_transactional()); +} + +void AdminCliTest::CheckAndDeleteImportedTable( + const string& keyspace, const string& table_name, bool same_ids) { + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + + const YBTableName yb_table_name(YQL_DATABASE_CQL, keyspace, table_name); + CheckImportedTable(table_.get(), yb_table_name, same_ids); + ASSERT_EQ(1, ASSERT_RESULT(NumTables(table_name))); + ASSERT_OK(client_->DeleteTable(yb_table_name, /* wait */ true)); + ASSERT_EQ(0, ASSERT_RESULT(NumTables(table_name))); +} + +void AdminCliTest::ImportTableAs( + const string& snapshot_file, const string& keyspace, const string& table_name) { + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, keyspace, table_name)); + CheckAndDeleteImportedTable(keyspace, table_name); +} + +void AdminCliTest::DoTestImportSnapshot(const string& format) { + CreateTable(Transactional::kFalse); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + // Create snapshot of default table that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + + string tmp_dir; + ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); + const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_snapshot.dat"); + + if (format.empty()) { + ASSERT_OK(RunAdminToolCommand("export_snapshot", snapshot_id, snapshot_file)); + } else { + ASSERT_OK(RunAdminToolCommand( + "export_snapshot", snapshot_id, snapshot_file, + "-TEST_metadata_file_format_version=" + format)); + } + + // Import snapshot into the existing table. + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); + CheckAndDeleteImportedTable(keyspace, table_name, /* same_ids */ true); + + // Import snapshot into original table from the snapshot. + // (The table was deleted by the call above.) + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); + CheckAndDeleteImportedTable(keyspace, table_name); + + // Import snapshot into non existing namespace. + ImportTableAs(snapshot_file, keyspace + "_new", table_name); + // Import snapshot into already existing namespace. + ImportTableAs(snapshot_file, keyspace, table_name + "_new"); + // Import snapshot into already existing namespace and table. + ImportTableAs(snapshot_file, keyspace, table_name); +} + +TEST_F(AdminCliTest, TestImportSnapshot) { + DoTestImportSnapshot(); + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +TEST_F(AdminCliTest, TestImportSnapshotInOldFormat1) { + DoTestImportSnapshot("1"); + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +TEST_F(AdminCliTest, TestImportSnapshotInOldFormatNoNamespaceName) { + DoTestImportSnapshot("-1"); + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +TEST_F(AdminCliTest, TestExportImportSnapshot) { + CreateTable(Transactional::kFalse); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + // Create snapshot of default table that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + + string tmp_dir; + ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); + const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_snapshot.dat"); + ASSERT_OK(RunAdminToolCommand("export_snapshot", snapshot_id, snapshot_file)); + // Import below will not create a new table - reusing the old one. + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, keyspace, table_name)); + + const YBTableName yb_table_name(YQL_DATABASE_CQL, keyspace, table_name); + CheckImportedTable(table_.get(), yb_table_name, /* same_ids */ true); + ASSERT_EQ(1, ASSERT_RESULT(NumTables(table_name))); + + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +TEST_F(AdminCliTest, TestRestoreSnapshotBasic) { + CreateTable(Transactional::kFalse); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + ASSERT_OK(WriteRow(CreateSession(), 1, 1)); + + // Create snapshot of default table that gets created. + LOG(INFO) << "Creating snapshot"; + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + ASSERT_RESULT(WaitForAllSnapshots()); + + ASSERT_OK(DeleteRow(CreateSession(), 1)); + ASSERT_NOK(SelectRow(CreateSession(), 1)); + + // Restore snapshot into the existing table. + LOG(INFO) << "Restoring snapshot"; + ASSERT_OK(RunAdminToolCommand("restore_snapshot", snapshot_id)); + ASSERT_OK(WaitForRestoreSnapshot()); + LOG(INFO) << "Restored snapshot"; + + ASSERT_OK(WaitFor( + [&]() -> Result { return SelectRow(CreateSession(), 1).ok(); }, 20s, + "Waiting for row from restored snapshot.")); +} + +TEST_F(AdminCliTest, TestRestoreSnapshotHybridTime) { + CreateTable(Transactional::kFalse); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + ASSERT_OK(WriteRow(CreateSession(), 1, 1)); + auto hybrid_time = cluster_->mini_tablet_server(0)->server()->Clock()->Now(); + ASSERT_OK(WriteRow(CreateSession(), 2, 2)); + + // Create snapshot of default table that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + ASSERT_RESULT(WaitForAllSnapshots()); + + // Restore snapshot into the existing table. + ASSERT_OK(RunAdminToolCommand( + "restore_snapshot", snapshot_id, std::to_string(hybrid_time.GetPhysicalValueMicros()))); + ASSERT_OK(WaitForRestoreSnapshot()); + + // Row before HybridTime present, row after should be missing now. + ASSERT_OK(WaitFor( + [&]() -> Result { + return SelectRow(CreateSession(), 1).ok() && !SelectRow(CreateSession(), 2).ok(); + }, + 20s, "Waiting for row from restored snapshot.")); +} + +TEST_F(AdminCliTest, TestRestoreSnapshotTimestamp) { + CreateTable(Transactional::kFalse); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + ASSERT_OK(WriteRow(CreateSession(), 1, 1)); + auto timestamp = DateTime::TimestampToString(DateTime::TimestampNow()); + LOG(INFO) << "Timestamp: " << timestamp; + auto write_wait = 2s; + std::this_thread::sleep_for(write_wait); + ASSERT_OK(WriteRow(CreateSession(), 2, 2)); + + // Create snapshot of default table that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + ASSERT_RESULT(WaitForAllSnapshots()); + + // Restore snapshot into the existing table. + ASSERT_OK(RunAdminToolCommand("restore_snapshot", snapshot_id, timestamp)); + ASSERT_OK(WaitForRestoreSnapshot()); + + // Row before Timestamp present, row after should be missing now. + ASSERT_OK(WaitFor( + [&]() -> Result { + return SelectRow(CreateSession(), 1).ok() && !SelectRow(CreateSession(), 2).ok(); + }, + 20s, "Waiting for row from restored snapshot.")); +} + +TEST_F(AdminCliTest, TestRestoreSnapshotInterval) { + CreateTable(Transactional::kFalse); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + auto clock = cluster_->mini_tablet_server(0)->server()->Clock(); + ASSERT_OK(WriteRow(CreateSession(), 1, 1)); + auto pre_sleep_ht = clock->Now(); + auto write_wait = 5s; + std::this_thread::sleep_for(write_wait); + ASSERT_OK(WriteRow(CreateSession(), 2, 2)); + + // Create snapshot of default table that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + ASSERT_RESULT(WaitForAllSnapshots()); + + // Restore snapshot into the existing table. + auto restore_ht = clock->Now(); + auto interval = restore_ht.GetPhysicalValueMicros() - pre_sleep_ht.GetPhysicalValueMicros(); + auto i_str = std::to_string(interval / 1000000) + "s"; + ASSERT_OK(RunAdminToolCommand("restore_snapshot", snapshot_id, "minus", i_str)); + ASSERT_OK(WaitForRestoreSnapshot()); + + ASSERT_OK(SelectRow(CreateSession(), 1)); + auto select2 = SelectRow(CreateSession(), 2); + ASSERT_NOK(select2); +} + +void AdminCliTest::CheckImportedTableWithIndex( + const string& keyspace, const string& table_name, const string& index_name, bool same_ids) { + const YBTableName yb_table_name(YQL_DATABASE_CQL, keyspace, table_name); + const YBTableName yb_index_name(YQL_DATABASE_CQL, keyspace, index_name); + + CheckImportedTable(table_.get(), yb_table_name, same_ids); + ASSERT_EQ(2, ASSERT_RESULT(NumTables(table_name))); + CheckImportedTable(index_.get(), yb_index_name, same_ids); + ASSERT_EQ(1, ASSERT_RESULT(NumTables(index_name))); + + YBTableInfo table_info = ASSERT_RESULT(client_->GetYBTableInfo(yb_table_name)); + YBTableInfo index_info = ASSERT_RESULT(client_->GetYBTableInfo(yb_index_name)); + // Check index ---> table relation. + ASSERT_EQ(index_info.index_info->indexed_table_id(), table_info.table_id); + // Check table ---> index relation. + ASSERT_EQ(table_info.index_map.size(), 1); + ASSERT_EQ(table_info.index_map.count(index_info.table_id), 1); + ASSERT_EQ(table_info.index_map.begin()->first, index_info.table_id); + ASSERT_EQ(table_info.index_map.begin()->second.table_id(), index_info.table_id); + ASSERT_EQ(table_info.index_map.begin()->second.indexed_table_id(), table_info.table_id); + + ASSERT_OK(client_->DeleteTable(yb_table_name, /* wait */ true)); + ASSERT_EQ(0, ASSERT_RESULT(NumTables(table_name))); +} + +void AdminCliTest::DoTestExportImportIndexSnapshot(Transactional transactional) { + CreateTable(transactional); + CreateIndex(transactional); + + // Default tables that were created. + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + const string& index_name = index_.name().table_name(); + const YBTableName yb_table_name(YQL_DATABASE_CQL, keyspace, table_name); + const YBTableName yb_index_name(YQL_DATABASE_CQL, keyspace, index_name); + + // Check there are 2 tables. + ASSERT_EQ(2, ASSERT_RESULT(NumTables(table_name))); + + // Create snapshot of default table and the attached index that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + + string tmp_dir; + ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); + const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_snapshot.dat"); + ASSERT_OK(RunAdminToolCommand("export_snapshot", snapshot_id, snapshot_file)); + + // Import table and index into the existing table and index. + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex(keyspace, table_name, index_name, /* same_ids */ true); + + // Import table and index with original names - not providing any names. + // (The table was deleted by the call above.) + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex(keyspace, table_name, index_name); + + // Import table and index with original names - using the old names. + ASSERT_OK( + RunAdminToolCommand("import_snapshot", snapshot_file, keyspace, table_name, index_name)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex(keyspace, table_name, index_name); + + // Import table and index with original names - providing only old table name. + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, keyspace, table_name)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex(keyspace, table_name, index_name); + + // Renaming table and index, but keeping the same keyspace. + ASSERT_OK(RunAdminToolCommand( + "import_snapshot", snapshot_file, keyspace, "new_" + table_name, "new_" + index_name)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex(keyspace, "new_" + table_name, "new_" + index_name); + + // Keeping the same table and index names, but renaming the keyspace. + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, "new_" + keyspace)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex("new_" + keyspace, table_name, index_name); + + // Repeat previous keyspace renaming case, but pass explicitly the same table name + // (and skip index name). + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file, "new_" + keyspace, table_name)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex("new_" + keyspace, table_name, index_name); + + // Import table and index into a new keyspace with old table and index names. + ASSERT_OK(RunAdminToolCommand( + "import_snapshot", snapshot_file, "new_" + keyspace, table_name, index_name)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex("new_" + keyspace, table_name, index_name); + + // Rename only index and keyspace, but keep the main table name. + ASSERT_OK(RunAdminToolCommand( + "import_snapshot", snapshot_file, "new_" + keyspace, table_name, "new_" + index_name)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex("new_" + keyspace, table_name, "new_" + index_name); + + // Import table and index with renaming into a new keyspace. + ASSERT_OK(RunAdminToolCommand( + "import_snapshot", snapshot_file, "new_" + keyspace, "new_" + table_name, + "new_" + index_name)); + // Wait for the new snapshot completion. + ASSERT_RESULT(WaitForAllSnapshots()); + CheckImportedTableWithIndex("new_" + keyspace, "new_" + table_name, "new_" + index_name); + + // Renaming table only, no new name for the index - expecting error. + ASSERT_NOK(RunAdminToolCommand("import_snapshot", snapshot_file, keyspace, "new_" + table_name)); + ASSERT_NOK(RunAdminToolCommand( + "import_snapshot", snapshot_file, "new_" + keyspace, "new_" + table_name)); +} + +TEST_F(AdminCliTest, TestExportImportIndexSnapshot) { + // Test non-transactional table. + DoTestExportImportIndexSnapshot(Transactional::kFalse); + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +TEST_F(AdminCliTest, TestExportImportIndexSnapshot_ForTransactional) { + // Test the recreated transactional table. + DoTestExportImportIndexSnapshot(Transactional::kTrue); + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +TEST_F(AdminCliTest, TestFailedRestoration) { + CreateTable(Transactional::kTrue); + const string& table_name = table_.name().table_name(); + const string& keyspace = table_.name().namespace_name(); + + // Create snapshot of default table that gets created. + ASSERT_OK(RunAdminToolCommand("create_snapshot", keyspace, table_name)); + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot()); + LOG(INFO) << "Created snapshot: " << snapshot_id; + + string tmp_dir; + ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); + const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_snapshot.dat"); + ASSERT_OK(RunAdminToolCommand("export_snapshot", snapshot_id, snapshot_file)); + // Import below will not create a new table - reusing the old one. + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); + + const YBTableName yb_table_name(YQL_DATABASE_CQL, keyspace, table_name); + CheckImportedTable(table_.get(), yb_table_name, /* same_ids */ true); + ASSERT_EQ(1, ASSERT_RESULT(NumTables(table_name))); + + auto new_snapshot_id = ASSERT_RESULT(GetCompletedSnapshot(2)); + if (new_snapshot_id == snapshot_id) { + new_snapshot_id = ASSERT_RESULT(GetCompletedSnapshot(2, 1)); + } + LOG(INFO) << "Imported snapshot: " << new_snapshot_id; + + ASSERT_OK(RunAdminToolCommand("restore_snapshot", new_snapshot_id)); + + const SysSnapshotEntryPB::State state = ASSERT_RESULT(WaitForRestoration()); + LOG(INFO) << "Restoration: " << SysSnapshotEntryPB::State_Name(state); + ASSERT_EQ(state, SysSnapshotEntryPB::FAILED); + + LOG(INFO) << "Test finished: " << CURRENT_TEST_CASE_AND_TEST_NAME_STR(); +} + +TEST_F(AdminCliTest, TestSetPreferredZone) { + const std::string c1z1 = "c1.r1.z1"; + const std::string c1z2 = "c1.r1.z2"; + const std::string c2z1 = "c2.r1.z1"; + const std::string c1z1_json = + "{\"placementCloud\":\"c1\",\"placementRegion\":\"r1\",\"placementZone\":\"z1\"}"; + const std::string c1z2_json = + "{\"placementCloud\":\"c1\",\"placementRegion\":\"r1\",\"placementZone\":\"z2\"}"; + const std::string c2z1_json = + "{\"placementCloud\":\"c2\",\"placementRegion\":\"r1\",\"placementZone\":\"z1\"}"; + const std::string affinitized_leaders_json_Start = "\"affinitizedLeaders\""; + const std::string multi_affinitized_leaders_json_start = + "\"multiAffinitizedLeaders\":[{\"zones\":["; + const std::string json_end = "]}]"; + + ASSERT_OK(RunAdminToolCommand( + "modify_placement_info", strings::Substitute("$0,$1,$2", c1z1, c1z2, c2z1), 5, "")); + + ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", "")); + auto output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); + ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); + ASSERT_EQ(output.find(multi_affinitized_leaders_json_start), string::npos); + + ASSERT_OK(RunAdminToolCommand("set_preferred_zones", c1z1)); + output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); + ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); + ASSERT_NE(output.find(multi_affinitized_leaders_json_start + c1z1_json + json_end), string::npos); + + ASSERT_OK(RunAdminToolCommand("set_preferred_zones", c1z1, c1z2, c2z1)); + output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); + ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); + ASSERT_NE( + output.find( + multi_affinitized_leaders_json_start + c1z1_json + "," + c1z2_json + "," + c2z1_json + + json_end), + string::npos); + + ASSERT_OK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:1", c1z1))); + output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); + ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); + ASSERT_NE(output.find(multi_affinitized_leaders_json_start + c1z1_json + json_end), string::npos); + + ASSERT_OK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:1", c1z1), c1z2)); + output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); + ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); + ASSERT_NE( + output.find(multi_affinitized_leaders_json_start + c1z1_json + "," + c1z2_json + json_end), + string::npos); + + ASSERT_OK(RunAdminToolCommand( + "set_preferred_zones", strings::Substitute("$0:1", c1z1), strings::Substitute("$0:2", c1z2), + strings::Substitute("$0:3", c2z1))); + output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); + ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); + ASSERT_NE( + output.find( + multi_affinitized_leaders_json_start + c1z1_json + "]},{\"zones\":[" + c1z2_json + + "]},{\"zones\":[" + c2z1_json + json_end), + string::npos); + + ASSERT_OK(RunAdminToolCommand( + "set_preferred_zones", strings::Substitute("$0:1", c1z1), strings::Substitute("$0:1", c1z2), + strings::Substitute("$0:2", c2z1))); + output = ASSERT_RESULT(RunAdminToolCommand("get_universe_config")); + ASSERT_EQ(output.find(affinitized_leaders_json_Start), string::npos); + ASSERT_NE( + output.find( + multi_affinitized_leaders_json_start + c1z1_json + "," + c1z2_json + "]},{\"zones\":[" + + c2z1_json + json_end), + string::npos); + + ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:", c1z1))); + ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:0", c1z1))); + ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:-13", c1z1))); + ASSERT_NOK(RunAdminToolCommand("set_preferred_zones", strings::Substitute("$0:2", c1z1))); + ASSERT_NOK(RunAdminToolCommand( + "set_preferred_zones", strings::Substitute("$0:1", c1z1), strings::Substitute("$0:3", c1z2))); + ASSERT_NOK(RunAdminToolCommand( + "set_preferred_zones", strings::Substitute("$0:2", c1z1), strings::Substitute("$0:2", c1z2), + strings::Substitute("$0:3", c2z1))); +} + +} // namespace tools +} // namespace yb diff --git a/src/yb/tools/yb-admin-test-base.cc b/src/yb/tools/yb-admin-test-base.cc new file mode 100644 index 000000000000..83f602c26089 --- /dev/null +++ b/src/yb/tools/yb-admin-test-base.cc @@ -0,0 +1,55 @@ +// Copyright (c) YugaByte, Inc. +// +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. + +#include "yb/tools/yb-admin-test-base.h" + +#include "yb/master/master_backup.pb.h" +#include "yb/master/master_backup.proxy.h" +#include "yb/tools/yb-admin_util.h" +#include "yb/util/backoff_waiter.h" + +namespace yb { +namespace tools { + +Result WaitForAllSnapshots( + master::MasterBackupProxy* const proxy) { + master::ListSnapshotsRequestPB req; + master::ListSnapshotsResponsePB resp; + RETURN_NOT_OK(WaitFor( + [proxy, &req, &resp]() -> Result { + rpc::RpcController rpc; + RETURN_NOT_OK(proxy->ListSnapshots(req, &resp, &rpc)); + for (auto const& snapshot : resp.snapshots()) { + if (snapshot.entry().state() != master::SysSnapshotEntryPB::COMPLETE) { + return false; + } + } + return true; + }, + 30s, "Waiting for all snapshots to complete")); + return resp; +} + +Result GetCompletedSnapshot( + master::MasterBackupProxy* const proxy, int num_snapshots, int idx) { + auto resp = VERIFY_RESULT(WaitForAllSnapshots(proxy)); + + if (resp.snapshots_size() != num_snapshots) { + return STATUS_FORMAT(Corruption, "Wrong snapshot count $0", resp.snapshots_size()); + } + + return SnapshotIdToString(resp.snapshots(idx).id()); +} + +} // namespace tools +} // namespace yb diff --git a/src/yb/tools/yb-admin-test-base.h b/src/yb/tools/yb-admin-test-base.h new file mode 100644 index 000000000000..27eb061a5d88 --- /dev/null +++ b/src/yb/tools/yb-admin-test-base.h @@ -0,0 +1,60 @@ +// Copyright (c) YugaByte, Inc. +// +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "yb/client/ql-dml-test-base.h" +#include "yb/tools/admin-test-base.h" + +namespace yb { +namespace tools { + +template +Result RunAdminToolCommand(MiniCluster* cluster, Args&&... args) { + return tools::RunAdminToolCommand(cluster->GetMasterAddresses(), std::forward(args)...); +} + +class AdminCliTestBase : public client::KeyValueTableTest { + protected: + template + Result RunAdminToolCommand(Args&&... args) { + return tools::RunAdminToolCommand(cluster_.get(), std::forward(args)...); + } + + template + Status RunAdminToolCommandAndGetErrorOutput(std::string* error_msg, Args&&... args) { + auto command = ToStringVector( + GetToolPath("yb-admin"), "-master_addresses", cluster_->GetMasterAddresses(), + "--never_fsync=true", std::forward(args)...); + LOG(INFO) << "Run tool: " << AsString(command); + return Subprocess::Call(command, error_msg, StdFdTypes{StdFdType::kErr}); + } + + template + Result RunAdminToolCommandJson(Args&&... args) { + auto raw = VERIFY_RESULT(RunAdminToolCommand(std::forward(args)...)); + rapidjson::Document result; + if (result.Parse(raw.c_str(), raw.length()).HasParseError()) { + return STATUS_FORMAT( + InvalidArgument, "Failed to parse json output $0: $1", result.GetParseError(), raw); + } + return result; + } +}; + +Result WaitForAllSnapshots(master::MasterBackupProxy* const proxy); + +Result GetCompletedSnapshot( + master::MasterBackupProxy* const proxy, int num_snapshots = 1, int idx = 0); +} // namespace tools +} // namespace yb diff --git a/src/yb/tools/yb-admin-xcluster-test.cc b/src/yb/tools/yb-admin-xcluster-test.cc new file mode 100644 index 000000000000..9fb027aa90ca --- /dev/null +++ b/src/yb/tools/yb-admin-xcluster-test.cc @@ -0,0 +1,905 @@ +// Copyright (c) YugaByte, Inc. +// +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. + +#include "yb/tools/yb-admin-test-base.h" +#include "yb/util/flags.h" + +#include "yb/client/client.h" +#include "yb/client/ql-dml-test-base.h" +#include "yb/client/schema.h" +#include "yb/client/table.h" +#include "yb/client/table_info.h" + +#include "yb/integration-tests/external_mini_cluster_ent.h" + +#include "yb/master/master_backup.proxy.h" +#include "yb/master/master_replication.proxy.h" + +#include "yb/rpc/secure_stream.h" + +#include "yb/tools/admin-test-base.h" +#include "yb/tools/yb-admin_util.h" + +#include "yb/tserver/mini_tablet_server.h" +#include "yb/client/table_alterer.h" +#include "yb/tserver/tablet_server.h" + +#include "yb/util/backoff_waiter.h" +#include "yb/util/date_time.h" +#include "yb/util/env_util.h" +#include "yb/util/monotime.h" +#include "yb/util/path_util.h" +#include "yb/util/subprocess.h" +#include "yb/util/tsan_util.h" + +DECLARE_uint64(TEST_yb_inbound_big_calls_parse_delay_ms); +DECLARE_int64(rpc_throttle_threshold_bytes); +DECLARE_bool(parallelize_bootstrap_producer); +DECLARE_bool(check_bootstrap_required); + +namespace yb { +namespace tools { + +using namespace std::literals; + +using std::string; + +using client::Transactional; +using client::YBTableName; +using master::ListSnapshotsResponsePB; +using master::MasterBackupProxy; +using master::MasterReplicationProxy; +using master::SysCDCStreamEntryPB; +using rpc::RpcController; + +namespace { +Result GetRecentStreamId(MiniCluster* cluster, TabletId target_table_id = "") { + // Return the first stream with tablet_id matching target_table_id using ListCDCStreams. + // If target_table_id is not specified, return the first stream. + const int kStreamUuidLength = 32; + string output = VERIFY_RESULT(RunAdminToolCommand(cluster, "list_cdc_streams")); + const string find_stream_id = "stream_id: \""; + const string find_table_id = "table_id: \""; + string target_stream_id; + + string::size_type stream_id_pos = output.find(find_stream_id); + string::size_type table_id_pos = output.find(find_table_id); + while (stream_id_pos != string::npos && table_id_pos != string::npos) { + string stream_id = output.substr(stream_id_pos + find_stream_id.size(), kStreamUuidLength); + string table_id = output.substr(table_id_pos + find_table_id.size(), kStreamUuidLength); + if (target_table_id.empty() || table_id == target_table_id) { + target_stream_id = stream_id; + break; + } + stream_id_pos = output.find(find_stream_id, stream_id_pos + kStreamUuidLength); + table_id_pos = output.find(find_table_id, table_id_pos + kStreamUuidLength); + } + return target_stream_id; +} +} // namespace + +// Configures two clusters with clients for the producer and consumer side of xcluster replication. +class XClusterAdminCliTest : public AdminCliTestBase { + public: + virtual int num_tablet_servers() { return 3; } + + void SetUp() override { + // Setup the default cluster as the consumer cluster. + AdminCliTestBase::SetUp(); + // Only create a table on the consumer, producer table may differ in tests. + CreateTable(Transactional::kTrue); + FLAGS_check_bootstrap_required = false; + + // Create the producer cluster. + opts.num_tablet_servers = num_tablet_servers(); + opts.cluster_id = kProducerClusterId; + producer_cluster_ = std::make_unique(opts); + ASSERT_OK(producer_cluster_->StartSync()); + ASSERT_OK(producer_cluster_->WaitForTabletServerCount(num_tablet_servers())); + producer_cluster_client_ = ASSERT_RESULT(producer_cluster_->CreateClient()); + } + + void DoTearDown() override { + if (producer_cluster_) { + producer_cluster_->Shutdown(); + } + AdminCliTestBase::DoTearDown(); + } + + template + Result RunAdminToolCommandOnProducer(Args&&... args) { + return tools::RunAdminToolCommand(producer_cluster_.get(), std::forward(args)...); + } + + Status WaitForSetupUniverseReplicationCleanUp(string producer_uuid) { + auto proxy = std::make_shared( + &client_->proxy_cache(), VERIFY_RESULT(cluster_->GetLeaderMiniMaster())->bound_rpc_addr()); + + master::GetUniverseReplicationRequestPB req; + master::GetUniverseReplicationResponsePB resp; + return WaitFor( + [proxy, &req, &resp, producer_uuid]() -> Result { + req.set_producer_id(producer_uuid); + RpcController rpc; + Status s = proxy->GetUniverseReplication(req, &resp, &rpc); + + return resp.has_error(); + }, + 20s, "Waiting for universe to delete"); + } + + protected: + Status CheckTableIsBeingReplicated( + const std::vector& tables, + SysCDCStreamEntryPB::State target_state = SysCDCStreamEntryPB::ACTIVE) { + string output = VERIFY_RESULT(RunAdminToolCommandOnProducer("list_cdc_streams")); + string state_search_str = + Format("value: \"$0\"", SysCDCStreamEntryPB::State_Name(target_state)); + + for (const auto& table_id : tables) { + // Ensure a stream object with table_id exists. + size_t table_id_pos = output.find(table_id); + if (table_id_pos == string::npos) { + return STATUS_FORMAT(NotFound, "Table id '$0' not found in output: $1", table_id, output); + } + + // Ensure that the strem object has the expected state value. + size_t state_pos = output.find(state_search_str, table_id_pos); + if (state_pos == string::npos) { + return STATUS_FORMAT( + NotFound, "Table id '$0' has the incorrect state value in output: $1", table_id, + output); + } + + // Ensure that the state value we captured earlier did not belong + // to different stream object. + size_t next_stream_obj_pos = output.find("streams {", table_id_pos); + if (next_stream_obj_pos != string::npos && next_stream_obj_pos <= state_pos) { + return STATUS_FORMAT( + NotFound, "Table id '$0' has no state value in output: $1", table_id, output); + } + } + return Status::OK(); + } + + Result ProducerBackupServiceProxy() { + if (!producer_backup_service_proxy_) { + producer_backup_service_proxy_.reset(new MasterBackupProxy( + &producer_cluster_client_->proxy_cache(), + VERIFY_RESULT(producer_cluster_->GetLeaderMasterBoundRpcAddr()))); + } + return producer_backup_service_proxy_.get(); + } + + const string kProducerClusterId = "producer"; + std::unique_ptr producer_cluster_client_; + std::unique_ptr producer_cluster_; + MiniClusterOptions opts; + + private: + std::unique_ptr producer_backup_service_proxy_; +}; + +TEST_F(XClusterAdminCliTest, TestSetupUniverseReplication) { + client::TableHandle producer_cluster_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); + + // Setup universe replication, this should only return once complete. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id())); + + // Check that the stream was properly created for this table. + ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); + + // Delete this universe so shutdown can proceed. + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); +} + +TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationChecksForColumnIdMismatch) { + client::TableHandle producer_table; + client::TableHandle consumer_table; + const YBTableName table_name(YQL_DATABASE_CQL, "my_keyspace", "column_id_mismatch_test_table"); + + client::kv_table_test::CreateTable( + Transactional::kTrue, + NumTablets(), + producer_cluster_client_.get(), + &producer_table, + table_name); + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), client_.get(), &consumer_table, table_name); + + // Drop a column from the consumer table. + { + auto table_alterer = client_.get()->NewTableAlterer(table_name); + ASSERT_OK(table_alterer->DropColumn(kValueColumn)->Alter()); + } + + // Add the same column back into the producer table. This results in a schema mismatch + // between the producer and consumer versions of the table. + { + auto table_alterer = client_.get()->NewTableAlterer(table_name); + table_alterer->AddColumn(kValueColumn)->Type(INT32); + ASSERT_OK(table_alterer->timeout(MonoDelta::FromSeconds(60 * kTimeMultiplier))->Alter()); + } + + // Try setting up replication, this should fail due to the schema mismatch. + ASSERT_NOK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + + producer_table->id())); + + // Make a snapshot of the producer table. + auto timestamp = DateTime::TimestampToString(DateTime::TimestampNow()); + auto producer_backup_proxy = ASSERT_RESULT(ProducerBackupServiceProxy()); + ASSERT_OK(RunAdminToolCommandOnProducer( + "create_snapshot", producer_table.name().namespace_name(), + producer_table.name().table_name())); + + const auto snapshot_id = ASSERT_RESULT(GetCompletedSnapshot(producer_backup_proxy, 1, 0)); + ASSERT_RESULT(WaitForAllSnapshots(producer_backup_proxy)); + + string tmp_dir; + ASSERT_OK(Env::Default()->GetTestDirectory(&tmp_dir)); + const auto snapshot_file = JoinPathSegments(tmp_dir, "exported_producer_snapshot.dat"); + ASSERT_OK(RunAdminToolCommandOnProducer("export_snapshot", snapshot_id, snapshot_file)); + + // Delete consumer table, then import snapshot of producer table into the existing + // consumer table. This should fix the schema mismatch issue. + ASSERT_OK(client_->DeleteTable(table_name, /* wait */ true)); + ASSERT_OK(RunAdminToolCommand("import_snapshot", snapshot_file)); + + // Try running SetupUniverseReplication again, this time it should succeed. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_table->id())); + + // Delete this universe so shutdown can proceed. + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); +} + +TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationFailsWithInvalidSchema) { + client::TableHandle producer_cluster_table; + + // Create a table with a different schema on the producer. + client::kv_table_test::CreateTable( + Transactional::kFalse, // Results in different schema! + NumTablets(), + producer_cluster_client_.get(), + &producer_cluster_table); + + // Try to setup universe replication, should return with a useful error. + string error_msg; + // First provide a non-existant table id. + // ASSERT_NOK since this should fail. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id() + "-BAD")); + + // Wait for the universe to be cleaned up + ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); + + // Now try with the correct table id. + // Note that SetupUniverseReplication should call DeleteUniverseReplication to + // clean up the environment on failure, so we don't need to explicitly call + // DeleteUniverseReplication here. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id())); + + // Verify that error message has relevant information. + ASSERT_TRUE(error_msg.find("Source and target schemas don't match") != string::npos); +} + +TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationFailsWithInvalidBootstrapId) { + client::TableHandle producer_cluster_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); + + // Try to setup universe replication with a fake bootstrap id, should return with a useful error. + string error_msg; + // ASSERT_NOK since this should fail. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id(), + "fake-bootstrap-id")); + + // Verify that error message has relevant information. + ASSERT_TRUE( + error_msg.find("Could not find CDC stream: stream_id: \"fake-bootstrap-id\"") != + string::npos); +} + +TEST_F(XClusterAdminCliTest, TestSetupUniverseReplicationCleanupOnFailure) { + client::TableHandle producer_cluster_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); + + string error_msg; + // Try to setup universe replication with a fake bootstrap id, should result in failure. + // ASSERT_NOK since this should fail. We should be able to make consecutive calls to + // SetupUniverseReplication without having to call DeleteUniverseReplication first. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id(), + "fake-bootstrap-id")); + + // Wait for the universe to be cleaned up + ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); + + // Try to setup universe replication with fake producer master address. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + "fake-producer-address", + producer_cluster_table->id())); + + // Wait for the universe to be cleaned up + ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); + + // Try to setup universe replication with fake producer table id. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + "fake-producer-table-id")); + + // Wait for the universe to be cleaned up + ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); + + // Test when producer and local table have different schema. + client::TableHandle producer_cluster_table2; + client::TableHandle consumer_table2; + const YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "different_schema_test_table"); + + client::kv_table_test::CreateTable( + Transactional::kFalse, // Results in different schema! + NumTablets(), + producer_cluster_client_.get(), + &producer_cluster_table2, + kTableName2); + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), client_.get(), &consumer_table2, kTableName2); + + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table2->id())); + + // Wait for the universe to be cleaned up + ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); + + // Verify that the environment is cleaned up correctly after failure. + // A valid call to SetupUniverseReplication after the failure should succeed + // without us having to first call DeleteUniverseReplication. + ASSERT_OK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id())); + // Verify table is being replicated. + ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); + + // Try calling SetupUniverseReplication again. This should fail as the producer + // is already present. However, in this case, DeleteUniverseReplication should + // not be called since the error was due to failing a sanity check. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id())); + // Verify the universe replication has not been deleted is still there. + ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); + + // Delete universe. + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); +} + +TEST_F(XClusterAdminCliTest, TestListCdcStreamsWithBootstrappedStreams) { + const int kStreamUuidLength = 32; + client::TableHandle producer_cluster_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); + + string output = ASSERT_RESULT(RunAdminToolCommandOnProducer("list_cdc_streams")); + // First check that the table and bootstrap status are not present. + ASSERT_EQ(output.find(producer_cluster_table->id()), string::npos); + ASSERT_EQ( + output.find(SysCDCStreamEntryPB::State_Name(SysCDCStreamEntryPB::INITIATED)), string::npos); + + // Bootstrap the producer. + output = ASSERT_RESULT( + RunAdminToolCommandOnProducer("bootstrap_cdc_producer", producer_cluster_table->id())); + // Get the bootstrap id (output format is "table id: 123, CDC bootstrap id: 123\n"). + string bootstrap_id = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); + + // Check list_cdc_streams again for the table and the status INITIATED. + ASSERT_OK( + CheckTableIsBeingReplicated({producer_cluster_table->id()}, SysCDCStreamEntryPB::INITIATED)); + + // Setup universe replication using the bootstrap_id + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id(), + bootstrap_id)); + + // Check list_cdc_streams again for the table and the status ACTIVE. + ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); + + // Try restarting the producer to ensure that the status persists. + ASSERT_OK(producer_cluster_->RestartSync()); + ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); + + // Delete this universe so shutdown can proceed. + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); +} + +TEST_F(XClusterAdminCliTest, TestRenameUniverseReplication) { + client::TableHandle producer_cluster_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); + + // Setup universe replication, this should only return once complete. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id())); + + // Check that the stream was properly created for this table. + ASSERT_OK(CheckTableIsBeingReplicated({producer_cluster_table->id()})); + + // Now rename the replication group and then try to perform operations on it. + std::string new_replication_id = "new_replication_id"; + ASSERT_OK(RunAdminToolCommand( + "alter_universe_replication", kProducerClusterId, "rename_id", new_replication_id)); + + // Assert that using old universe id fails. + ASSERT_NOK(RunAdminToolCommand("set_universe_replication_enabled", kProducerClusterId, 0)); + // But using correct name should succeed. + ASSERT_OK(RunAdminToolCommand("set_universe_replication_enabled", new_replication_id, 0)); + + // Also create a second stream so we can verify name collisions. + std::string collision_id = "collision_id"; + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + collision_id, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id())); + ASSERT_NOK(RunAdminToolCommand( + "alter_universe_replication", new_replication_id, "rename_id", collision_id)); + + // Using correct name should still succeed. + ASSERT_OK(RunAdminToolCommand("set_universe_replication_enabled", new_replication_id, 1)); + + // Also test that we can rename again. + std::string new_replication_id2 = "new_replication_id2"; + ASSERT_OK(RunAdminToolCommand( + "alter_universe_replication", new_replication_id, "rename_id", new_replication_id2)); + + // Assert that using old universe ids fails. + ASSERT_NOK(RunAdminToolCommand("set_universe_replication_enabled", kProducerClusterId, 1)); + ASSERT_NOK(RunAdminToolCommand("set_universe_replication_enabled", new_replication_id, 1)); + // But using new correct name should succeed. + ASSERT_OK(RunAdminToolCommand("set_universe_replication_enabled", new_replication_id2, 1)); + + // Delete this universe so shutdown can proceed. + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", new_replication_id2)); + // Also delete second one too. + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", collision_id)); +} + +class XClusterAlterUniverseAdminCliTest : public XClusterAdminCliTest { + public: + void SetUp() override { + YB_SKIP_TEST_IN_TSAN(); + + // Use more masters so we can test set_master_addresses + opts.num_masters = 3; + + XClusterAdminCliTest::SetUp(); + } +}; + +TEST_F(XClusterAlterUniverseAdminCliTest, TestAlterUniverseReplication) { + client::TableHandle producer_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); + + // Create an additional table to test with as well. + const YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "ql_client_test_table2"); + client::TableHandle consumer_table2; + client::TableHandle producer_table2; + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), client_.get(), &consumer_table2, kTableName2); + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table2, + kTableName2); + + // Setup replication with both tables, this should only return once complete. + // Only use the leader master address initially. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + ASSERT_RESULT(producer_cluster_->GetLeaderMiniMaster())->bound_rpc_addr_str(), + producer_table->id() + "," + producer_table2->id())); + + // Test set_master_addresses, use all the master addresses now. + ASSERT_OK(RunAdminToolCommand( + "alter_universe_replication", + kProducerClusterId, + "set_master_addresses", + producer_cluster_->GetMasterAddresses())); + ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id(), producer_table2->id()})); + + // Test removing a table. + ASSERT_OK(RunAdminToolCommand( + "alter_universe_replication", kProducerClusterId, "remove_table", producer_table->id())); + ASSERT_OK(CheckTableIsBeingReplicated({producer_table2->id()})); + ASSERT_NOK(CheckTableIsBeingReplicated({producer_table->id()})); + + // Test adding a table. + ASSERT_OK(RunAdminToolCommand( + "alter_universe_replication", kProducerClusterId, "add_table", producer_table->id())); + ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id(), producer_table2->id()})); + + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); +} + +TEST_F(XClusterAlterUniverseAdminCliTest, TestAlterUniverseReplicationWithBootstrapId) { + const int kStreamUuidLength = 32; + client::TableHandle producer_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); + + // Create an additional table to test with as well. + const YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "ql_client_test_table2"); + client::TableHandle consumer_table2; + client::TableHandle producer_table2; + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), client_.get(), &consumer_table2, kTableName2); + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table2, + kTableName2); + + // Get bootstrap ids for both producer tables and get bootstrap ids. + string output = + ASSERT_RESULT(RunAdminToolCommandOnProducer("bootstrap_cdc_producer", producer_table->id())); + string bootstrap_id1 = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); + ASSERT_OK(CheckTableIsBeingReplicated( + {producer_table->id()}, master::SysCDCStreamEntryPB_State_INITIATED)); + + output = + ASSERT_RESULT(RunAdminToolCommandOnProducer("bootstrap_cdc_producer", producer_table2->id())); + string bootstrap_id2 = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); + ASSERT_OK(CheckTableIsBeingReplicated( + {producer_table2->id()}, master::SysCDCStreamEntryPB_State_INITIATED)); + + // Setup replication with first table, this should only return once complete. + // Only use the leader master address initially. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + ASSERT_RESULT(producer_cluster_->GetLeaderMiniMaster())->bound_rpc_addr_str(), + producer_table->id(), + bootstrap_id1)); + + // Test adding the second table with bootstrap id + ASSERT_OK(RunAdminToolCommand( + "alter_universe_replication", + kProducerClusterId, + "add_table", + producer_table2->id(), + bootstrap_id2)); + ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id(), producer_table2->id()})); + + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); +} + +// delete_cdc_stream tests +TEST_F(XClusterAdminCliTest, TestDeleteCDCStreamWithConsumerSetup) { + client::TableHandle producer_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); + + // Setup universe replication, this should only return once complete. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_table->id())); + ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id()})); + + string stream_id = ASSERT_RESULT(GetRecentStreamId(producer_cluster_.get())); + + // Should fail as it should meet the conditions to be stopped. + ASSERT_NOK(RunAdminToolCommandOnProducer("delete_cdc_stream", stream_id)); + // Should pass as we force it. + ASSERT_OK(RunAdminToolCommandOnProducer("delete_cdc_stream", stream_id, "force_delete")); + // Delete universe should fail as we've force deleted the stream. + ASSERT_NOK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); + ASSERT_OK( + RunAdminToolCommand("delete_universe_replication", kProducerClusterId, "ignore-errors")); +} + +TEST_F(XClusterAdminCliTest, TestDeleteCDCStreamWithAlterUniverse) { + client::TableHandle producer_table; + constexpr int kNumTables = 3; + const client::YBTableName kTableName2(YQL_DATABASE_CQL, "my_keyspace", "test_table2"); + + // Create identical table on producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); + + // Create some additional tables. The reason is that the number of tables removed using + // alter_universe_replication remove_table must be less than the total number of tables. + string producer_table_ids_str = producer_table->id(); + std::vector producer_table_ids = {producer_table->id()}; + for (int i = 1; i < kNumTables; i++) { + client::TableHandle tmp_producer_table, tmp_consumer_table; + const client::YBTableName kTestTableName( + YQL_DATABASE_CQL, "my_keyspace", Format("test_table_$0", i)); + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), client_.get(), &tmp_consumer_table, kTestTableName); + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &tmp_producer_table, + kTestTableName); + producer_table_ids_str += Format(",$0", tmp_producer_table->id()); + producer_table_ids.push_back(tmp_producer_table->id()); + } + + // Setup universe replication. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_table_ids_str)); + ASSERT_OK(CheckTableIsBeingReplicated(producer_table_ids)); + + // Obtain the stream ID for the first table. + string stream_id = + ASSERT_RESULT(GetRecentStreamId(producer_cluster_.get(), producer_table->id())); + ASSERT_FALSE(stream_id.empty()); + + // Mark one stream as deleted. + ASSERT_OK(RunAdminToolCommandOnProducer("delete_cdc_stream", stream_id, "force_delete")); + + // Remove table should fail as its stream is marked as deleting on producer. + ASSERT_NOK(RunAdminToolCommand( + "alter_universe_replication", kProducerClusterId, "remove_table", producer_table->id())); + ASSERT_OK(RunAdminToolCommand( + "alter_universe_replication", + kProducerClusterId, + "remove_table", + producer_table->id(), + "ignore-errors")); +} + +TEST_F(XClusterAdminCliTest, TestWaitForReplicationDrain) { + client::TableHandle producer_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); + + // Setup universe replication. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_table->id())); + ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id()})); + string stream_id = ASSERT_RESULT(GetRecentStreamId(producer_cluster_.get())); + + // API should succeed with correctly formatted arguments. + ASSERT_OK(RunAdminToolCommandOnProducer("wait_for_replication_drain", stream_id)); + ASSERT_OK(RunAdminToolCommandOnProducer( + "wait_for_replication_drain", stream_id, GetCurrentTimeMicros())); + ASSERT_OK(RunAdminToolCommandOnProducer("wait_for_replication_drain", stream_id, "minus", "3s")); + + // API should fail with an invalid stream ID. + ASSERT_NOK(RunAdminToolCommandOnProducer("wait_for_replication_drain", "abc")); + + // API should fail with an invalid target_time format. + ASSERT_NOK(RunAdminToolCommandOnProducer("wait_for_replication_drain", stream_id, 123)); + ASSERT_NOK( + RunAdminToolCommandOnProducer("wait_for_replication_drain", stream_id, "minus", "hello")); +} + +TEST_F(XClusterAdminCliTest, TestDeleteCDCStreamWithBootstrap) { + const int kStreamUuidLength = 32; + client::TableHandle producer_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_table); + + string output = + ASSERT_RESULT(RunAdminToolCommandOnProducer("bootstrap_cdc_producer", producer_table->id())); + // Get the bootstrap id (output format is "table id: 123, CDC bootstrap id: 123\n"). + string bootstrap_id = output.substr(output.find_last_of(' ') + 1, kStreamUuidLength); + + // Setup universe replication, this should only return once complete. + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_table->id(), + bootstrap_id)); + ASSERT_OK(CheckTableIsBeingReplicated({producer_table->id()})); + + // Should fail as it should meet the conditions to be stopped. + ASSERT_NOK(RunAdminToolCommandOnProducer("delete_cdc_stream", bootstrap_id)); + // Delete should work fine from deleting from universe. + ASSERT_OK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); +} + +TEST_F(XClusterAdminCliTest, TestDeleteCDCStreamWithCreateCDCStream) { + // Create CDC stream + ASSERT_OK(RunAdminToolCommand("create_cdc_stream", table_->id())); + + string stream_id = ASSERT_RESULT(GetRecentStreamId(cluster_.get())); + + // Should be deleted. + ASSERT_OK(RunAdminToolCommand("delete_cdc_stream", stream_id)); +} + +TEST_F(XClusterAdminCliTest, TestFailedSetupUniverseWithDeletion) { + client::TableHandle producer_cluster_table; + + // Create an identical table on the producer. + client::kv_table_test::CreateTable( + Transactional::kTrue, NumTablets(), producer_cluster_client_.get(), &producer_cluster_table); + + string error_msg; + // Setup universe replication, this should only return once complete. + // First provide a non-existant table id. + // ASSERT_NOK since this should fail. + ASSERT_NOK(RunAdminToolCommandAndGetErrorOutput( + &error_msg, + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id() + "-BAD")); + + ASSERT_OK(WaitForSetupUniverseReplicationCleanUp(kProducerClusterId)); + + // Universe should be deleted by BG cleanup + ASSERT_NOK(RunAdminToolCommand("delete_universe_replication", kProducerClusterId)); + + ASSERT_OK(RunAdminToolCommand( + "setup_universe_replication", + kProducerClusterId, + producer_cluster_->GetMasterAddresses(), + producer_cluster_table->id())); + std::this_thread::sleep_for(5s); +} + +class XClusterAdminCliTest_Large : public XClusterAdminCliTest { + public: + void SetUp() override { + // Skip this test in TSAN since the test will time out waiting + // for table creation to finish. + YB_SKIP_TEST_IN_TSAN(); + + XClusterAdminCliTest::SetUp(); + } + int num_tablet_servers() override { return 5; } +}; + +TEST_F(XClusterAdminCliTest_Large, TestBootstrapProducerPerformance) { + const int table_count = 10; + const int tablet_count = 5; + const int expected_runtime_seconds = 15 * kTimeMultiplier; + const std::string keyspace = "my_keyspace"; + std::vector tables; + + for (int i = 0; i < table_count; i++) { + client::TableHandle th; + tables.push_back(th); + + // Build the table. + client::YBSchemaBuilder builder; + builder.AddColumn(kKeyColumn)->Type(INT32)->HashPrimaryKey()->NotNull(); + builder.AddColumn(kValueColumn)->Type(INT32); + + TableProperties table_properties; + table_properties.SetTransactional(true); + builder.SetTableProperties(table_properties); + + const YBTableName table_name( + YQL_DATABASE_CQL, keyspace, Format("bootstrap_producer_performance_test_table_$0", i)); + ASSERT_OK(producer_cluster_client_.get()->CreateNamespaceIfNotExists( + table_name.namespace_name(), table_name.namespace_type())); + + ASSERT_OK( + tables.at(i).Create(table_name, tablet_count, producer_cluster_client_.get(), &builder)); + } + + std::string table_ids = tables.at(0)->id(); + for (size_t i = 1; i < tables.size(); ++i) { + table_ids += "," + tables.at(i)->id(); + } + + // Wait for load balancer to be idle until we call bootstrap_cdc_producer. + // This prevents TABLET_DATA_TOMBSTONED errors when we make rpc calls. + // Todo: need to improve bootstrap behaviour with load balancer. + ASSERT_OK(WaitFor( + [this, table_ids]() -> Result { + return producer_cluster_client_->IsLoadBalancerIdle(); + }, + MonoDelta::FromSeconds(120 * kTimeMultiplier), + "Waiting for load balancer to be idle")); + + // Add delays to all rpc calls to simulate live environment. + FLAGS_TEST_yb_inbound_big_calls_parse_delay_ms = 5; + FLAGS_rpc_throttle_threshold_bytes = 0; + // Enable parallelized version of BootstrapProducer. + FLAGS_parallelize_bootstrap_producer = true; + + // Check that bootstrap_cdc_producer returns within time limit. + ASSERT_OK(WaitFor( + [this, table_ids]() -> Result { + auto res = RunAdminToolCommandOnProducer("bootstrap_cdc_producer", table_ids); + return res.ok(); + }, + MonoDelta::FromSeconds(expected_runtime_seconds), + "Waiting for bootstrap_cdc_producer to complete")); +} + +} // namespace tools +} // namespace yb diff --git a/src/yb/tools/yb-admin_cli.cc b/src/yb/tools/yb-admin_cli.cc index d2d3d077d6cc..5f1616397cf0 100644 --- a/src/yb/tools/yb-admin_cli.cc +++ b/src/yb/tools/yb-admin_cli.cc @@ -35,19 +35,26 @@ #include #include +#include #include #include "yb/common/json_util.h" +#include "yb/gutil/strings/util.h" +#include "yb/master/master_backup.pb.h" #include "yb/master/master_defaults.h" #include "yb/tools/yb-admin_client.h" +#include "yb/tools/yb-admin_util.h" +#include "yb/util/env.h" #include "yb/util/flags.h" #include "yb/util/logging.h" +#include "yb/util/pb_util.h" #include "yb/util/status_format.h" #include "yb/util/stol_utils.h" #include "yb/util/string_case.h" +#include "yb/util/jsonwriter.h" DEFINE_UNKNOWN_string(master_addresses, "localhost:7100", "Comma-separated list of YB Master server addresses"); @@ -65,6 +72,7 @@ using std::make_pair; using std::next; using std::pair; using std::string; +using std::vector; using yb::client::YBTableName; using strings::Substitute; @@ -82,6 +90,7 @@ namespace { constexpr auto kBlacklistAdd = "ADD"; constexpr auto kBlacklistRemove = "REMOVE"; constexpr int32 kDefaultRpcPort = 9100; +const string kMinus = "minus"; const std::string namespace_expression = ":\n [(ycql|ysql).] (default ycql.)"; @@ -90,13 +99,13 @@ const std::string table_expression = const std::string index_expression = ":\n | tableid."; -Status GetUniverseConfig(ClusterAdminClientClass* client, +Status GetUniverseConfig(ClusterAdminClient* client, const ClusterAdminCli::CLIArguments&) { RETURN_NOT_OK_PREPEND(client->GetUniverseConfig(), "Unable to get universe config"); return Status::OK(); } -Status ChangeBlacklist(ClusterAdminClientClass* client, +Status ChangeBlacklist(ClusterAdminClient* client, const ClusterAdminCli::CLIArguments& args, bool blacklist_leader, const std::string& errStr) { if (args.size() < 2) { @@ -117,7 +126,7 @@ Status ChangeBlacklist(ClusterAdminClientClass* client, } Status MasterLeaderStepDown( - ClusterAdminClientClass* client, + ClusterAdminClient* client, const ClusterAdminCli::CLIArguments& args) { const auto leader_uuid = VERIFY_RESULT(client->GetMasterLeaderUuid()); return client->MasterLeaderStepDown( @@ -125,7 +134,7 @@ Status MasterLeaderStepDown( } Status LeaderStepDown( - ClusterAdminClientClass* client, + ClusterAdminClient* client, const ClusterAdminCli::CLIArguments& args) { if (args.size() < 1) { return ClusterAdminCli::kInvalidArguments; @@ -187,6 +196,172 @@ Status PrioritizedError(Status hi_pri_status, Status low_pri_status) { return hi_pri_status.ok() ? std::move(low_pri_status) : std::move(hi_pri_status); } +template +Result GetOptionalArg(const Args& args, size_t idx) { + if (args.size() <= idx) { + return T::Nil(); + } + if (args.size() > idx + 1) { + return STATUS_FORMAT( + InvalidArgument, "Too many arguments for command, at most $0 expected, but $1 found", + idx + 1, args.size()); + } + return VERIFY_RESULT(T::FromString(args[idx])); +} + +Status ListSnapshots(ClusterAdminClient* client, const EnumBitSet& flags) { + auto snapshot_response = VERIFY_RESULT(client->ListSnapshots(flags)); + + rapidjson::Document document(rapidjson::kObjectType); + bool json = flags.Test(ListSnapshotsFlag::JSON); + + if (snapshot_response.has_current_snapshot_id()) { + if (json) { + AddStringField( + "current_snapshot_id", SnapshotIdToString(snapshot_response.current_snapshot_id()), + &document, &document.GetAllocator()); + } else { + std::cout << "Current snapshot id: " + << SnapshotIdToString(snapshot_response.current_snapshot_id()) << std::endl; + } + } + + rapidjson::Value json_snapshots(rapidjson::kArrayType); + if (!json) { + if (snapshot_response.snapshots_size()) { + // Using 2 tabs so that the header can be aligned to the time. + std::cout << RightPadToUuidWidth("Snapshot UUID") << kColumnSep << "State" << kColumnSep + << kColumnSep << "Creation Time" << std::endl; + } else { + std::cout << "No snapshots" << std::endl; + } + } + + for (master::SnapshotInfoPB& snapshot : *snapshot_response.mutable_snapshots()) { + rapidjson::Value json_snapshot(rapidjson::kObjectType); + if (json) { + AddStringField( + "id", SnapshotIdToString(snapshot.id()), &json_snapshot, &document.GetAllocator()); + const auto& entry = snapshot.entry(); + AddStringField( + "state", master::SysSnapshotEntryPB::State_Name(entry.state()), &json_snapshot, + &document.GetAllocator()); + AddStringField( + "snapshot_time", HybridTimeToString(HybridTime::FromPB(entry.snapshot_hybrid_time())), + &json_snapshot, &document.GetAllocator()); + AddStringField( + "previous_snapshot_time", + HybridTimeToString(HybridTime::FromPB(entry.previous_snapshot_hybrid_time())), + &json_snapshot, &document.GetAllocator()); + } else { + std::cout << SnapshotIdToString(snapshot.id()) << kColumnSep + << master::SysSnapshotEntryPB::State_Name(snapshot.entry().state()) << kColumnSep + << HybridTimeToString(HybridTime::FromPB(snapshot.entry().snapshot_hybrid_time())) + << std::endl; + } + + // Not implemented in json mode. + if (flags.Test(ListSnapshotsFlag::SHOW_DETAILS)) { + for (master::SysRowEntry& entry : *snapshot.mutable_entry()->mutable_entries()) { + string decoded_data; + switch (entry.type()) { + case master::SysRowEntryType::NAMESPACE: { + auto meta = + VERIFY_RESULT(pb_util::ParseFromSlice(entry.data())); + meta.clear_transaction(); + decoded_data = JsonWriter::ToJson(meta, JsonWriter::COMPACT); + break; + } + case master::SysRowEntryType::UDTYPE: { + auto meta = + VERIFY_RESULT(pb_util::ParseFromSlice(entry.data())); + decoded_data = JsonWriter::ToJson(meta, JsonWriter::COMPACT); + break; + } + case master::SysRowEntryType::TABLE: { + auto meta = + VERIFY_RESULT(pb_util::ParseFromSlice(entry.data())); + meta.clear_schema(); + meta.clear_partition_schema(); + meta.clear_index_info(); + meta.clear_indexes(); + meta.clear_transaction(); + decoded_data = JsonWriter::ToJson(meta, JsonWriter::COMPACT); + break; + } + default: + break; + } + + if (!decoded_data.empty()) { + entry.set_data("DATA"); + std::cout << kColumnSep + << StringReplace( + JsonWriter::ToJson(entry, JsonWriter::COMPACT), "\"DATA\"", decoded_data, + false) + << std::endl; + } + } + } + if (json) { + json_snapshots.PushBack(json_snapshot, document.GetAllocator()); + } + } + + if (json) { + document.AddMember("snapshots", json_snapshots, document.GetAllocator()); + std::cout << common::PrettyWriteRapidJsonToString(document) << std::endl; + return Status::OK(); + } + + auto restorations_result = + VERIFY_RESULT(client->ListSnapshotRestorations(TxnSnapshotRestorationId::Nil())); + if (restorations_result.restorations_size() == 0) { + std::cout << "No snapshot restorations" << std::endl; + } else if (flags.Test(ListSnapshotsFlag::NOT_SHOW_RESTORED)) { + std::cout << "Not show fully RESTORED entries" << std::endl; + } + + bool title_printed = false; + for (const auto& restoration : restorations_result.restorations()) { + if (!flags.Test(ListSnapshotsFlag::NOT_SHOW_RESTORED) || + restoration.entry().state() != master::SysSnapshotEntryPB::RESTORED) { + if (!title_printed) { + std::cout << RightPadToUuidWidth("Restoration UUID") << kColumnSep << "State" << std::endl; + title_printed = true; + } + std::cout << TryFullyDecodeTxnSnapshotRestorationId(restoration.id()) << kColumnSep + << master::SysSnapshotEntryPB::State_Name(restoration.entry().state()) << std::endl; + } + } + + return Status::OK(); +} + +Result ListSnapshotRestorations( + ClusterAdminClient* client, const TxnSnapshotRestorationId& restoration_id) { + auto resp = VERIFY_RESULT(client->ListSnapshotRestorations(restoration_id)); + rapidjson::Document result; + result.SetObject(); + rapidjson::Value json_restorations(rapidjson::kArrayType); + for (const auto& restoration : resp.restorations()) { + rapidjson::Value json_restoration(rapidjson::kObjectType); + AddStringField( + "id", VERIFY_RESULT(FullyDecodeTxnSnapshotRestorationId(restoration.id())).ToString(), + &json_restoration, &result.GetAllocator()); + AddStringField( + "snapshot_id", + VERIFY_RESULT(FullyDecodeTxnSnapshotId(restoration.entry().snapshot_id())).ToString(), + &json_restoration, &result.GetAllocator()); + AddStringField( + "state", master::SysSnapshotEntryPB_State_Name(restoration.entry().state()), + &json_restoration, &result.GetAllocator()); + json_restorations.PushBack(json_restoration, result.GetAllocator()); + } + result.AddMember("restorations", json_restorations, result.GetAllocator()); + return result; +} + } // namespace std::string ClusterAdminCli::GetArgumentExpressions(const std::string& usage_arguments) { @@ -230,7 +405,7 @@ Status ClusterAdminCli::Run(int argc, char** argv) { ParseCommandLineFlags(&argc, &argv, true); InitGoogleLoggingSafe(prog_name.c_str()); - std::unique_ptr client; + std::unique_ptr client; const string addrs = FLAGS_master_addresses; if (!FLAGS_init_master_addrs.empty()) { std::vector init_master_addrs; @@ -238,11 +413,11 @@ Status ClusterAdminCli::Run(int argc, char** argv) { FLAGS_init_master_addrs, master::kMasterDefaultPort, &init_master_addrs)); - client.reset(new ClusterAdminClientClass( + client.reset(new ClusterAdminClient( init_master_addrs[0], MonoDelta::FromMilliseconds(FLAGS_timeout_ms))); } else { - client.reset(new ClusterAdminClientClass( + client.reset(new ClusterAdminClient( addrs, MonoDelta::FromMilliseconds(FLAGS_timeout_ms))); } @@ -316,12 +491,12 @@ void ClusterAdminCli::SetUsage(const string& prog_name) { } Result DdlLog( - ClusterAdminClientClass* client, const ClusterAdminCli::CLIArguments& args) { + ClusterAdminClient* client, const ClusterAdminCli::CLIArguments& args) { RETURN_NOT_OK(CheckArgumentsCount(args.size(), 0, 0)); return client->DdlLog(); } -void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClientClass* client) { +void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClient* client) { DCHECK_ONLY_NOTNULL(client); Register( @@ -1044,10 +1219,730 @@ void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClientClass* client) { "Unable to promote AutoFlags"); return Status::OK(); }); + + std::string options = ""; + for (auto flag : ListSnapshotsFlagList()) { + options += Format(" [$0]", flag); + } + Register("list_snapshots", std::move(options), [client](const CLIArguments& args) -> Status { + EnumBitSet flags; + + for (size_t i = 0; i < args.size(); ++i) { + std::string uppercase_flag; + ToUpperCase(args[i], &uppercase_flag); + + bool found = false; + for (auto flag : ListSnapshotsFlagList()) { + if (uppercase_flag == ToString(flag)) { + flags.Set(flag); + found = true; + break; + } + } + if (!found) { + return STATUS_FORMAT(InvalidArgument, "Wrong flag: $0", args[i]); + } + } + + RETURN_NOT_OK_PREPEND(ListSnapshots(client, flags), "Unable to list snapshots"); + return Status::OK(); + }); + + Register( + "create_snapshot", + "
    " + " [
    ]..." + " [] (default 60, set 0 to skip flushing)", + [client](const CLIArguments& args) -> Status { + int timeout_secs = 60; + const auto tables = VERIFY_RESULT(ResolveTableNames( + client, args.begin(), args.end(), [&timeout_secs](auto i, const auto& end) -> Status { + if (std::next(i) == end) { + timeout_secs = VERIFY_RESULT(CheckedStoi(*i)); + return Status::OK(); + } + return ClusterAdminCli::kInvalidArguments; + })); + + for (auto table : tables) { + if (table.is_cql_namespace() && table.is_system()) { + return STATUS( + InvalidArgument, "Cannot create snapshot of YCQL system table", table.table_name()); + } + } + + RETURN_NOT_OK_PREPEND( + client->CreateSnapshot(tables, true, timeout_secs), + Substitute("Unable to create snapshot of tables: $0", yb::ToString(tables))); + return Status::OK(); + }); + + RegisterJson( + "list_snapshot_restorations", + " []", + [client](const CLIArguments& args) -> Result { + auto restoration_id = VERIFY_RESULT(GetOptionalArg(args, 0)); + return ListSnapshotRestorations(client, restoration_id); + }); + + RegisterJson( + "create_snapshot_schedule", + " " + " " + " ", + [client](const CLIArguments& args) -> Result { + RETURN_NOT_OK(CheckArgumentsCount(args.size(), 3, 3)); + auto interval = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[0]))); + auto retention = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[1]))); + const auto tables = VERIFY_RESULT(ResolveTableNames( + client, args.begin() + 2, args.end(), TailArgumentsProcessor(), true)); + // This is just a paranoid check, should never happen. + if (tables.size() != 1 || !tables[0].has_namespace()) { + return STATUS(InvalidArgument, "Expecting exactly one keyspace argument"); + } + if (tables[0].namespace_type() != YQL_DATABASE_CQL && + tables[0].namespace_type() != YQL_DATABASE_PGSQL) { + return STATUS( + InvalidArgument, "Snapshot schedule can only be setup on YCQL or YSQL namespace"); + } + return client->CreateSnapshotSchedule(tables[0], interval, retention); + }); + + RegisterJson( + "list_snapshot_schedules", + " []", + [client](const CLIArguments& args) -> Result { + RETURN_NOT_OK(CheckArgumentsCount(args.size(), 0, 1)); + auto schedule_id = VERIFY_RESULT(GetOptionalArg(args, 0)); + return client->ListSnapshotSchedules(schedule_id); + }); + + RegisterJson( + "delete_snapshot_schedule", + " ", + [client](const CLIArguments& args) -> Result { + RETURN_NOT_OK(CheckArgumentsCount(args.size(), 1, 1)); + auto schedule_id = VERIFY_RESULT(SnapshotScheduleId::FromString(args[0])); + return client->DeleteSnapshotSchedule(schedule_id); + }); + + RegisterJson( + "restore_snapshot_schedule", + Format(" ( | $0 )", kMinus), + [client](const CLIArguments& args) -> Result { + RETURN_NOT_OK(CheckArgumentsCount(args.size(), 2, 3)); + auto schedule_id = VERIFY_RESULT(SnapshotScheduleId::FromString(args[0])); + HybridTime restore_at; + if (args.size() == 2) { + restore_at = VERIFY_RESULT(HybridTime::ParseHybridTime(args[1])); + } else { + if (args[1] != kMinus) { + return ClusterAdminCli::kInvalidArguments; + } + restore_at = VERIFY_RESULT(HybridTime::ParseHybridTime("-" + args[2])); + } + + return client->RestoreSnapshotSchedule(schedule_id, restore_at); + }); + + RegisterJson( + "edit_snapshot_schedule", + " (interval | retention " + "){1,2}", + [client](const CLIArguments& args) -> Result { + if (args.size() != 3 && args.size() != 5) { + return STATUS( + InvalidArgument, Format("Expected 3 or 5 arguments, received $0", args.size())); + } + auto schedule_id = VERIFY_RESULT(SnapshotScheduleId::FromString(args[0])); + std::optional new_interval; + std::optional new_retention; + for (size_t i = 1; i + 1 < args.size(); i += 2) { + if (args[i] == "interval") { + if (new_interval) { + return STATUS(InvalidArgument, "Repeated interval"); + } + new_interval = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[i + 1]))); + } else if (args[i] == "retention") { + if (new_retention) { + return STATUS(InvalidArgument, "Repeated retention"); + } + new_retention = MonoDelta::FromMinutes(VERIFY_RESULT(CheckedStold(args[i + 1]))); + } else { + return STATUS( + InvalidArgument, + Format("Expected either \"retention\" or \"interval\", got: $0", args[i])); + } + } + return client->EditSnapshotSchedule(schedule_id, new_interval, new_retention); + }); + + Register( + "create_keyspace_snapshot", " [ycql.]", + [client](const CLIArguments& args) -> Status { + if (args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + + const TypedNamespaceName keyspace = VERIFY_RESULT(ParseNamespaceName(args[0])); + SCHECK_NE( + keyspace.db_type, YQL_DATABASE_PGSQL, InvalidArgument, + Format("Wrong keyspace type: $0", YQLDatabase_Name(keyspace.db_type))); + + RETURN_NOT_OK_PREPEND( + client->CreateNamespaceSnapshot(keyspace), + Substitute("Unable to create snapshot of keyspace: $0", keyspace.name)); + return Status::OK(); + }); + + Register( + "create_database_snapshot", " [ysql.]", + [client](const CLIArguments& args) -> Status { + if (args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + + const TypedNamespaceName database = + VERIFY_RESULT(ParseNamespaceName(args[0], YQL_DATABASE_PGSQL)); + SCHECK_EQ( + database.db_type, YQL_DATABASE_PGSQL, InvalidArgument, + Format("Wrong database type: $0", YQLDatabase_Name(database.db_type))); + + RETURN_NOT_OK_PREPEND( + client->CreateNamespaceSnapshot(database), + Substitute("Unable to create snapshot of database: $0", database.name)); + return Status::OK(); + }); + + Register( + "restore_snapshot", Format(" [ | $0 ]", kMinus), + [client](const CLIArguments& args) -> Status { + if (args.size() < 1 || 3 < args.size()) { + return ClusterAdminCli::kInvalidArguments; + } else if (args.size() == 3 && args[1] != kMinus) { + return ClusterAdminCli::kInvalidArguments; + } + const string snapshot_id = args[0]; + HybridTime timestamp; + if (args.size() == 2) { + timestamp = VERIFY_RESULT(HybridTime::ParseHybridTime(args[1])); + } else if (args.size() == 3) { + timestamp = VERIFY_RESULT(HybridTime::ParseHybridTime("-" + args[2])); + } + + RETURN_NOT_OK_PREPEND( + client->RestoreSnapshot(snapshot_id, timestamp), + Substitute("Unable to restore snapshot $0", snapshot_id)); + return Status::OK(); + }); + + Register( + "export_snapshot", " ", + [client](const CLIArguments& args) -> Status { + if (args.size() != 2) { + return ClusterAdminCli::kInvalidArguments; + } + + const string snapshot_id = args[0]; + const string file_name = args[1]; + RETURN_NOT_OK_PREPEND( + client->CreateSnapshotMetaFile(snapshot_id, file_name), + Substitute("Unable to export snapshot $0 to file $1", snapshot_id, file_name)); + return Status::OK(); + }); + + Register( + "import_snapshot", " [ []...]", + [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + + const string file_name = args[0]; + TypedNamespaceName keyspace; + size_t num_tables = 0; + vector tables; + + if (args.size() >= 2) { + keyspace = VERIFY_RESULT(ParseNamespaceName(args[1])); + num_tables = args.size() - 2; + + if (num_tables > 0) { + LOG_IF(DFATAL, keyspace.name.empty()) << "Uninitialized keyspace: " << keyspace.name; + tables.reserve(num_tables); + + for (size_t i = 0; i < num_tables; ++i) { + tables.push_back(YBTableName(keyspace.db_type, keyspace.name, args[2 + i])); + } + } + } + + const string msg = num_tables > 0 + ? Substitute( + "Unable to import tables $0 from snapshot meta file $1", + yb::ToString(tables), file_name) + : Substitute("Unable to import snapshot meta file $0", file_name); + + RETURN_NOT_OK_PREPEND(client->ImportSnapshotMetaFile(file_name, keyspace, tables), msg); + return Status::OK(); + }); + + Register("delete_snapshot", " ", [client](const CLIArguments& args) -> Status { + if (args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + + const string snapshot_id = args[0]; + RETURN_NOT_OK_PREPEND( + client->DeleteSnapshot(snapshot_id), + Substitute("Unable to delete snapshot $0", snapshot_id)); + return Status::OK(); + }); + + Register("list_replica_type_counts", "
    ", [client](const CLIArguments& args) -> Status { + const auto table_name = VERIFY_RESULT(ResolveSingleTableName(client, args.begin(), args.end())); + RETURN_NOT_OK_PREPEND( + client->ListReplicaTypeCounts(table_name), + "Unable to list live and read-only replica counts"); + return Status::OK(); + }); + + Register( + "set_preferred_zones", + " [:] [[:]]...", + [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + RETURN_NOT_OK_PREPEND(client->SetPreferredZones(args), "Unable to set preferred zones"); + return Status::OK(); + }); + + Register("rotate_universe_key", " ", [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + RETURN_NOT_OK_PREPEND(client->RotateUniverseKey(args[0]), "Unable to rotate universe key."); + return Status::OK(); + }); + + Register("disable_encryption", "", [client](const CLIArguments& args) -> Status { + RETURN_NOT_OK_PREPEND(client->DisableEncryption(), "Unable to disable encryption."); + return Status::OK(); + }); + + Register("is_encryption_enabled", "", [client](const CLIArguments& args) -> Status { + RETURN_NOT_OK_PREPEND(client->IsEncryptionEnabled(), "Unable to get encryption status."); + return Status::OK(); + }); + + Register( + "add_universe_key_to_all_masters", " ", + [client](const CLIArguments& args) -> Status { + if (args.size() != 2) { + return ClusterAdminCli::kInvalidArguments; + } + string key_id = args[0]; + faststring contents; + RETURN_NOT_OK(ReadFileToString(Env::Default(), args[1], &contents)); + string universe_key = contents.ToString(); + + RETURN_NOT_OK_PREPEND( + client->AddUniverseKeyToAllMasters(key_id, universe_key), + "Unable to add universe key to all masters."); + return Status::OK(); + }); + + Register( + "all_masters_have_universe_key_in_memory", " ", + [client](const CLIArguments& args) -> Status { + if (args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + RETURN_NOT_OK_PREPEND( + client->AllMastersHaveUniverseKeyInMemory(args[0]), + "Unable to check whether master has universe key in memory."); + return Status::OK(); + }); + + Register( + "rotate_universe_key_in_memory", " ", [client](const CLIArguments& args) -> Status { + if (args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + string key_id = args[0]; + + RETURN_NOT_OK_PREPEND( + client->RotateUniverseKeyInMemory(key_id), "Unable rotate universe key in memory."); + return Status::OK(); + }); + + Register("disable_encryption_in_memory", "", [client](const CLIArguments& args) -> Status { + if (args.size() != 0) { + return ClusterAdminCli::kInvalidArguments; + } + RETURN_NOT_OK_PREPEND(client->DisableEncryptionInMemory(), "Unable to disable encryption."); + return Status::OK(); + }); + + Register( + "write_universe_key_to_file", " ", + [client](const CLIArguments& args) -> Status { + if (args.size() != 2) { + return ClusterAdminCli::kInvalidArguments; + } + RETURN_NOT_OK_PREPEND( + client->WriteUniverseKeyToFile(args[0], args[1]), "Unable to write key to file"); + return Status::OK(); + }); + + Register("create_cdc_stream", " ", [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + const string table_id = args[0]; + RETURN_NOT_OK_PREPEND( + client->CreateCDCStream(table_id), + Substitute("Unable to create CDC stream for table $0", table_id)); + return Status::OK(); + }); + + Register( + "create_change_data_stream", " [] []", + [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + + std::string checkpoint_type = yb::ToString("IMPLICIT"); + std::string record_type = yb::ToString("CHANGE"); + std::string uppercase_checkpoint_type; + std::string uppercase_record_type; + + if (args.size() > 1) { + ToUpperCase(args[1], &uppercase_checkpoint_type); + if (uppercase_checkpoint_type != yb::ToString("EXPLICIT") && + uppercase_checkpoint_type != yb::ToString("IMPLICIT")) { + return ClusterAdminCli::kInvalidArguments; + } + checkpoint_type = uppercase_checkpoint_type; + } + + if (args.size() > 2) { + ToUpperCase(args[2], &uppercase_record_type); + if (uppercase_record_type != yb::ToString("ALL") && + uppercase_record_type != yb::ToString("CHANGE")) { + return ClusterAdminCli::kInvalidArguments; + } + record_type = uppercase_record_type; + } + + const string namespace_name = args[0]; + + const TypedNamespaceName database = + VERIFY_RESULT(ParseNamespaceName(args[0], YQL_DATABASE_PGSQL)); + SCHECK_EQ( + database.db_type, YQL_DATABASE_PGSQL, InvalidArgument, + Format("Wrong database type: $0", YQLDatabase_Name(database.db_type))); + + RETURN_NOT_OK_PREPEND( + client->CreateCDCSDKDBStream(database, checkpoint_type, record_type), + Substitute("Unable to create CDC stream for database $0", namespace_name)); + return Status::OK(); + }); + + Register( + "delete_cdc_stream", " [force_delete]", + [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + const string stream_id = args[0]; + bool force_delete = false; + if (args.size() >= 2 && args[1] == "force_delete") { + force_delete = true; + } + RETURN_NOT_OK_PREPEND( + client->DeleteCDCStream(stream_id, force_delete), + Substitute("Unable to delete CDC stream id $0", stream_id)); + return Status::OK(); + }); + + Register( + "delete_change_data_stream", " ", [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + + const std::string db_stream_id = args[0]; + RETURN_NOT_OK_PREPEND( + client->DeleteCDCSDKDBStream(db_stream_id), + Substitute("Unable to delete CDC database stream id $0", db_stream_id)); + return Status::OK(); + }); + + Register("list_cdc_streams", " []", [client](const CLIArguments& args) -> Status { + if (args.size() != 0 && args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + const string table_id = (args.size() == 1 ? args[0] : ""); + RETURN_NOT_OK_PREPEND( + client->ListCDCStreams(table_id), + Substitute("Unable to list CDC streams for table $0", table_id)); + return Status::OK(); + }); + + Register( + "list_change_data_streams", " []", [client](const CLIArguments& args) -> Status { + if (args.size() != 0 && args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + const string namespace_name = args.size() == 1 ? args[0] : ""; + string msg = (args.size() == 1) + ? Substitute("Unable to list CDC streams for namespace $0", namespace_name) + : "Unable to list CDC streams"; + + RETURN_NOT_OK_PREPEND(client->ListCDCSDKStreams(namespace_name), msg); + return Status::OK(); + }); + + Register( + "get_change_data_stream_info", " ", + [client](const CLIArguments& args) -> Status { + if (args.size() != 0 && args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + const string db_stream_id = args.size() == 1 ? args[0] : ""; + RETURN_NOT_OK_PREPEND( + client->GetCDCDBStreamInfo(db_stream_id), + Substitute("Unable to list CDC stream info for database stream $0", db_stream_id)); + return Status::OK(); + }); + + Register( + "setup_universe_replication", + " " + " []", + [client](const CLIArguments& args) -> Status { + if (args.size() < 3) { + return ClusterAdminCli::kInvalidArguments; + } + const string producer_uuid = args[0]; + + vector producer_addresses; + boost::split(producer_addresses, args[1], boost::is_any_of(",")); + + vector table_uuids; + boost::split(table_uuids, args[2], boost::is_any_of(",")); + + vector producer_bootstrap_ids; + if (args.size() == 4) { + boost::split(producer_bootstrap_ids, args[3], boost::is_any_of(",")); + } + + RETURN_NOT_OK_PREPEND( + client->SetupUniverseReplication( + producer_uuid, producer_addresses, table_uuids, producer_bootstrap_ids), + Substitute("Unable to setup replication from universe $0", producer_uuid)); + return Status::OK(); + }); + + Register( + "delete_universe_replication", " [ignore-errors]", + [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + const string producer_id = args[0]; + bool ignore_errors = false; + if (args.size() >= 2 && args[1] == "ignore-errors") { + ignore_errors = true; + } + RETURN_NOT_OK_PREPEND( + client->DeleteUniverseReplication(producer_id, ignore_errors), + Substitute("Unable to delete replication for universe $0", producer_id)); + return Status::OK(); + }); + + Register( + "alter_universe_replication", + " " + " (set_master_addresses [] |" + " add_table []" + " [] |" + " remove_table [] [ignore-errors] |" + " rename_id )", + [client](const CLIArguments& args) -> Status { + if (args.size() < 3 || args.size() > 4) { + return ClusterAdminCli::kInvalidArguments; + } + if (args.size() == 4 && args[1] != "add_table" && args[1] != "remove_table") { + return ClusterAdminCli::kInvalidArguments; + } + + const string producer_uuid = args[0]; + vector master_addresses; + vector add_tables; + vector remove_tables; + vector bootstrap_ids_to_add; + string new_producer_universe_id = ""; + bool remove_table_ignore_errors = false; + + vector newElem, *lst; + if (args[1] == "set_master_addresses") { + lst = &master_addresses; + } else if (args[1] == "add_table") { + lst = &add_tables; + } else if (args[1] == "remove_table") { + lst = &remove_tables; + if (args.size() == 4 && args[3] == "ignore-errors") { + remove_table_ignore_errors = true; + } + } else if (args[1] == "rename_id") { + lst = nullptr; + new_producer_universe_id = args[2]; + } else { + return ClusterAdminCli::kInvalidArguments; + } + + if (lst) { + boost::split(newElem, args[2], boost::is_any_of(",")); + lst->insert(lst->end(), newElem.begin(), newElem.end()); + + if (args[1] == "add_table" && args.size() == 4) { + boost::split(bootstrap_ids_to_add, args[3], boost::is_any_of(",")); + } + } + + RETURN_NOT_OK_PREPEND( + client->AlterUniverseReplication( + producer_uuid, + master_addresses, + add_tables, + remove_tables, + bootstrap_ids_to_add, + new_producer_universe_id, + remove_table_ignore_errors), + Substitute("Unable to alter replication for universe $0", producer_uuid)); + + return Status::OK(); + }); + + Register( + "change_xcluster_role", " ", [client](const CLIArguments& args) -> Status { + if (args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + auto xcluster_role = args[0]; + if (xcluster_role == "STANDBY") { + return client->ChangeXClusterRole(cdc::XClusterRole::STANDBY); + } + if (xcluster_role == "ACTIVE") { + return client->ChangeXClusterRole(cdc::XClusterRole::ACTIVE); + } + return STATUS( + InvalidArgument, Format("Expected one of STANDBY OR ACTIVE, found $0", args[0])); + }); + + Register( + "set_universe_replication_enabled", " (0|1)", + [client](const CLIArguments& args) -> Status { + if (args.size() < 2) { + return ClusterAdminCli::kInvalidArguments; + } + const string producer_id = args[0]; + const bool is_enabled = VERIFY_RESULT(CheckedStoi(args[1])) != 0; + RETURN_NOT_OK_PREPEND( + client->SetUniverseReplicationEnabled(producer_id, is_enabled), + Substitute( + "Unable to $0 replication for universe $1", + is_enabled ? "enable" : "disable", + producer_id)); + return Status::OK(); + }); + + Register( + "bootstrap_cdc_producer", " ", + [client](const CLIArguments& args) -> Status { + if (args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + + vector table_ids; + boost::split(table_ids, args[0], boost::is_any_of(",")); + + RETURN_NOT_OK_PREPEND( + client->BootstrapProducer(table_ids), "Unable to bootstrap CDC producer"); + return Status::OK(); + }); + + Register( + "wait_for_replication_drain", + Format( + " " + " [ | $0 ]", + kMinus), + [client](const CLIArguments& args) -> Status { + RETURN_NOT_OK(CheckArgumentsCount(args.size(), 1, 3)); + vector stream_ids; + boost::split(stream_ids, args[0], boost::is_any_of(",")); + string target_time; + if (args.size() == 2) { + target_time = args[1]; + } else if (args.size() == 3) { + if (args[1] != kMinus) { + return ClusterAdminCli::kInvalidArguments; + } + target_time = "-" + args[2]; + } + + return client->WaitForReplicationDrain(stream_ids, target_time); + }); + + Register( + "setup_namespace_universe_replication", + " ", + [client](const CLIArguments& args) -> Status { + RETURN_NOT_OK(CheckArgumentsCount(args.size(), 3, 3)); + const string producer_uuid = args[0]; + vector producer_addresses; + boost::split(producer_addresses, args[1], boost::is_any_of(",")); + TypedNamespaceName producer_namespace = VERIFY_RESULT(ParseNamespaceName(args[2])); + + RETURN_NOT_OK_PREPEND( + client->SetupNSUniverseReplication( + producer_uuid, producer_addresses, producer_namespace), + Substitute("Unable to setup namespace replication from universe $0", producer_uuid)); + return Status::OK(); + }); + + Register( + "get_replication_status", " []", + [client](const CLIArguments& args) -> Status { + if (args.size() != 0 && args.size() != 1) { + return ClusterAdminCli::kInvalidArguments; + } + const string producer_universe_uuid = args.size() == 1 ? args[0] : ""; + RETURN_NOT_OK_PREPEND( + client->GetReplicationInfo(producer_universe_uuid), "Unable to get replication status"); + return Status::OK(); + }); + + RegisterJson( + "get_xcluster_estimated_data_loss", "", + [client](const CLIArguments& args) -> Result { + return client->GetXClusterEstimatedDataLoss(); + }); + + RegisterJson( + "get_xcluster_safe_time", "", + [client](const CLIArguments& args) -> Result { + return client->GetXClusterSafeTime(); + }); } // NOLINT, prevents long function message Result> ResolveTableNames( - ClusterAdminClientClass* client, + ClusterAdminClient* client, CLIArgumentsIterator i, const CLIArgumentsIterator& end, const TailArgumentsProcessor& tail_processor, @@ -1094,7 +1989,7 @@ Result> ResolveTableNames( return tables; } -Result ResolveSingleTableName(ClusterAdminClientClass* client, +Result ResolveSingleTableName(ClusterAdminClient* client, CLIArgumentsIterator i, const CLIArgumentsIterator& end, TailArgumentsProcessor tail_processor) { @@ -1123,7 +2018,7 @@ Status CheckArgumentsCount(size_t count, size_t min, size_t max) { } // namespace yb int main(int argc, char** argv) { - yb::Status s = yb::tools::enterprise::ClusterAdminCli().Run(argc, argv); + yb::Status s = yb::tools::ClusterAdminCli().Run(argc, argv); if (s.ok()) { return 0; } diff --git a/src/yb/tools/yb-admin_cli.h b/src/yb/tools/yb-admin_cli.h index 7a8f2cd31969..b9133fb4b006 100644 --- a/src/yb/tools/yb-admin_cli.h +++ b/src/yb/tools/yb-admin_cli.h @@ -50,8 +50,6 @@ class YBTableName; namespace tools { -typedef enterprise::ClusterAdminClient ClusterAdminClientClass; - // Tool to administer a cluster from the CLI. class ClusterAdminCli { public: @@ -77,7 +75,7 @@ class ClusterAdminCli { void RegisterJson(std::string&& cmd_name, std::string&& cmd_args, JsonAction&& action); void SetUsage(const std::string& prog_name); - virtual void RegisterCommandHandlers(ClusterAdminClientClass* client); + virtual void RegisterCommandHandlers(ClusterAdminClient* client); private: Status RunCommand( @@ -92,14 +90,14 @@ using TailArgumentsProcessor = std::function; Result> ResolveTableNames( - ClusterAdminClientClass* client, + ClusterAdminClient* client, CLIArgumentsIterator i, const CLIArgumentsIterator& end, const TailArgumentsProcessor& tail_processor = TailArgumentsProcessor(), bool allow_namespace_only = false); Result ResolveSingleTableName( - ClusterAdminClientClass* client, + ClusterAdminClient* client, CLIArgumentsIterator i, const CLIArgumentsIterator& end, TailArgumentsProcessor tail_processor = TailArgumentsProcessor()); diff --git a/ent/src/yb/tools/yb-admin_client_ent_test.cc b/src/yb/tools/yb-admin_client-test.cc similarity index 96% rename from ent/src/yb/tools/yb-admin_client_ent_test.cc rename to src/yb/tools/yb-admin_client-test.cc index 5329d4ea661a..02d20bf52010 100644 --- a/ent/src/yb/tools/yb-admin_client_ent_test.cc +++ b/src/yb/tools/yb-admin_client-test.cc @@ -12,6 +12,7 @@ #include +#include "yb/master/master_backup.pb.h" #include "yb/tools/yb-admin_client.h" #include "yb/util/backoff_waiter.h" #include "yb/util/pb_util.h" @@ -25,13 +26,12 @@ DECLARE_bool(TEST_hang_on_namespace_transition); namespace yb { namespace tools { -namespace enterprise { // Tests for the client. Used to verify behaviour that cannot be verified by using yb-admin as an // external process. class ClusterAdminClientTest : public pgwrapper::PgCommandTestBase { public: - std::unique_ptr cluster_admin_client_; + std::unique_ptr cluster_admin_client_; protected: ClusterAdminClientTest() : pgwrapper::PgCommandTestBase(false, false) {} @@ -39,7 +39,7 @@ class ClusterAdminClientTest : public pgwrapper::PgCommandTestBase { void SetUp() override { pgwrapper::PgCommandTestBase::SetUp(); ASSERT_OK(CreateClient()); - cluster_admin_client_ = std::make_unique( + cluster_admin_client_ = std::make_unique( cluster_->GetMasterAddresses(), MonoDelta::FromSeconds(60)); ASSERT_OK(cluster_admin_client_->Init()); } @@ -147,6 +147,5 @@ TEST_F( ASSERT_OK(cluster_admin_client_->CreateNamespaceSnapshot(database)); } -} // namespace enterprise } // namespace tools } // namespace yb diff --git a/src/yb/tools/yb-admin_client.cc b/src/yb/tools/yb-admin_client.cc index 0f47a16aa493..d6506368a564 100644 --- a/src/yb/tools/yb-admin_client.cc +++ b/src/yb/tools/yb-admin_client.cc @@ -34,6 +34,7 @@ #include #include +#include #include #include @@ -43,6 +44,7 @@ #include #include +#include "yb/cdc/cdc_service.h" #include "yb/client/client.h" #include "yb/client/table.h" #include "yb/client/table_creator.h" @@ -50,6 +52,7 @@ #include "yb/client/table_info.h" #include "yb/common/json_util.h" +#include "yb/common/ql_type_util.h" #include "yb/common/redis_constants_common.h" #include "yb/common/transaction.h" #include "yb/common/wire_protocol.h" @@ -66,6 +69,7 @@ #include "yb/master/master_cluster.proxy.h" #include "yb/master/master_ddl.proxy.h" #include "yb/master/master_encryption.proxy.h" +#include "yb/master/master_error.h" #include "yb/master/master_replication.proxy.h" #include "yb/master/master_defaults.h" #include "yb/master/master_util.h" @@ -73,10 +77,15 @@ #include "yb/rpc/messenger.h" #include "yb/rpc/proxy.h" +#include "yb/rpc/secure_stream.h" +#include "yb/server/secure.h" +#include "yb/tools/yb-admin_util.h" #include "yb/tserver/tserver_admin.proxy.h" #include "yb/tserver/tserver_service.proxy.h" +#include "yb/encryption/encryption_util.h" + #include "yb/util/format.h" #include "yb/util/net/net_util.h" #include "yb/util/protobuf_util.h" @@ -86,6 +95,7 @@ #include "yb/util/string_case.h" #include "yb/util/string_util.h" #include "yb/util/flags.h" +#include "yb/util/tostring.h" DEFINE_UNKNOWN_bool(wait_if_no_leader_master, false, "When yb-admin connects to the cluster and no leader master is present, " @@ -97,17 +107,23 @@ DEFINE_UNKNOWN_string(certs_dir_name, "", DEFINE_UNKNOWN_string(client_node_name, "", "Client node name."); -DEFINE_UNKNOWN_bool( - disable_graceful_transition, false, +DEFINE_UNKNOWN_bool(disable_graceful_transition, false, "During a leader stepdown, disable graceful leadership transfer " "to an up to date peer"); +DEFINE_test_flag(int32, metadata_file_format_version, 0, + "Used in 'export_snapshot' metadata file format (0 means using latest format)."); + +DECLARE_bool(use_client_to_server_encryption); +DECLARE_int32(yb_client_admin_operation_timeout_sec); + // Maximum number of elements to dump on unexpected errors. static constexpr int MAX_NUM_ELEMENTS_TO_SHOW_ON_ERROR = 10; PB_ENUM_FORMATTERS(yb::PeerRole); PB_ENUM_FORMATTERS(yb::AppStatusPB::ErrorCode); PB_ENUM_FORMATTERS(yb::tablet::RaftGroupStatePB); +PB_ENUM_FORMATTERS(yb::master::SysSnapshotEntryPB::State); namespace yb { namespace tools { @@ -128,6 +144,7 @@ using client::YBClientBuilder; using client::YBTableName; using rpc::MessengerBuilder; using rpc::RpcController; +using pb_util::ParseFromSlice; using strings::Substitute; using tserver::TabletServerServiceProxy; using tserver::TabletServerAdminServiceProxy; @@ -141,10 +158,36 @@ using consensus::RaftPeerPB; using consensus::RunLeaderElectionRequestPB; using consensus::RunLeaderElectionResponsePB; +using master::BackupRowEntryPB; +using master::ChangeEncryptionInfoRequestPB; +using master::ChangeEncryptionInfoResponsePB; +using master::CreateSnapshotRequestPB; +using master::CreateSnapshotResponsePB; +using master::DeleteSnapshotRequestPB; +using master::DeleteSnapshotResponsePB; +using master::IdPairPB; +using master::ImportSnapshotMetaRequestPB; +using master::ImportSnapshotMetaResponsePB; +using master::ImportSnapshotMetaResponsePB_TableMetaPB; using master::ListMastersRequestPB; using master::ListMastersResponsePB; +using master::ListSnapshotRestorationsRequestPB; +using master::ListSnapshotRestorationsResponsePB; +using master::ListSnapshotsRequestPB; +using master::ListSnapshotsResponsePB; +using master::ListTablesRequestPB; +using master::ListTablesResponsePB; using master::ListTabletServersRequestPB; using master::ListTabletServersResponsePB; +using master::RestoreSnapshotRequestPB; +using master::RestoreSnapshotResponsePB; +using master::SnapshotInfoPB; +using master::SysNamespaceEntryPB; +using master::SysRowEntry; +using master::SysRowEntryType; +using master::SysSnapshotEntryPB; +using master::SysTablesEntryPB; +using master::SysUDTypeEntryPB; using master::TabletLocationsPB; using master::TSInfoPB; @@ -292,6 +335,69 @@ DotStringParts SplitByDot(const std::string& str) { return result; } +template +Result SnapshotScheduleInfoToJson( + const master::SnapshotScheduleInfoPB& schedule, Allocator* allocator) { + rapidjson::Value json_schedule(rapidjson::kObjectType); + AddStringField( + "id", VERIFY_RESULT(FullyDecodeSnapshotScheduleId(schedule.id())).ToString(), &json_schedule, + allocator); + + const auto& filter = schedule.options().filter(); + string filter_output; + // The user input should only have 1 entry, at namespace level. + if (filter.tables().tables_size() == 1) { + const auto& table_id = filter.tables().tables(0); + if (table_id.has_namespace_()) { + string database_type; + if (table_id.namespace_().database_type() == YQL_DATABASE_PGSQL) { + database_type = "ysql"; + } else if (table_id.namespace_().database_type() == YQL_DATABASE_CQL) { + database_type = "ycql"; + } + if (!database_type.empty()) { + filter_output = Format("$0.$1", database_type, table_id.namespace_().name()); + } + } + } + // If the user input was non standard, just display the whole debug PB. + if (filter_output.empty()) { + filter_output = filter.ShortDebugString(); + DCHECK(false) << "Non standard filter " << filter_output; + } + rapidjson::Value options(rapidjson::kObjectType); + AddStringField("filter", filter_output, &options, allocator); + auto interval_min = schedule.options().interval_sec() / MonoTime::kSecondsPerMinute; + AddStringField("interval", Format("$0 min", interval_min), &options, allocator); + auto retention_min = schedule.options().retention_duration_sec() / MonoTime::kSecondsPerMinute; + AddStringField("retention", Format("$0 min", retention_min), &options, allocator); + auto delete_time = HybridTime::FromPB(schedule.options().delete_time()); + if (delete_time) { + AddStringField("delete_time", HybridTimeToString(delete_time), &options, allocator); + } + + json_schedule.AddMember("options", options, *allocator); + rapidjson::Value json_snapshots(rapidjson::kArrayType); + for (const auto& snapshot : schedule.snapshots()) { + rapidjson::Value json_snapshot(rapidjson::kObjectType); + AddStringField( + "id", VERIFY_RESULT(FullyDecodeTxnSnapshotId(snapshot.id())).ToString(), &json_snapshot, + allocator); + auto snapshot_ht = HybridTime::FromPB(snapshot.entry().snapshot_hybrid_time()); + AddStringField("snapshot_time", HybridTimeToString(snapshot_ht), &json_snapshot, allocator); + auto previous_snapshot_ht = + HybridTime::FromPB(snapshot.entry().previous_snapshot_hybrid_time()); + if (previous_snapshot_ht) { + AddStringField( + "previous_snapshot_time", HybridTimeToString(previous_snapshot_ht), &json_snapshot, + allocator); + } + json_snapshots.PushBack(json_snapshot, *allocator); + } + json_schedule.AddMember("snapshots", json_snapshots, *allocator); + return json_schedule; +} + } // anonymous namespace class TableNameResolver::Impl { @@ -1638,7 +1744,7 @@ Status ClusterAdminClient::FlushTables(const std::vector& table_nam bool is_compaction) { RETURN_NOT_OK(yb_client_->FlushTables(table_names, add_indexes, timeout_secs, is_compaction)); cout << (is_compaction ? "Compacted " : "Flushed ") - << ToString(table_names) << " tables" + << yb::ToString(table_names) << " tables" << (add_indexes ? " and associated indexes." : ".") << endl; return Status::OK(); } @@ -1650,7 +1756,7 @@ Status ClusterAdminClient::FlushTablesById( bool is_compaction) { RETURN_NOT_OK(yb_client_->FlushTables(table_ids, add_indexes, timeout_secs, is_compaction)); cout << (is_compaction ? "Compacted " : "Flushed ") - << ToString(table_ids) << " tables" + << yb::ToString(table_ids) << " tables" << (add_indexes ? " and associated indexes." : ".") << endl; return Status::OK(); } @@ -2250,6 +2356,1687 @@ Result ClusterAdminClient::BuildTableNameResolver( tables, VERIFY_RESULT(yb_client_->ListTables()), std::move(namespace_ids)); } +Result ClusterAdminClient::ListSnapshots(const ListSnapshotsFlags& flags) { + ListSnapshotsResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + ListSnapshotsRequestPB req; + req.set_list_deleted_snapshots(flags.Test(ListSnapshotsFlag::SHOW_DELETED)); + auto* flags_pb = req.mutable_detail_options(); + // Explicitly set all boolean fields as the defaults of this proto are inconsistent. + if (flags.Test(ListSnapshotsFlag::SHOW_DETAILS)) { + flags_pb->set_show_namespace_details(true); + flags_pb->set_show_udtype_details(true); + flags_pb->set_show_table_details(true); + flags_pb->set_show_tablet_details(false); + } else { + flags_pb->set_show_namespace_details(false); + flags_pb->set_show_udtype_details(false); + flags_pb->set_show_table_details(false); + flags_pb->set_show_tablet_details(false); + } + return master_backup_proxy_->ListSnapshots(req, &resp, rpc); + })); + return resp; +} + +Status ClusterAdminClient::CreateSnapshot( + const vector& tables, const bool add_indexes, const int flush_timeout_secs) { + if (flush_timeout_secs > 0) { + const auto status = FlushTables(tables, add_indexes, flush_timeout_secs, false); + if (status.IsTimedOut()) { + cout << status.ToString(false) << " (ignored)" << endl; + } else if (!status.ok() && !status.IsNotFound()) { + return status; + } + } + + CreateSnapshotResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + CreateSnapshotRequestPB req; + for (const YBTableName& table_name : tables) { + table_name.SetIntoTableIdentifierPB(req.add_tables()); + } + + req.set_add_indexes(add_indexes); + req.set_add_ud_types(true); // No-op for YSQL. + req.set_transaction_aware(true); + return master_backup_proxy_->CreateSnapshot(req, &resp, rpc); + })); + + cout << "Started snapshot creation: " << SnapshotIdToString(resp.snapshot_id()) << endl; + return Status::OK(); +} + +Status ClusterAdminClient::CreateNamespaceSnapshot(const TypedNamespaceName& ns) { + ListTablesResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + ListTablesRequestPB req; + + req.mutable_namespace_()->set_name(ns.name); + req.mutable_namespace_()->set_database_type(ns.db_type); + req.set_exclude_system_tables(true); + req.add_relation_type_filter(master::USER_TABLE_RELATION); + req.add_relation_type_filter(master::INDEX_TABLE_RELATION); + req.add_relation_type_filter(master::MATVIEW_TABLE_RELATION); + return master_ddl_proxy_->ListTables(req, &resp, rpc); + })); + + if (resp.tables_size() == 0) { + return STATUS_FORMAT(InvalidArgument, "No tables found in namespace: $0", ns.name); + } + + vector tables(resp.tables_size()); + for (int i = 0; i < resp.tables_size(); ++i) { + const auto& table = resp.tables(i); + tables[i].set_table_id(table.id()); + tables[i].set_namespace_id(table.namespace_().id()); + tables[i].set_pgschema_name(table.pgschema_name()); + + RSTATUS_DCHECK( + table.relation_type() == master::USER_TABLE_RELATION || + table.relation_type() == master::INDEX_TABLE_RELATION || + table.relation_type() == master::MATVIEW_TABLE_RELATION, + InternalError, Format("Invalid relation type: $0", table.relation_type())); + RSTATUS_DCHECK_EQ( + table.namespace_().name(), ns.name, InternalError, + Format("Invalid namespace name: $0", table.namespace_().name())); + RSTATUS_DCHECK_EQ( + table.namespace_().database_type(), ns.db_type, InternalError, + Format( + "Invalid namespace type: $0", + YQLDatabase_Name(table.namespace_().database_type()))); + } + + return CreateSnapshot(tables, /* add_indexes */ false); +} + +Result ClusterAdminClient::ListSnapshotRestorations( + const TxnSnapshotRestorationId& restoration_id) { + master::ListSnapshotRestorationsResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + master::ListSnapshotRestorationsRequestPB req; + if (restoration_id) { + req.set_restoration_id(restoration_id.data(), restoration_id.size()); + } + return master_backup_proxy_->ListSnapshotRestorations(req, &resp, rpc); + })); + return resp; +} + +Result ClusterAdminClient::CreateSnapshotSchedule( + const client::YBTableName& keyspace, MonoDelta interval, MonoDelta retention) { + master::CreateSnapshotScheduleResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + master::CreateSnapshotScheduleRequestPB req; + + auto& options = *req.mutable_options(); + auto& filter_tables = *options.mutable_filter()->mutable_tables()->mutable_tables(); + keyspace.SetIntoTableIdentifierPB(filter_tables.Add()); + + options.set_interval_sec(interval.ToSeconds()); + options.set_retention_duration_sec(retention.ToSeconds()); + return master_backup_proxy_->CreateSnapshotSchedule(req, &resp, rpc); + })); + + rapidjson::Document document; + document.SetObject(); + + AddStringField( + "schedule_id", + VERIFY_RESULT(FullyDecodeSnapshotScheduleId(resp.snapshot_schedule_id())).ToString(), + &document, &document.GetAllocator()); + return document; +} + +Result ClusterAdminClient::ListSnapshotSchedules( + const SnapshotScheduleId& schedule_id) { + master::ListSnapshotSchedulesResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [this, &resp, &schedule_id](RpcController* rpc) { + master::ListSnapshotSchedulesRequestPB req; + if (schedule_id) { + req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); + } + return master_backup_proxy_->ListSnapshotSchedules(req, &resp, rpc); + })); + + rapidjson::Document result; + result.SetObject(); + rapidjson::Value json_schedules(rapidjson::kArrayType); + for (const auto& schedule : resp.schedules()) { + json_schedules.PushBack( + VERIFY_RESULT(SnapshotScheduleInfoToJson(schedule, &result.GetAllocator())), + result.GetAllocator()); + } + result.AddMember("schedules", json_schedules, result.GetAllocator()); + return result; +} + +Result ClusterAdminClient::DeleteSnapshotSchedule( + const SnapshotScheduleId& schedule_id) { + master::DeleteSnapshotScheduleResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [this, &resp, &schedule_id](RpcController* rpc) { + master::DeleteSnapshotScheduleRequestPB req; + req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); + + return master_backup_proxy_->DeleteSnapshotSchedule(req, &resp, rpc); + })); + + rapidjson::Document document; + document.SetObject(); + AddStringField("schedule_id", schedule_id.ToString(), &document, &document.GetAllocator()); + return document; +} + +bool SnapshotSuitableForRestoreAt(const SysSnapshotEntryPB& entry, HybridTime restore_at) { + return (entry.state() == master::SysSnapshotEntryPB::COMPLETE || + entry.state() == master::SysSnapshotEntryPB::CREATING) && + HybridTime::FromPB(entry.snapshot_hybrid_time()) >= restore_at && + HybridTime::FromPB(entry.previous_snapshot_hybrid_time()) < restore_at; +} + +Result ClusterAdminClient::SuitableSnapshotId( + const SnapshotScheduleId& schedule_id, HybridTime restore_at, CoarseTimePoint deadline) { + for (;;) { + auto last_snapshot_time = HybridTime::kMin; + { + RpcController rpc; + rpc.set_deadline(deadline); + master::ListSnapshotSchedulesRequestPB req; + master::ListSnapshotSchedulesResponsePB resp; + if (schedule_id) { + req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); + } + + RETURN_NOT_OK_PREPEND( + master_backup_proxy_->ListSnapshotSchedules(req, &resp, &rpc), + "Failed to list snapshot schedules"); + + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + + if (resp.schedules().size() < 1) { + return STATUS_FORMAT(InvalidArgument, "Unknown schedule: $0", schedule_id); + } + + for (const auto& snapshot : resp.schedules()[0].snapshots()) { + auto snapshot_hybrid_time = HybridTime::FromPB(snapshot.entry().snapshot_hybrid_time()); + last_snapshot_time = std::max(last_snapshot_time, snapshot_hybrid_time); + if (SnapshotSuitableForRestoreAt(snapshot.entry(), restore_at)) { + return VERIFY_RESULT(FullyDecodeTxnSnapshotId(snapshot.id())); + } + } + if (last_snapshot_time > restore_at) { + return STATUS_FORMAT( + IllegalState, "Cannot restore at $0, last snapshot: $1, snapshots: $2", restore_at, + last_snapshot_time, resp.schedules()[0].snapshots()); + } + } + RpcController rpc; + rpc.set_deadline(deadline); + master::CreateSnapshotRequestPB req; + master::CreateSnapshotResponsePB resp; + req.set_schedule_id(schedule_id.data(), schedule_id.size()); + RETURN_NOT_OK_PREPEND( + master_backup_proxy_->CreateSnapshot(req, &resp, &rpc), "Failed to create snapshot"); + if (resp.has_error()) { + auto status = StatusFromPB(resp.error().status()); + if (master::MasterError(status) == master::MasterErrorPB::PARALLEL_SNAPSHOT_OPERATION) { + std::this_thread::sleep_until(std::min(deadline, CoarseMonoClock::now() + 1s)); + continue; + } + return status; + } + return FullyDecodeTxnSnapshotId(resp.snapshot_id()); + } +} + +Status ClusterAdminClient::DisableTabletSplitsDuringRestore(CoarseTimePoint deadline) { + // TODO(Sanket): Eventually all of this logic needs to be moved + // to the master and exposed as APIs for the clients to consume. + const auto splitting_disabled_until = + CoarseMonoClock::Now() + MonoDelta::FromSeconds(kPitrSplitDisableDurationSecs); + // Disable splitting and then wait for all pending splits to complete before + // starting restoration. + VERIFY_RESULT_PREPEND( + DisableTabletSplitsInternal(kPitrSplitDisableDurationSecs * 1000, kPitrFeatureName), + "Failed to disable tablet split before restore."); + + while (CoarseMonoClock::Now() < std::min(splitting_disabled_until, deadline)) { + // Wait for existing split operations to complete. + const auto resp = VERIFY_RESULT_PREPEND( + IsTabletSplittingCompleteInternal( + true /* wait_for_parent_deletion */, + deadline - CoarseMonoClock::now() /* timeout */), + "Tablet splitting did not complete. Cannot restore."); + if (resp.is_tablet_splitting_complete()) { + break; + } + SleepFor(MonoDelta::FromMilliseconds(kPitrSplitDisableCheckFreqMs)); + } + + if (CoarseMonoClock::now() >= deadline) { + return STATUS(TimedOut, "Timed out waiting for tablet splitting to complete."); + } + + // Return if we have used almost all of our time in waiting for splitting to complete, + // since we can't guarantee that another split does not start. + if (CoarseMonoClock::now() + MonoDelta::FromSeconds(3) >= splitting_disabled_until) { + return STATUS( + TimedOut, "Not enough time after disabling splitting to disable ", "splitting again."); + } + + // Disable for kPitrSplitDisableDurationSecs again so the restore has the full amount of time with + // splitting disables. This overwrites the previous value since the feature_name is the same so + // overall the time is still kPitrSplitDisableDurationSecs. + VERIFY_RESULT_PREPEND( + DisableTabletSplitsInternal(kPitrSplitDisableDurationSecs * 1000, kPitrFeatureName), + "Failed to disable tablet split before restore."); + + return Status::OK(); +} + +Result ClusterAdminClient::RestoreSnapshotScheduleDeprecated( + const SnapshotScheduleId& schedule_id, HybridTime restore_at) { + auto deadline = CoarseMonoClock::now() + timeout_; + + // Disable splitting for the entire run of restore. + RETURN_NOT_OK(DisableTabletSplitsDuringRestore(deadline)); + + // Get the suitable snapshot to restore from. + auto snapshot_id = VERIFY_RESULT(SuitableSnapshotId(schedule_id, restore_at, deadline)); + + for (;;) { + RpcController rpc; + rpc.set_deadline(deadline); + master::ListSnapshotsRequestPB req; + req.set_snapshot_id(snapshot_id.data(), snapshot_id.size()); + master::ListSnapshotsResponsePB resp; + RETURN_NOT_OK_PREPEND( + master_backup_proxy_->ListSnapshots(req, &resp, &rpc), "Failed to list snapshots"); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + if (resp.snapshots().size() != 1) { + return STATUS_FORMAT( + IllegalState, "Wrong number of snapshots received $0", resp.snapshots().size()); + } + if (resp.snapshots()[0].entry().state() == master::SysSnapshotEntryPB::COMPLETE) { + if (SnapshotSuitableForRestoreAt(resp.snapshots()[0].entry(), restore_at)) { + break; + } + return STATUS_FORMAT(IllegalState, "Snapshot is not suitable for restore at $0", restore_at); + } + auto now = CoarseMonoClock::now(); + if (now >= deadline) { + return STATUS_FORMAT(TimedOut, "Timed out to complete a snapshot $0", snapshot_id); + } + std::this_thread::sleep_until(std::min(deadline, now + 100ms)); + } + + RpcController rpc; + rpc.set_deadline(deadline); + RestoreSnapshotRequestPB req; + RestoreSnapshotResponsePB resp; + req.set_snapshot_id(snapshot_id.data(), snapshot_id.size()); + req.set_restore_ht(restore_at.ToUint64()); + RETURN_NOT_OK_PREPEND( + master_backup_proxy_->RestoreSnapshot(req, &resp, &rpc), "Failed to restore snapshot"); + + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + + auto restoration_id = VERIFY_RESULT(FullyDecodeTxnSnapshotRestorationId(resp.restoration_id())); + + rapidjson::Document document; + document.SetObject(); + + AddStringField("snapshot_id", snapshot_id.ToString(), &document, &document.GetAllocator()); + AddStringField("restoration_id", restoration_id.ToString(), &document, &document.GetAllocator()); + + return document; +} + +Result ClusterAdminClient::RestoreSnapshotSchedule( + const SnapshotScheduleId& schedule_id, HybridTime restore_at) { + auto deadline = CoarseMonoClock::now() + timeout_; + + RpcController rpc; + rpc.set_deadline(deadline); + master::RestoreSnapshotScheduleRequestPB req; + master::RestoreSnapshotScheduleResponsePB resp; + req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); + req.set_restore_ht(restore_at.ToUint64()); + + Status s = master_backup_proxy_->RestoreSnapshotSchedule(req, &resp, &rpc); + if (!s.ok()) { + if (s.IsRemoteError() && + rpc.error_response()->code() == rpc::ErrorStatusPB::ERROR_NO_SUCH_METHOD) { + cout << "WARNING: fallback to RestoreSnapshotScheduleDeprecated." << endl; + return RestoreSnapshotScheduleDeprecated(schedule_id, restore_at); + } + RETURN_NOT_OK_PREPEND( + s, Format("Failed to restore snapshot from schedule: $0", schedule_id.ToString())); + } + + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + + auto snapshot_id = VERIFY_RESULT(FullyDecodeTxnSnapshotId(resp.snapshot_id())); + auto restoration_id = VERIFY_RESULT(FullyDecodeTxnSnapshotRestorationId(resp.restoration_id())); + + rapidjson::Document document; + document.SetObject(); + + AddStringField("snapshot_id", snapshot_id.ToString(), &document, &document.GetAllocator()); + AddStringField("restoration_id", restoration_id.ToString(), &document, &document.GetAllocator()); + + return document; +} + +Result ClusterAdminClient::EditSnapshotSchedule( + const SnapshotScheduleId& schedule_id, std::optional new_interval, + std::optional new_retention) { + master::EditSnapshotScheduleResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + master::EditSnapshotScheduleRequestPB req; + req.set_snapshot_schedule_id(schedule_id.data(), schedule_id.size()); + if (new_interval) { + req.set_interval_sec(new_interval->ToSeconds()); + } + if (new_retention) { + req.set_retention_duration_sec(new_retention->ToSeconds()); + } + return master_backup_proxy_->EditSnapshotSchedule(req, &resp, rpc); + })); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + rapidjson::Document document; + document.SetObject(); + rapidjson::Value json_schedule = + VERIFY_RESULT(SnapshotScheduleInfoToJson(resp.schedule(), &document.GetAllocator())); + document.AddMember("schedule", json_schedule, document.GetAllocator()); + return document; +} + +Status ClusterAdminClient::RestoreSnapshot(const string& snapshot_id, HybridTime timestamp) { + RestoreSnapshotResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + RestoreSnapshotRequestPB req; + req.set_snapshot_id(StringToSnapshotId(snapshot_id)); + if (timestamp) { + req.set_restore_ht(timestamp.ToUint64()); + } + return master_backup_proxy_->RestoreSnapshot(req, &resp, rpc); + })); + + cout << "Started restoring snapshot: " << snapshot_id << endl + << "Restoration id: " << FullyDecodeTxnSnapshotRestorationId(resp.restoration_id()) << endl; + if (timestamp) { + cout << "Restore at: " << timestamp << endl; + } + return Status::OK(); +} + +Status ClusterAdminClient::DeleteSnapshot(const std::string& snapshot_id) { + DeleteSnapshotResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + DeleteSnapshotRequestPB req; + req.set_snapshot_id(StringToSnapshotId(snapshot_id)); + return master_backup_proxy_->DeleteSnapshot(req, &resp, rpc); + })); + + cout << "Deleted snapshot: " << snapshot_id << endl; + return Status::OK(); +} + +Status ClusterAdminClient::CreateSnapshotMetaFile( + const string& snapshot_id, const string& file_name) { + ListSnapshotsResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + ListSnapshotsRequestPB req; + req.set_snapshot_id(StringToSnapshotId(snapshot_id)); + + // Format 0 - latest format (== Format 2 at the moment). + // Format -1 - old format (no 'namespace_name' in the Table entry). + // Format 1 - old format. + // Format 2 - new format. + if (FLAGS_TEST_metadata_file_format_version == 0 || + FLAGS_TEST_metadata_file_format_version >= 2) { + req.set_prepare_for_backup(true); + } + return master_backup_proxy_->ListSnapshots(req, &resp, rpc); + })); + + if (resp.snapshots_size() > 1) { + LOG(WARNING) << "Requested snapshot metadata for snapshot '" << snapshot_id << "', but got " + << resp.snapshots_size() << " snapshots in the response"; + } + + SnapshotInfoPB* snapshot = nullptr; + for (SnapshotInfoPB& snapshot_entry : *resp.mutable_snapshots()) { + if (SnapshotIdToString(snapshot_entry.id()) == snapshot_id) { + snapshot = &snapshot_entry; + break; + } + } + if (!snapshot) { + return STATUS_FORMAT( + InternalError, "Response contained $0 entries but no entry for snapshot '$1'", + resp.snapshots_size(), snapshot_id); + } + + if (FLAGS_TEST_metadata_file_format_version == -1) { + // Remove 'namespace_name' from SysTablesEntryPB. + SysSnapshotEntryPB& sys_entry = *snapshot->mutable_entry(); + for (SysRowEntry& entry : *sys_entry.mutable_entries()) { + if (entry.type() == SysRowEntryType::TABLE) { + auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); + meta.clear_namespace_name(); + entry.set_data(meta.SerializeAsString()); + } + } + } + + cout << "Exporting snapshot " << snapshot_id << " (" << snapshot->entry().state() << ") to file " + << file_name << endl; + + // Serialize snapshot protobuf to given path. + RETURN_NOT_OK(pb_util::WritePBContainerToPath( + Env::Default(), file_name, *snapshot, pb_util::OVERWRITE, pb_util::SYNC)); + + cout << "Snapshot metadata was saved into file: " << file_name << endl; + return Status::OK(); +} + +string ClusterAdminClient::GetDBTypeName(const SysNamespaceEntryPB& pb) { + const YQLDatabase db_type = GetDatabaseType(pb); + switch (db_type) { + case YQL_DATABASE_UNKNOWN: + return "UNKNOWN DB TYPE"; + case YQL_DATABASE_CQL: + return "YCQL keyspace"; + case YQL_DATABASE_PGSQL: + return "YSQL database"; + case YQL_DATABASE_REDIS: + return "YEDIS namespace"; + } + FATAL_INVALID_ENUM_VALUE(YQLDatabase, db_type); +} + +Status ClusterAdminClient::UpdateUDTypes( + QLTypePB* pb_type, bool* update_meta, const NSNameToNameMap& ns_name_to_name) { + return IterateAndDoForUDT( + pb_type, [update_meta, &ns_name_to_name](QLTypePB::UDTypeInfo* udtype_info) -> Status { + auto ns_it = ns_name_to_name.find(udtype_info->keyspace_name()); + if (ns_it == ns_name_to_name.end()) { + return STATUS_FORMAT( + InvalidArgument, "No metadata for keyspace '$0' referenced in UDType '$1'", + udtype_info->keyspace_name(), udtype_info->name()); + } + + if (udtype_info->keyspace_name() != ns_it->second) { + udtype_info->set_keyspace_name(ns_it->second); + *DCHECK_NOTNULL(update_meta) = true; + } + return Status::OK(); + }); +} + +Status ClusterAdminClient::ImportSnapshotMetaFile( + const string& file_name, + const TypedNamespaceName& keyspace, + const vector& tables) { + cout << "Read snapshot meta file " << file_name << endl; + + ImportSnapshotMetaRequestPB req; + + SnapshotInfoPB* const snapshot_info = req.mutable_snapshot(); + + // Read snapshot protobuf from given path. + RETURN_NOT_OK(pb_util::ReadPBContainerFromPath(Env::Default(), file_name, snapshot_info)); + + if (!snapshot_info->has_format_version()) { + SCHECK_EQ( + 0, snapshot_info->backup_entries_size(), InvalidArgument, + Format( + "Metadata file in Format 1 has backup entries from Format 2: $0", + snapshot_info->backup_entries_size())); + + // Repack PB data loaded in the old format. + // Init BackupSnapshotPB based on loaded SnapshotInfoPB. + SysSnapshotEntryPB& sys_entry = *snapshot_info->mutable_entry(); + snapshot_info->mutable_backup_entries()->Reserve(sys_entry.entries_size()); + for (SysRowEntry& entry : *sys_entry.mutable_entries()) { + snapshot_info->add_backup_entries()->mutable_entry()->Swap(&entry); + } + + sys_entry.clear_entries(); + snapshot_info->set_format_version(2); + } + + cout << "Importing snapshot " << SnapshotIdToString(snapshot_info->id()) << " (" + << snapshot_info->entry().state() << ")" << endl; + + // Map: Old namespace ID -> [Old name, New name] pair. + typedef pair NSNamePair; + typedef std::unordered_map NSIdToNameMap; + NSIdToNameMap ns_id_to_name; + NSNameToNameMap ns_name_to_name; + + size_t table_index = 0; + bool was_table_renamed = false; + for (BackupRowEntryPB& backup_entry : *snapshot_info->mutable_backup_entries()) { + SysRowEntry& entry = *backup_entry.mutable_entry(); + const YBTableName table_name = + table_index < tables.size() ? tables[table_index] : YBTableName(); + + switch (entry.type()) { + case SysRowEntryType::NAMESPACE: { + auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); + const string db_type = GetDBTypeName(meta); + cout << "Target imported " << db_type + << " name: " << (keyspace.name.empty() ? meta.name() : keyspace.name) << endl + << db_type << " being imported: " << meta.name() << endl; + + if (!keyspace.name.empty() && keyspace.name != meta.name()) { + ns_id_to_name[entry.id()] = NSNamePair(meta.name(), keyspace.name); + ns_name_to_name[meta.name()] = keyspace.name; + + meta.set_name(keyspace.name); + entry.set_data(meta.SerializeAsString()); + } else { + ns_id_to_name[entry.id()] = NSNamePair(meta.name(), meta.name()); + ns_name_to_name[meta.name()] = meta.name(); + } + break; + } + case SysRowEntryType::UDTYPE: { + // Note: UDT renaming is not supported. + auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); + const auto ns_it = ns_id_to_name.find(meta.namespace_id()); + cout << "Target imported user-defined type name: " + << (ns_it == ns_id_to_name.end() ? "[" + meta.namespace_id() + "]" + : ns_it->second.second) + << "." << meta.name() << endl + << "User-defined type being imported: " + << (ns_it == ns_id_to_name.end() ? "[" + meta.namespace_id() + "]" + : ns_it->second.first) + << "." << meta.name() << endl; + + bool update_meta = false; + // Update QLTypePB::UDTypeInfo::keyspace_name in the UDT params. + for (int i = 0; i < meta.field_names_size(); ++i) { + RETURN_NOT_OK(UpdateUDTypes(meta.mutable_field_types(i), &update_meta, ns_name_to_name)); + } + + if (update_meta) { + entry.set_data(meta.SerializeAsString()); + } + break; + } + case SysRowEntryType::TABLE: { + if (was_table_renamed && table_name.empty()) { + // Renaming is allowed for all tables OR for no one table. + return STATUS_FORMAT( + InvalidArgument, + "There is no name for table (including indexes) number: $0", + table_index); + } + + auto meta = VERIFY_RESULT(ParseFromSlice(entry.data())); + const auto colocated_prefix = meta.colocated() ? "colocated " : ""; + + if (meta.indexed_table_id().empty()) { + cout << "Table type: " << colocated_prefix << "table" << endl; + } else { + cout << "Table type: " << colocated_prefix << "index (attaching to the old table id " + << meta.indexed_table_id() << ")" << endl; + } + + if (!table_name.empty()) { + DCHECK(table_name.has_namespace()); + DCHECK(table_name.has_table()); + cout << "Target imported " << colocated_prefix << "table name: " << table_name.ToString() + << endl; + } else if (!keyspace.name.empty()) { + cout << "Target imported " << colocated_prefix << "table name: " << keyspace.name << "." + << meta.name() << endl; + } + + // Print old table name before the table renaming in the code below. + cout << (meta.colocated() ? "Colocated t" : "T") << "able being imported: " + << (meta.namespace_name().empty() ? "[" + meta.namespace_id() + "]" + : meta.namespace_name()) + << "." << meta.name() << endl; + + bool update_meta = false; + if (!table_name.empty() && table_name.table_name() != meta.name()) { + meta.set_name(table_name.table_name()); + update_meta = true; + was_table_renamed = true; + } + if (!keyspace.name.empty() && keyspace.name != meta.namespace_name()) { + meta.set_namespace_name(keyspace.name); + update_meta = true; + } + + if (meta.name().empty()) { + return STATUS(IllegalState, "Could not find table name from snapshot metadata"); + } + + // Update QLTypePB::UDTypeInfo::keyspace_name in all UDT params in the table Schema. + SchemaPB* const schema = meta.mutable_schema(); + // Recursively update ids in used user-defined types. + for (int i = 0; i < schema->columns_size(); ++i) { + QLTypePB* const pb_type = schema->mutable_columns(i)->mutable_type(); + RETURN_NOT_OK(UpdateUDTypes(pb_type, &update_meta, ns_name_to_name)); + } + + // Update the table name if needed. + if (update_meta) { + entry.set_data(meta.SerializeAsString()); + } + + ++table_index; + break; + } + default: + break; + } + } + + // RPC timeout is a function of the number of tables that needs to be imported. + ImportSnapshotMetaResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader( + &resp, + [&](RpcController* rpc) { return master_backup_proxy_->ImportSnapshotMeta(req, &resp, rpc); }, + timeout_ * std::max(static_cast(1), table_index))); + + const int kObjectColumnWidth = 16; + const auto pad_object_type = [](const string& s) { + return RightPadToWidth(s, kObjectColumnWidth); + }; + + cout << "Successfully applied snapshot." << endl + << pad_object_type("Object") << kColumnSep << RightPadToUuidWidth("Old ID") << kColumnSep + << RightPadToUuidWidth("New ID") << endl; + + const RepeatedPtrField& tables_meta = + resp.tables_meta(); + CreateSnapshotRequestPB snapshot_req; + CreateSnapshotResponsePB snapshot_resp; + + for (int i = 0; i < tables_meta.size(); ++i) { + const ImportSnapshotMetaResponsePB_TableMetaPB& table_meta = tables_meta.Get(i); + const string& new_table_id = table_meta.table_ids().new_id(); + + cout << pad_object_type("Keyspace") << kColumnSep << table_meta.namespace_ids().old_id() + << kColumnSep << table_meta.namespace_ids().new_id() << endl; + + if (!ImportSnapshotMetaResponsePB_TableType_IsValid(table_meta.table_type())) { + return STATUS_FORMAT(InternalError, "Found unknown table type: ", table_meta.table_type()); + } + + const string table_type = AllCapsToCamelCase( + ImportSnapshotMetaResponsePB_TableType_Name(table_meta.table_type())); + cout << pad_object_type(table_type) << kColumnSep << table_meta.table_ids().old_id() + << kColumnSep << new_table_id << endl; + + const RepeatedPtrField& udts_map = table_meta.ud_types_ids(); + for (int j = 0; j < udts_map.size(); ++j) { + const IdPairPB& pair = udts_map.Get(j); + cout << pad_object_type("UDType") << kColumnSep << pair.old_id() << kColumnSep + << pair.new_id() << endl; + } + + const RepeatedPtrField& tablets_map = table_meta.tablets_ids(); + for (int j = 0; j < tablets_map.size(); ++j) { + const IdPairPB& pair = tablets_map.Get(j); + cout << pad_object_type(Format("Tablet $0", j)) << kColumnSep << pair.old_id() << kColumnSep + << pair.new_id() << endl; + } + + RETURN_NOT_OK(yb_client_->WaitForCreateTableToFinish( + new_table_id, + CoarseMonoClock::Now() + + MonoDelta::FromSeconds(FLAGS_yb_client_admin_operation_timeout_sec))); + + snapshot_req.mutable_tables()->Add()->set_table_id(new_table_id); + } + + // All indexes already are in the request. Do not add them twice. + snapshot_req.set_add_indexes(false); + snapshot_req.set_transaction_aware(true); + snapshot_req.set_imported(true); + // Create new snapshot. + RETURN_NOT_OK(RequestMasterLeader(&snapshot_resp, [&](RpcController* rpc) { + return master_backup_proxy_->CreateSnapshot(snapshot_req, &snapshot_resp, rpc); + })); + + cout << pad_object_type("Snapshot") << kColumnSep << SnapshotIdToString(snapshot_info->id()) + << kColumnSep << SnapshotIdToString(snapshot_resp.snapshot_id()) << endl; + + return Status::OK(); +} + +Status ClusterAdminClient::ListReplicaTypeCounts(const YBTableName& table_name) { + vector tablet_ids, ranges; + RETURN_NOT_OK(yb_client_->GetTablets(table_name, 0, &tablet_ids, &ranges)); + master::GetTabletLocationsResponsePB resp; + RETURN_NOT_OK(RequestMasterLeader(&resp, [&](RpcController* rpc) { + master::GetTabletLocationsRequestPB req; + for (const auto& tablet_id : tablet_ids) { + req.add_tablet_ids(tablet_id); + } + return master_client_proxy_->GetTabletLocations(req, &resp, rpc); + })); + + struct ReplicaCounts { + int live_count; + int read_only_count; + string placement_uuid; + }; + std::map replica_map; + + std::cout << "Tserver ID\t\tPlacement ID\t\tLive count\t\tRead only count\n"; + + for (int tablet_idx = 0; tablet_idx < resp.tablet_locations_size(); tablet_idx++) { + const master::TabletLocationsPB& locs = resp.tablet_locations(tablet_idx); + for (int replica_idx = 0; replica_idx < locs.replicas_size(); replica_idx++) { + const auto& replica = locs.replicas(replica_idx); + const string& ts_uuid = replica.ts_info().permanent_uuid(); + const string& placement_uuid = + replica.ts_info().has_placement_uuid() ? replica.ts_info().placement_uuid() : ""; + bool is_replica_read_only = + replica.member_type() == consensus::PeerMemberType::PRE_OBSERVER || + replica.member_type() == consensus::PeerMemberType::OBSERVER; + int live_count = is_replica_read_only ? 0 : 1; + int read_only_count = 1 - live_count; + if (replica_map.count(ts_uuid) == 0) { + replica_map[ts_uuid].live_count = live_count; + replica_map[ts_uuid].read_only_count = read_only_count; + replica_map[ts_uuid].placement_uuid = placement_uuid; + } else { + ReplicaCounts* counts = &replica_map[ts_uuid]; + counts->live_count += live_count; + counts->read_only_count += read_only_count; + } + } + } + + for (auto const& tserver : replica_map) { + std::cout << tserver.first << "\t\t" << tserver.second.placement_uuid << "\t\t" + << tserver.second.live_count << "\t\t" << tserver.second.read_only_count + << std::endl; + } + + return Status::OK(); +} + +Status ClusterAdminClient::SetPreferredZones(const std::vector& preferred_zones) { + rpc::RpcController rpc; + master::SetPreferredZonesRequestPB req; + master::SetPreferredZonesResponsePB resp; + rpc.set_timeout(timeout_); + + std::set zones; + std::set visited_priorities; + + for (const string& zone : preferred_zones) { + if (zones.find(zone) != zones.end()) { + return STATUS_SUBSTITUTE( + InvalidArgument, "Invalid argument for preferred zone $0, values should not repeat", + zone); + } + + std::vector zone_priority_split = + strings::Split(zone, ":", strings::AllowEmpty()); + if (zone_priority_split.size() == 0 || zone_priority_split.size() > 2) { + return STATUS_SUBSTITUTE( + InvalidArgument, + "Invalid argument for preferred zone $0, should have format cloud.region.zone[:priority]", + zone); + } + + std::vector tokens = + strings::Split(zone_priority_split[0], ".", strings::AllowEmpty()); + if (tokens.size() != 3) { + return STATUS_SUBSTITUTE( + InvalidArgument, + "Invalid argument for preferred zone $0, should have format cloud.region.zone:[priority]", + zone); + } + + uint priority = 1; + + if (zone_priority_split.size() == 2) { + auto result = CheckedStoi(zone_priority_split[1]); + if (!result.ok() || result.get() < 1) { + return STATUS_SUBSTITUTE( + InvalidArgument, + "Invalid argument for preferred zone $0, priority should be non-zero positive integer", + zone); + } + priority = static_cast(result.get()); + } + + // Max priority if each zone has a unique priority value can only be the size of the input + // array + if (priority > preferred_zones.size()) { + return STATUS( + InvalidArgument, + "Priority value cannot be more than the number of zones in the preferred list since each " + "priority should be associated with at least one zone from the list"); + } + + CloudInfoPB* cloud_info = nullptr; + visited_priorities.insert(priority); + + while (req.multi_preferred_zones_size() < static_cast(priority)) { + req.add_multi_preferred_zones(); + } + + auto current_list = req.mutable_multi_preferred_zones(priority - 1); + cloud_info = current_list->add_zones(); + + cloud_info->set_placement_cloud(tokens[0]); + cloud_info->set_placement_region(tokens[1]); + cloud_info->set_placement_zone(tokens[2]); + + zones.emplace(zone); + + if (priority == 1) { + // Handle old clusters which can only handle a single priority. New clusters will ignore this + // member as multi_preferred_zones is already set. + cloud_info = req.add_preferred_zones(); + cloud_info->set_placement_cloud(tokens[0]); + cloud_info->set_placement_region(tokens[1]); + cloud_info->set_placement_zone(tokens[2]); + } + } + + int size = static_cast(visited_priorities.size()); + if (size > 0 && (*(visited_priorities.rbegin()) != size)) { + return STATUS_SUBSTITUTE( + InvalidArgument, "Invalid argument, each priority should have at least one zone"); + } + + RETURN_NOT_OK(master_cluster_proxy_->SetPreferredZones(req, &resp, &rpc)); + + if (resp.has_error()) { + return STATUS(ServiceUnavailable, resp.error().status().message()); + } + + return Status::OK(); +} + +Status ClusterAdminClient::RotateUniverseKey(const std::string& key_path) { + return SendEncryptionRequest(key_path, true); +} + +Status ClusterAdminClient::DisableEncryption() { return SendEncryptionRequest("", false); } + +Status ClusterAdminClient::SendEncryptionRequest( + const std::string& key_path, bool enable_encryption) { + RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); + rpc::RpcController rpc; + rpc.set_timeout(timeout_); + + // Get the cluster config from the master leader. + master::ChangeEncryptionInfoRequestPB encryption_info_req; + master::ChangeEncryptionInfoResponsePB encryption_info_resp; + encryption_info_req.set_encryption_enabled(enable_encryption); + if (key_path != "") { + encryption_info_req.set_key_path(key_path); + } + RETURN_NOT_OK_PREPEND( + master_encryption_proxy_->ChangeEncryptionInfo( + encryption_info_req, &encryption_info_resp, &rpc), + "MasterServiceImpl::ChangeEncryptionInfo call fails.") + + if (encryption_info_resp.has_error()) { + return StatusFromPB(encryption_info_resp.error().status()); + } + return Status::OK(); +} + +Status ClusterAdminClient::IsEncryptionEnabled() { + RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); + rpc::RpcController rpc; + rpc.set_timeout(timeout_); + + master::IsEncryptionEnabledRequestPB req; + master::IsEncryptionEnabledResponsePB resp; + RETURN_NOT_OK_PREPEND( + master_encryption_proxy_->IsEncryptionEnabled(req, &resp, &rpc), + "MasterServiceImpl::IsEncryptionEnabled call fails."); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + + std::cout << "Encryption status: " + << (resp.encryption_enabled() ? Format("ENABLED with key id $0", resp.key_id()) + : "DISABLED") + << std::endl; + return Status::OK(); +} + +Status ClusterAdminClient::AddUniverseKeyToAllMasters( + const std::string& key_id, const std::string& universe_key) { + RETURN_NOT_OK(encryption::EncryptionParams::IsValidKeySize( + universe_key.size() - encryption::EncryptionParams::kBlockSize)); + + master::AddUniverseKeysRequestPB req; + master::AddUniverseKeysResponsePB resp; + auto* universe_keys = req.mutable_universe_keys(); + (*universe_keys->mutable_map())[key_id] = universe_key; + + for (auto hp : VERIFY_RESULT(HostPort::ParseStrings(master_addr_list_, 7100))) { + rpc::RpcController rpc; + rpc.set_timeout(timeout_); + master::MasterEncryptionProxy proxy(proxy_cache_.get(), hp); + RETURN_NOT_OK_PREPEND( + proxy.AddUniverseKeys(req, &resp, &rpc), + Format("MasterServiceImpl::AddUniverseKeys call fails on host $0.", hp.ToString())); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + std::cout << Format("Successfully added key to the node $0.\n", hp.ToString()); + } + + return Status::OK(); +} + +Status ClusterAdminClient::AllMastersHaveUniverseKeyInMemory(const std::string& key_id) { + master::HasUniverseKeyInMemoryRequestPB req; + master::HasUniverseKeyInMemoryResponsePB resp; + req.set_version_id(key_id); + + for (auto hp : VERIFY_RESULT(HostPort::ParseStrings(master_addr_list_, 7100))) { + rpc::RpcController rpc; + rpc.set_timeout(timeout_); + master::MasterEncryptionProxy proxy(proxy_cache_.get(), hp); + RETURN_NOT_OK_PREPEND( + proxy.HasUniverseKeyInMemory(req, &resp, &rpc), + "MasterServiceImpl::ChangeEncryptionInfo call fails."); + + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + if (!resp.has_key()) { + return STATUS_FORMAT(TryAgain, "Node $0 does not have universe key in memory", hp); + } + + std::cout << Format( + "Node $0 has universe key in memory: $1\n", hp.ToString(), resp.has_key()); + } + + return Status::OK(); +} + +Status ClusterAdminClient::RotateUniverseKeyInMemory(const std::string& key_id) { + RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); + rpc::RpcController rpc; + rpc.set_timeout(timeout_); + + master::ChangeEncryptionInfoRequestPB req; + master::ChangeEncryptionInfoResponsePB resp; + req.set_encryption_enabled(true); + req.set_in_memory(true); + req.set_version_id(key_id); + RETURN_NOT_OK_PREPEND( + master_encryption_proxy_->ChangeEncryptionInfo(req, &resp, &rpc), + "MasterServiceImpl::ChangeEncryptionInfo call fails."); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + + std::cout << "Rotated universe key in memory\n"; + return Status::OK(); +} + +Status ClusterAdminClient::DisableEncryptionInMemory() { + RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); + rpc::RpcController rpc; + rpc.set_timeout(timeout_); + + master::ChangeEncryptionInfoRequestPB req; + master::ChangeEncryptionInfoResponsePB resp; + req.set_encryption_enabled(false); + RETURN_NOT_OK_PREPEND( + master_encryption_proxy_->ChangeEncryptionInfo(req, &resp, &rpc), + "MasterServiceImpl::ChangeEncryptionInfo call fails."); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + + std::cout << "Encryption disabled\n"; + return Status::OK(); +} + +Status ClusterAdminClient::WriteUniverseKeyToFile( + const std::string& key_id, const std::string& file_name) { + RETURN_NOT_OK_PREPEND(WaitUntilMasterLeaderReady(), "Wait for master leader failed!"); + rpc::RpcController rpc; + rpc.set_timeout(timeout_); + + master::GetUniverseKeyRegistryRequestPB req; + master::GetUniverseKeyRegistryResponsePB resp; + RETURN_NOT_OK_PREPEND( + master_encryption_proxy_->GetUniverseKeyRegistry(req, &resp, &rpc), + "MasterServiceImpl::ChangeEncryptionInfo call fails."); + if (resp.has_error()) { + return StatusFromPB(resp.error().status()); + } + + auto universe_keys = resp.universe_keys(); + const auto& it = universe_keys.map().find(key_id); + if (it == universe_keys.map().end()) { + return STATUS_FORMAT(NotFound, "Could not find key with id $0", key_id); + } + + RETURN_NOT_OK(WriteStringToFile(Env::Default(), Slice(it->second), file_name)); + + std::cout << "Finished writing to file\n"; + return Status::OK(); +} + +Status ClusterAdminClient::CreateCDCSDKDBStream( + const TypedNamespaceName& ns, const std::string& checkpoint_type, + const std::string& record_type) { + HostPort ts_addr = VERIFY_RESULT(GetFirstRpcAddressForTS()); + auto cdc_proxy = std::make_unique(proxy_cache_.get(), ts_addr); + + cdc::CreateCDCStreamRequestPB req; + cdc::CreateCDCStreamResponsePB resp; + + req.set_namespace_name(ns.name); + + if (record_type == yb::ToString("ALL")) { + req.set_record_type(cdc::CDCRecordType::ALL); + } else { + req.set_record_type(cdc::CDCRecordType::CHANGE); + } + + req.set_record_format(cdc::CDCRecordFormat::PROTO); + req.set_source_type(cdc::CDCRequestSource::CDCSDK); + if (checkpoint_type == yb::ToString("EXPLICIT")) { + req.set_checkpoint_type(cdc::CDCCheckpointType::EXPLICIT); + } else { + req.set_checkpoint_type(cdc::CDCCheckpointType::IMPLICIT); + } + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(cdc_proxy->CreateCDCStream(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error creating stream: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "CDC Stream ID: " << resp.db_stream_id() << endl; + return Status::OK(); +} + +Status ClusterAdminClient::CreateCDCStream(const TableId& table_id) { + master::CreateCDCStreamRequestPB req; + master::CreateCDCStreamResponsePB resp; + req.set_table_id(table_id); + req.mutable_options()->Reserve(3); + + auto record_type_option = req.add_options(); + record_type_option->set_key(cdc::kRecordType); + record_type_option->set_value(CDCRecordType_Name(cdc::CDCRecordType::CHANGE)); + + auto record_format_option = req.add_options(); + record_format_option->set_key(cdc::kRecordFormat); + record_format_option->set_value(CDCRecordFormat_Name(cdc::CDCRecordFormat::JSON)); + + auto source_type_option = req.add_options(); + source_type_option->set_key(cdc::kSourceType); + source_type_option->set_value(CDCRequestSource_Name(cdc::CDCRequestSource::XCLUSTER)); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->CreateCDCStream(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error creating stream: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "CDC Stream ID: " << resp.stream_id() << endl; + return Status::OK(); +} + +Status ClusterAdminClient::DeleteCDCSDKDBStream(const std::string& db_stream_id) { + master::DeleteCDCStreamRequestPB req; + master::DeleteCDCStreamResponsePB resp; + req.add_stream_id(db_stream_id); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->DeleteCDCStream(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error deleting stream: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "Successfully deleted Change Data Stream ID: " << db_stream_id << endl; + return Status::OK(); +} + +Status ClusterAdminClient::DeleteCDCStream(const std::string& stream_id, bool force_delete) { + master::DeleteCDCStreamRequestPB req; + master::DeleteCDCStreamResponsePB resp; + req.add_stream_id(stream_id); + req.set_force_delete(force_delete); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->DeleteCDCStream(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error deleting stream: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "Successfully deleted CDC Stream ID: " << stream_id << endl; + return Status::OK(); +} + +Status ClusterAdminClient::ListCDCStreams(const TableId& table_id) { + master::ListCDCStreamsRequestPB req; + master::ListCDCStreamsResponsePB resp; + req.set_id_type(yb::master::IdTypePB::TABLE_ID); + if (!table_id.empty()) { + req.set_table_id(table_id); + } + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->ListCDCStreams(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error getting CDC stream list: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "CDC Streams: \r\n" << resp.DebugString(); + return Status::OK(); +} + +Status ClusterAdminClient::ListCDCSDKStreams(const std::string& namespace_name) { + master::ListCDCStreamsRequestPB req; + master::ListCDCStreamsResponsePB resp; + req.set_id_type(yb::master::IdTypePB::NAMESPACE_ID); + + if (!namespace_name.empty()) { + cout << "Filtering out DB streams for the namespace: " << namespace_name << "\n\n"; + master::GetNamespaceInfoResponsePB namespace_info_resp; + RETURN_NOT_OK(yb_client_->GetNamespaceInfo( + "", namespace_name, YQL_DATABASE_PGSQL, &namespace_info_resp)); + req.set_namespace_id(namespace_info_resp.namespace_().id()); + } + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->ListCDCStreams(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error getting CDC stream list: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "CDC Streams: \r\n" << resp.DebugString(); + return Status::OK(); +} + +Status ClusterAdminClient::GetCDCDBStreamInfo(const std::string& db_stream_id) { + master::GetCDCDBStreamInfoRequestPB req; + master::GetCDCDBStreamInfoResponsePB resp; + req.set_db_stream_id(db_stream_id); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->GetCDCDBStreamInfo(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error getting info corresponding to CDC db stream : " + << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "CDC DB Stream Info: \r\n" << resp.DebugString(); + return Status::OK(); +} + +Status ClusterAdminClient::WaitForSetupUniverseReplicationToFinish(const string& producer_uuid) { + master::IsSetupUniverseReplicationDoneRequestPB req; + req.set_producer_id(producer_uuid); + for (;;) { + master::IsSetupUniverseReplicationDoneResponsePB resp; + RpcController rpc; + rpc.set_timeout(timeout_); + Status s = master_replication_proxy_->IsSetupUniverseReplicationDone(req, &resp, &rpc); + + if (!s.ok() || resp.has_error()) { + LOG(WARNING) << "Encountered error while waiting for setup_universe_replication to complete" + << " : " << (!s.ok() ? s.ToString() : resp.error().status().message()); + } + if (resp.has_done() && resp.done()) { + return StatusFromPB(resp.replication_error()); + } + + // Still processing, wait and then loop again. + std::this_thread::sleep_for(100ms); + } +} + +Status ClusterAdminClient::SetupUniverseReplication( + const string& producer_uuid, const vector& producer_addresses, + const vector& tables, const vector& producer_bootstrap_ids) { + master::SetupUniverseReplicationRequestPB req; + master::SetupUniverseReplicationResponsePB resp; + req.set_producer_id(producer_uuid); + + req.mutable_producer_master_addresses()->Reserve(narrow_cast(producer_addresses.size())); + for (const auto& addr : producer_addresses) { + // HostPort::FromString() expects a default port. + auto hp = VERIFY_RESULT(HostPort::FromString(addr, master::kMasterDefaultPort)); + HostPortToPB(hp, req.add_producer_master_addresses()); + } + + req.mutable_producer_table_ids()->Reserve(narrow_cast(tables.size())); + for (const auto& table : tables) { + req.add_producer_table_ids(table); + } + + for (const auto& producer_bootstrap_id : producer_bootstrap_ids) { + req.add_producer_bootstrap_ids(producer_bootstrap_id); + } + + RpcController rpc; + rpc.set_timeout(timeout_); + auto setup_result_status = master_replication_proxy_->SetupUniverseReplication(req, &resp, &rpc); + + setup_result_status = WaitForSetupUniverseReplicationToFinish(producer_uuid); + + if (resp.has_error()) { + cout << "Error setting up universe replication: " << resp.error().status().message() + << endl; + Status status_from_error = StatusFromPB(resp.error().status()); + + return status_from_error; + } + + // Clean up config files if setup fails to complete. + if (!setup_result_status.ok()) { + cout << "Error waiting for universe replication setup to complete: " + << setup_result_status.message().ToBuffer() << endl; + return setup_result_status; + } + + cout << "Replication setup successfully" << endl; + return Status::OK(); +} + +Status ClusterAdminClient::DeleteUniverseReplication( + const std::string& producer_id, bool ignore_errors) { + master::DeleteUniverseReplicationRequestPB req; + master::DeleteUniverseReplicationResponsePB resp; + req.set_producer_id(producer_id); + req.set_ignore_errors(ignore_errors); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->DeleteUniverseReplication(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error deleting universe replication: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + if (resp.warnings().size() > 0) { + cout << "Encountered the following warnings while running delete_universe_replication:" + << endl; + for (const auto& warning : resp.warnings()) { + cout << " - " << warning.message() << endl; + } + } + + cout << "Replication deleted successfully" << endl; + return Status::OK(); +} + +Status ClusterAdminClient::AlterUniverseReplication( + const std::string& producer_uuid, + const std::vector& producer_addresses, + const std::vector& add_tables, + const std::vector& remove_tables, + const std::vector& producer_bootstrap_ids_to_add, + const std::string& new_producer_universe_id, + bool remove_table_ignore_errors) { + master::AlterUniverseReplicationRequestPB req; + master::AlterUniverseReplicationResponsePB resp; + req.set_producer_id(producer_uuid); + req.set_remove_table_ignore_errors(remove_table_ignore_errors); + + if (!producer_addresses.empty()) { + req.mutable_producer_master_addresses()->Reserve( + narrow_cast(producer_addresses.size())); + for (const auto& addr : producer_addresses) { + // HostPort::FromString() expects a default port. + auto hp = VERIFY_RESULT(HostPort::FromString(addr, master::kMasterDefaultPort)); + HostPortToPB(hp, req.add_producer_master_addresses()); + } + } + + if (!add_tables.empty()) { + req.mutable_producer_table_ids_to_add()->Reserve(narrow_cast(add_tables.size())); + for (const auto& table : add_tables) { + req.add_producer_table_ids_to_add(table); + } + + if (!producer_bootstrap_ids_to_add.empty()) { + // There msut be a bootstrap id for every table id. + if (producer_bootstrap_ids_to_add.size() != add_tables.size()) { + cout << "The number of bootstrap ids must equal the number of table ids. " + << "Use separate alter commands if only some tables are being bootstrapped." << endl; + return STATUS(InternalError, "Invalid number of bootstrap ids"); + } + + req.mutable_producer_bootstrap_ids_to_add()->Reserve( + narrow_cast(producer_bootstrap_ids_to_add.size())); + for (const auto& bootstrap_id : producer_bootstrap_ids_to_add) { + req.add_producer_bootstrap_ids_to_add(bootstrap_id); + } + } + } + + if (!remove_tables.empty()) { + req.mutable_producer_table_ids_to_remove()->Reserve(narrow_cast(remove_tables.size())); + for (const auto& table : remove_tables) { + req.add_producer_table_ids_to_remove(table); + } + } + + if (!new_producer_universe_id.empty()) { + req.set_new_producer_universe_id(new_producer_universe_id); + } + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->AlterUniverseReplication(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error altering universe replication: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + if (!add_tables.empty()) { + // If we are adding tables, then wait for the altered producer to be deleted (this happens + // once it is merged with the original). + RETURN_NOT_OK(WaitForSetupUniverseReplicationToFinish(producer_uuid + ".ALTER")); + } + + cout << "Replication altered successfully" << endl; + return Status::OK(); +} + +Status ClusterAdminClient::ChangeXClusterRole(cdc::XClusterRole role) { + master::ChangeXClusterRoleRequestPB req; + master::ChangeXClusterRoleResponsePB resp; + req.set_role(role); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->ChangeXClusterRole(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error changing role: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "Changed role successfully" << endl; + return Status::OK(); +} + +Status ClusterAdminClient::SetUniverseReplicationEnabled( + const std::string& producer_id, bool is_enabled) { + master::SetUniverseReplicationEnabledRequestPB req; + master::SetUniverseReplicationEnabledResponsePB resp; + req.set_producer_id(producer_id); + req.set_is_enabled(is_enabled); + const string toggle = (is_enabled ? "enabl" : "disabl"); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->SetUniverseReplicationEnabled(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error " << toggle << "ing " + << "universe replication: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "Replication " << toggle << "ed successfully" << endl; + return Status::OK(); +} + +Result ClusterAdminClient::GetFirstRpcAddressForTS() { + RepeatedPtrField servers; + RETURN_NOT_OK(ListTabletServers(&servers)); + for (const ListTabletServersResponsePB::Entry& server : servers) { + if (server.has_registration() && + !server.registration().common().private_rpc_addresses().empty()) { + return HostPortFromPB(server.registration().common().private_rpc_addresses(0)); + } + } + + return STATUS(NotFound, "Didn't find a server registered with the Master"); +} + +Status ClusterAdminClient::BootstrapProducer(const vector& table_ids) { + HostPort ts_addr = VERIFY_RESULT(GetFirstRpcAddressForTS()); + auto cdc_proxy = std::make_unique(proxy_cache_.get(), ts_addr); + + cdc::BootstrapProducerRequestPB bootstrap_req; + cdc::BootstrapProducerResponsePB bootstrap_resp; + for (const auto& table_id : table_ids) { + bootstrap_req.add_table_ids(table_id); + } + RpcController rpc; + rpc.set_timeout(MonoDelta::FromSeconds(std::max(timeout_.ToSeconds(), 120.0))); + RETURN_NOT_OK(cdc_proxy->BootstrapProducer(bootstrap_req, &bootstrap_resp, &rpc)); + + if (bootstrap_resp.has_error()) { + cout << "Error bootstrapping consumer: " << bootstrap_resp.error().status().message() + << endl; + return StatusFromPB(bootstrap_resp.error().status()); + } + + if (implicit_cast(bootstrap_resp.cdc_bootstrap_ids().size()) != table_ids.size()) { + cout << "Received invalid number of bootstrap ids: " << bootstrap_resp.ShortDebugString(); + return STATUS(InternalError, "Invalid number of bootstrap ids"); + } + + int i = 0; + for (const auto& bootstrap_id : bootstrap_resp.cdc_bootstrap_ids()) { + cout << "table id: " << table_ids[i++] << ", CDC bootstrap id: " << bootstrap_id << endl; + } + return Status::OK(); +} + +Status ClusterAdminClient::WaitForReplicationDrain( + const std::vector& stream_ids, const string& target_time) { + master::WaitForReplicationDrainRequestPB req; + master::WaitForReplicationDrainResponsePB resp; + for (const auto& stream_id : stream_ids) { + req.add_stream_ids(stream_id); + } + // If target_time is not provided, it will be set to current time in the master API. + if (!target_time.empty()) { + auto result = HybridTime::ParseHybridTime(target_time); + if (!result.ok()) { + return STATUS(InvalidArgument, "Error parsing target_time: " + result.ToString()); + } + req.set_target_time(result->GetPhysicalValueMicros()); + } + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->WaitForReplicationDrain(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error waiting for replication drain: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + std::unordered_map> undrained_streams; + for (const auto& stream_info : resp.undrained_stream_info()) { + undrained_streams[stream_info.stream_id()].push_back(stream_info.tablet_id()); + } + if (!undrained_streams.empty()) { + cout << "Found undrained replications:" << endl; + for (const auto& stream_to_tablets : undrained_streams) { + cout << "- Under Stream " << stream_to_tablets.first << ":" << endl; + for (const auto& tablet_id : stream_to_tablets.second) { + cout << " - Tablet: " << tablet_id << endl; + } + } + } else { + cout << "All replications are caught-up." << endl; + } + return Status::OK(); +} + +Status ClusterAdminClient::SetupNSUniverseReplication( + const std::string& producer_uuid, + const std::vector& producer_addresses, + const TypedNamespaceName& producer_namespace) { + switch (producer_namespace.db_type) { + case YQL_DATABASE_CQL: + break; + case YQL_DATABASE_PGSQL: + return STATUS( + InvalidArgument, "YSQL not currently supported for namespace-level replication setup"); + default: + return STATUS(InvalidArgument, "Unsupported namespace type"); + } + + master::SetupNSUniverseReplicationRequestPB req; + master::SetupNSUniverseReplicationResponsePB resp; + req.set_producer_id(producer_uuid); + req.set_producer_ns_name(producer_namespace.name); + req.set_producer_ns_type(producer_namespace.db_type); + + req.mutable_producer_master_addresses()->Reserve(narrow_cast(producer_addresses.size())); + for (const auto& addr : producer_addresses) { + auto hp = VERIFY_RESULT(HostPort::FromString(addr, master::kMasterDefaultPort)); + HostPortToPB(hp, req.add_producer_master_addresses()); + } + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->SetupNSUniverseReplication(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error setting up namespace-level universe replication: " + << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "Namespace-level replication setup successfully" << endl; + return Status::OK(); +} + +Status ClusterAdminClient::GetReplicationInfo(const std::string& universe_uuid) { + master::GetReplicationStatusRequestPB req; + master::GetReplicationStatusResponsePB resp; + + if (!universe_uuid.empty()) { + req.set_universe_id(universe_uuid); + } + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->GetReplicationStatus(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error getting replication status: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << resp.DebugString(); + return Status::OK(); +} + +Result ClusterAdminClient::GetXClusterEstimatedDataLoss() { + master::GetXClusterEstimatedDataLossRequestPB req; + master::GetXClusterEstimatedDataLossResponsePB resp; + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->GetXClusterEstimatedDataLoss(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error getting xCluster estimated data loss values: " + << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + rapidjson::Document document; + document.SetArray(); + for (const auto& data_loss : resp.namespace_data_loss()) { + rapidjson::Value json_entry(rapidjson::kObjectType); + AddStringField( + "namespace_id", data_loss.namespace_id(), &json_entry, &document.GetAllocator()); + + // Use 1 second granularity. + int64_t data_loss_s = MonoDelta::FromMicroseconds(data_loss.data_loss_us()).ToSeconds(); + AddStringField( + "data_loss_sec", std::to_string(data_loss_s), &json_entry, &document.GetAllocator()); + document.PushBack(json_entry, document.GetAllocator()); + } + + return document; +} + +Result ClusterAdminClient::GetXClusterSafeTime() { + master::GetXClusterSafeTimeRequestPB req; + master::GetXClusterSafeTimeResponsePB resp; + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->GetXClusterSafeTime(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error getting xCluster safe time values: " << resp.error().status().message() + << endl; + return StatusFromPB(resp.error().status()); + } + + rapidjson::Document document; + document.SetArray(); + for (const auto& safe_time : resp.namespace_safe_times()) { + rapidjson::Value json_entry(rapidjson::kObjectType); + AddStringField( + "namespace_id", safe_time.namespace_id(), &json_entry, &document.GetAllocator()); + const auto& st = HybridTime::FromPB(safe_time.safe_time_ht()); + AddStringField("safe_time", HybridTimeToString(st), &json_entry, &document.GetAllocator()); + AddStringField( + "safe_time_epoch", std::to_string(st.GetPhysicalValueMicros()), &json_entry, + &document.GetAllocator()); + document.PushBack(json_entry, document.GetAllocator()); + } + + return document; +} + string RightPadToUuidWidth(const string &s) { return RightPadToWidth(s, kNumCharactersInUuid); } diff --git a/src/yb/tools/yb-admin_client.h b/src/yb/tools/yb-admin_client.h index ee20762751ea..ee98fc954494 100644 --- a/src/yb/tools/yb-admin_client.h +++ b/src/yb/tools/yb-admin_client.h @@ -50,6 +50,7 @@ #include "yb/util/type_traits.h" #include "yb/common/entity_ids.h" #include "yb/consensus/consensus_types.pb.h" +#include "yb/common/snapshot.h" #include "yb/master/master_client.pb.h" #include "yb/master/master_cluster.pb.h" @@ -72,6 +73,14 @@ class YBClient; namespace tools { +// Flags for list_snapshot command. +YB_DEFINE_ENUM(ListSnapshotsFlag, (SHOW_DETAILS)(NOT_SHOW_RESTORED)(SHOW_DELETED)(JSON)); +using ListSnapshotsFlags = EnumBitSet; + +// Constants for disabling tablet splitting during PITR restores. +static constexpr double kPitrSplitDisableDurationSecs = 600; +static constexpr double kPitrSplitDisableCheckFreqMs = 500; + struct TypedNamespaceName { YQLDatabase db_type = YQL_DATABASE_UNKNOWN; std::string name; @@ -310,6 +319,113 @@ class ClusterAdminClient { Status ListAllNamespaces(); + // Snapshot operations. + Result ListSnapshots(const ListSnapshotsFlags& flags); + Status CreateSnapshot(const std::vector& tables, + const bool add_indexes = true, + const int flush_timeout_secs = 0); + Status CreateNamespaceSnapshot(const TypedNamespaceName& ns); + Result ListSnapshotRestorations( + const TxnSnapshotRestorationId& restoration_id); + Result CreateSnapshotSchedule(const client::YBTableName& keyspace, + MonoDelta interval, MonoDelta retention); + Result ListSnapshotSchedules(const SnapshotScheduleId& schedule_id); + Result DeleteSnapshotSchedule(const SnapshotScheduleId& schedule_id); + Result RestoreSnapshotSchedule( + const SnapshotScheduleId& schedule_id, HybridTime restore_at); + Status RestoreSnapshot(const std::string& snapshot_id, HybridTime timestamp); + + Result EditSnapshotSchedule( + const SnapshotScheduleId& schedule_id, + std::optional new_interval, + std::optional new_retention); + + Status DeleteSnapshot(const std::string& snapshot_id); + + Status CreateSnapshotMetaFile(const std::string& snapshot_id, + const std::string& file_name); + Status ImportSnapshotMetaFile(const std::string& file_name, + const TypedNamespaceName& keyspace, + const std::vector& tables); + Status ListReplicaTypeCounts(const client::YBTableName& table_name); + + Status SetPreferredZones(const std::vector& preferred_zones); + + Status RotateUniverseKey(const std::string& key_path); + + Status DisableEncryption(); + + Status IsEncryptionEnabled(); + + Status AddUniverseKeyToAllMasters( + const std::string& key_id, const std::string& universe_key); + + Status AllMastersHaveUniverseKeyInMemory(const std::string& key_id); + + Status RotateUniverseKeyInMemory(const std::string& key_id); + + Status DisableEncryptionInMemory(); + + Status WriteUniverseKeyToFile(const std::string& key_id, const std::string& file_name); + + Status CreateCDCStream(const TableId& table_id); + + Status CreateCDCSDKDBStream( + const TypedNamespaceName& ns, const std::string& CheckPointType, + const std::string& RecordType); + + Status DeleteCDCStream(const std::string& stream_id, bool force_delete = false); + + Status DeleteCDCSDKDBStream(const std::string& db_stream_id); + + Status ListCDCStreams(const TableId& table_id); + + Status ListCDCSDKStreams(const std::string& namespace_name); + + Status GetCDCDBStreamInfo(const std::string& db_stream_id); + + Status SetupUniverseReplication(const std::string& producer_uuid, + const std::vector& producer_addresses, + const std::vector& tables, + const std::vector& producer_bootstrap_ids); + + Status DeleteUniverseReplication(const std::string& producer_id, + bool ignore_errors = false); + + Status AlterUniverseReplication( + const std::string& producer_uuid, + const std::vector& producer_addresses, + const std::vector& add_tables, + const std::vector& remove_tables, + const std::vector& producer_bootstrap_ids_to_add, + const std::string& new_producer_universe_id, + bool remove_table_ignore_errors = false); + + Status RenameUniverseReplication(const std::string& old_universe_name, + const std::string& new_universe_name); + + Status WaitForSetupUniverseReplicationToFinish(const std::string& producer_uuid); + + Status ChangeXClusterRole(cdc::XClusterRole role); + + Status SetUniverseReplicationEnabled(const std::string& producer_id, + bool is_enabled); + + Status BootstrapProducer(const std::vector& table_id); + + Status WaitForReplicationDrain(const std::vector& stream_ids, + const std::string& target_time); + + Status SetupNSUniverseReplication(const std::string& producer_uuid, + const std::vector& producer_addresses, + const TypedNamespaceName& producer_namespace); + + Status GetReplicationInfo(const std::string& universe_uuid); + + Result GetXClusterEstimatedDataLoss(); + + Result GetXClusterSafeTime(); + protected: // Fetch the locations of the replicas for a given tablet from the Master. Status GetTabletLocations(const TabletId& tablet_id, @@ -436,10 +552,30 @@ class ClusterAdminClient { const Object& obj, const Request& req, const char* error_message = nullptr, const MonoDelta timeout = MonoDelta()); - private: using NamespaceMap = std::unordered_map; Result GetNamespaceMap(); + Result SuitableSnapshotId( + const SnapshotScheduleId& schedule_id, HybridTime restore_at, CoarseTimePoint deadline); + + Status SendEncryptionRequest(const std::string& key_path, bool enable_encryption); + + Result GetFirstRpcAddressForTS(); + + void CleanupEnvironmentOnSetupUniverseReplicationFailure( + const std::string& producer_uuid, const Status& failure_status); + + Status DisableTabletSplitsDuringRestore(CoarseTimePoint deadline); + + Result RestoreSnapshotScheduleDeprecated( + const SnapshotScheduleId& schedule_id, HybridTime restore_at); + + std::string GetDBTypeName(const master::SysNamespaceEntryPB& pb); + // Map: Old name -> New name. + typedef std::unordered_map NSNameToNameMap; + Status UpdateUDTypes( + QLTypePB* pb_type, bool* update_meta, const NSNameToNameMap& ns_name_to_name); + NamespaceMap namespace_map_; DISALLOW_COPY_AND_ASSIGN(ClusterAdminClient); From 60cc5524717e4247e6d1445d07b8fa14165b584c Mon Sep 17 00:00:00 2001 From: Dwight Hodge <79169168+ddhodge@users.noreply.github.com> Date: Thu, 2 Mar 2023 20:57:34 -0500 Subject: [PATCH 53/81] [docs] A page with many elements (#14039) Co-authored-by: Alex Ball --- .../contribute/docs/all-page-elements.md | 339 ++++++++++++++++++ .../preview/contribute/docs/include-file.md | 15 + .../contribute/docs/include-markdown.md | 15 + .../contribute/docs/widgets-and-shortcodes.md | 6 +- 4 files changed, 372 insertions(+), 3 deletions(-) create mode 100644 docs/content/preview/contribute/docs/all-page-elements.md create mode 100644 docs/content/preview/contribute/docs/include-file.md create mode 100644 docs/content/preview/contribute/docs/include-markdown.md diff --git a/docs/content/preview/contribute/docs/all-page-elements.md b/docs/content/preview/contribute/docs/all-page-elements.md new file mode 100644 index 000000000000..667747449c6f --- /dev/null +++ b/docs/content/preview/contribute/docs/all-page-elements.md @@ -0,0 +1,339 @@ +--- +title: YugabyteDB Managed quick start +headerTitle: Page with elements +linkTitle: Page with elements +headcontent: Just about every page element on a single page +description: YugabyteDB Documentation page elements on a single page. +layout: single +type: docs +menu: + preview: + identifier: all-page-elements + parent: docs-edit + weight: 9000 +--- + +This page demonstrates styles and widgets used for the YugabyteDB Documentation site. + +## Tab widgets + + + +This is a top-level tab widget, that uses different files for each tab. Everything that follows is in this tab's file. If you change tabs, you get a whole new page. + +### Second tab widget style + + + +This is a second-level tab widget, that uses different files for each tab - same as the one above, just styled differently. + +Everything that follows is in this tab's file. If you change tabs, you get a whole new page. + +### In-page tab widget + +This tab widget doesn't use separate files to fill in the content and then link between. Here the content is placed inside the `tabpane` shortcode. + +{{< tabpane text=true >}} + + {{% tab header="Tab" lang="java" %}} + +Another tab style; here all the tab contents are in-page. + +### Headings inside this widget don't show up in RightNav + +To build and run the application, do the following: + +1. Clone the sample application to your computer: + + ```sh + git clone https://github.com/YugabyteDB-Samples/yugabyte-simple-java-app.git && cd yugabyte-simple-java-app + ``` + +1. Provide connection parameters. + + The application needs to establish a connection to the YugabyteDB cluster. To do this: + + - Set the following configuration parameters: + + - **host** - the host name of your YugabyteDB cluster. To obtain a YugabyteDB Managed cluster host name, sign in to YugabyteDB Managed, select your cluster on the **Clusters** page, and click **Settings**. The host is displayed under **Connection Parameters**. + - **port** - the port number that will be used by the JDBC driver (the default YugabyteDB YSQL port is 5433). + + - Save the file. + +The end of this tab. + + {{% /tab %}} + + {{% tab header="Another tab" lang="go" %}} + +The contents of the next tab. You can keep adding tabs in similar fashion. + +### Headings inside this widget don't show up in RightNav + +To build and run the application, do the following: + +1. Clone the sample application to your computer: + + ```sh + git clone https://github.com/YugabyteDB-Samples/yugabyte-simple-go-app.git && cd yugabyte-simple-go-app + ``` + +1. Provide connection parameters. + + {{% /tab %}} + +{{< /tabpane >}} + +### Pills + +These pills automatically re-flow based on page width; just keep adding list items. Use these pills for landing pages or as an alternative to tabs. + + + +## Table + +The following is a basic markdown table. + +| Table | A column | +| :---- | :------- | +| A row | Another column in a table. Maybe to describe stuff. Might have bulleted lists etc, but that all has to be done using HTML. | +| Another row | Another row in a table. | +| Another row | Another column in a table. | + +## Notes and blockquote + +An ordinary paragraph. + +>**Blockquote** +> +>Blockquote text +> +>Some more blockquote text. + +{{< note title="Note" >}} + +This is an actual Note. Pay special attention to this thing, maybe. + +{{< /note >}} + +{{< tip title="Tip" >}} + +This is an actual Tip. Maybe there's a cool alternative to what you are doing. + +{{< /tip >}} + +{{< warning title="Warning" >}} + +This is an actual Warning. Whatever you are doing may threaten your data if you do it wrong. + +{{< /warning >}} + +## Images and video + +What follows is an image: + +![Connect using cloud shell](/images/yb-cloud/cloud-connect-shell.gif) + +What follows is an embedded YouTube video: + +{{< youtube id="qYMcNzWotkI" title="Deploy a fault tolerant cluster in YugabyteDB Managed" >}} + +## Lists + +What follows are some numbered steps, some code, text indented under a step: + +1. Numbered list with indented lists. + + 1. Second level numbered list indent. + + 1. Second level numbered list indent. + + 1. Third level numbered list indent. + + 1. Third level numbered list indent. + +1. Second list item. Some text in **bold**. Some text in _italics_. + +1. Third list item. + + Indented text under a list. + + ```output + Some code + ``` + +1. Fourth list item. + +Bulleted lists with levels of indent: + +- Bulleted list with indented lists. + + - Second level Bulleted list indent. + + - Second level Bulleted list indent. + + - Third level Bulleted list indent. + + - Third level Bulleted list indent. + +### Heading 3 + +#### Heading 4 + +Some bullets: + +- a cluster deployed in YugabyteDB Managed. +- the cluster CA certificate; YugabyteDB Managed uses TLS to secure connections to the database. +- your computer added to the cluster IP allow list. + +Refer to [Before you begin](../../../develop/build-apps/cloud-add-ip/). + +## Glossary entries + +Glossary term +: Definition. This text is a definition for a glossary term, or any sort of definition list. + +Glossary term +: Definition. This text is a definition for a glossary term, or any sort of definition list. +: Another paragraph in the definition. + +Glossary term +: Definition. This text is a definition for a glossary term, or any sort of definition list. +: Another paragraph in the definition. + +## Details tag + +The details HTML tag is used to create an interactive widget that the user can open and close. By default, the widget is closed. When open, it expands, and displays the contents. + +
    + Long code block + +These contents might be a very long bit of code or somesuch. + +
    + +## Syntax documentation + +```sh +ysqlsh [
    Start
    - {moment(alertDetails.createTime).toString()} + {ybFormatDate(alertDetails.createTime)}
    End
    - {alertDetails.resolvedTime ? moment(alertDetails.resolvedTime).toString() : '-'} + {alertDetails.resolvedTime ? ybFormatDate(alertDetails.resolvedTime) : '-'}
    @@ -162,13 +162,13 @@ export default class AlertDetails extends Component {
      {alertDetails.resolvedTime && (
    • -
      Alert resolved on {alertDetails.resolvedTime}
      +
      Alert resolved on {ybFormatDate(alertDetails.resolvedTime)}
      • )} {alertDetails.acknowledgedTime && (
    • - Alert acknowledged on {alertDetails.acknowledgedTime} + Alert acknowledged on {ybFormatDate(alertDetails.acknowledgedTime)}
      • )} @@ -176,7 +176,7 @@ export default class AlertDetails extends Component {
    • - Alert Triggered on {alertDetails.createTime} + Alert Triggered on {ybFormatDate(alertDetails.createTime)}
      • diff --git a/managed/ui/src/components/alerts/MaintenanceWindow/CreateMaintenanceWindow.tsx b/managed/ui/src/components/alerts/MaintenanceWindow/CreateMaintenanceWindow.tsx index d337b845a902..e2d3e0c38ba1 100644 --- a/managed/ui/src/components/alerts/MaintenanceWindow/CreateMaintenanceWindow.tsx +++ b/managed/ui/src/components/alerts/MaintenanceWindow/CreateMaintenanceWindow.tsx @@ -16,12 +16,12 @@ import { useMutation } from 'react-query'; import { convertUTCStringToDate, createMaintenanceWindow, - formatDateToUTC, MaintenanceWindowSchema, updateMaintenanceWindow } from '.'; import { toast } from 'react-toastify'; import { createErrorMessage } from '../../../utils/ObjectUtils'; +import { convertToISODateString, YBTimeFormats } from '../../../redesign/helpers/DateUtils'; // eslint-disable-next-line @typescript-eslint/no-var-requires const reactWidgets = require('react-widgets'); @@ -54,7 +54,7 @@ const initialValues = { selectedUniverse: [] }; -const DATE_FORMAT = 'YYYY-DD-MMMM'; +const DATE_FORMAT = YBTimeFormats.YB_DATE_ONLY_TIMESTAMP; const validationSchema = Yup.object().shape({ name: Yup.string().required('Enter name'), @@ -208,7 +208,7 @@ export const CreateMaintenanceWindow: FC = ({ formats={DATE_FORMAT} min={new Date()} onChange={(time: Date) => - setFieldValue('startTime' as never, formatDateToUTC(time), false) + setFieldValue('startTime' as never, convertToISODateString(time), false) } defaultValue={ values['startTime'] ? convertUTCStringToDate(values['startTime']) : null @@ -224,7 +224,7 @@ export const CreateMaintenanceWindow: FC = ({ step={10} min={moment(new Date()).add(1, 'm').toDate()} onChange={(time: Date) => - setFieldValue('endTime' as never, formatDateToUTC(time), false) + setFieldValue('endTime' as never, convertToISODateString(time), false) } defaultValue={values['endTime'] ? convertUTCStringToDate(values['endTime']) : null} /> diff --git a/managed/ui/src/components/alerts/MaintenanceWindow/MaintenanceWindowsList.tsx b/managed/ui/src/components/alerts/MaintenanceWindow/MaintenanceWindowsList.tsx index d4ffba499137..d99be679005b 100644 --- a/managed/ui/src/components/alerts/MaintenanceWindow/MaintenanceWindowsList.tsx +++ b/managed/ui/src/components/alerts/MaintenanceWindow/MaintenanceWindowsList.tsx @@ -6,12 +6,12 @@ import { convertUTCStringToMoment, deleteMaintenanceWindow, formatDateToUTC, - formatUTCStringToLocal, getMaintenanceWindowList, MaintenanceWindowSchema, MaintenanceWindowState, updateMaintenanceWindow } from '.'; +import { convertToISODateString, dateStrToMoment, ybFormatDate } from '../../../redesign/helpers/DateUtils'; import { YBButton, YBCheckBox } from '../../common/forms/fields'; import { YBLoading } from '../../common/indicators'; import { YBConfirmModal } from '../../modals'; @@ -77,13 +77,13 @@ const GetMaintenanceWindowActions = ({ const extendTime = useMutation( ({ window, minutesToExtend }: { window: MaintenanceWindowSchema; minutesToExtend: number }) => { - const currentEndTime = convertUTCStringToMoment(window.endTime).add( + const currentEndTime = dateStrToMoment(window.endTime).add( minutesToExtend, 'minute' ); return updateMaintenanceWindow({ ...window, - endTime: formatDateToUTC(currentEndTime.toDate()) + endTime: convertToISODateString(currentEndTime.toDate()) }); }, { @@ -266,7 +266,7 @@ export const MaintenanceWindowsList: FC = ({ className="no-border" dataAlign="left" width={'20%'} - dataFormat={(cell) => formatUTCStringToLocal(cell)} + dataFormat={(cell) => ybFormatDate(cell)} dataSort > Start Time @@ -277,7 +277,7 @@ export const MaintenanceWindowsList: FC = ({ className="no-border" dataAlign="left" width={'20%'} - dataFormat={(cell) => formatUTCStringToLocal(cell)} + dataFormat={(cell) => ybFormatDate(cell)} dataSort > End Time diff --git a/managed/ui/src/components/backupv2/common/BackupAPI.ts b/managed/ui/src/components/backupv2/common/BackupAPI.ts index 782275276c7c..a83315241898 100644 --- a/managed/ui/src/components/backupv2/common/BackupAPI.ts +++ b/managed/ui/src/components/backupv2/common/BackupAPI.ts @@ -11,7 +11,9 @@ import axios from 'axios'; import { Dictionary, groupBy } from 'lodash'; import { IBackup, Keyspace_Table, RESTORE_ACTION_TYPE, TIME_RANGE_STATE } from '..'; import { ROOT_URL } from '../../../config'; +import { convertToISODateString } from '../../../redesign/helpers/DateUtils'; import { MILLISECONDS_IN } from '../scheduled/ScheduledBackupUtils'; + import { BACKUP_API_TYPES, Backup_Options_Type, @@ -21,6 +23,7 @@ import { ThrottleParameters } from './IBackup'; + export function getBackupsList( page = 0, limit = 10, @@ -59,8 +62,8 @@ export function getBackupsList( payload.filter['states'] = [states[0].value]; } if (timeRange.startTime && timeRange.endTime) { - payload.filter['dateRangeStart'] = timeRange.startTime.toISOString(); - payload.filter['dateRangeEnd'] = timeRange.endTime.toISOString(); + payload.filter['dateRangeStart'] = convertToISODateString(timeRange.startTime); + payload.filter['dateRangeEnd'] = convertToISODateString(timeRange.endTime); } if (Array.isArray(moreFilters) && moreFilters?.length > 0) { diff --git a/managed/ui/src/components/backupv2/common/BackupUtils.tsx b/managed/ui/src/components/backupv2/common/BackupUtils.tsx index ac043f62f405..0ec9cb670eff 100644 --- a/managed/ui/src/components/backupv2/common/BackupUtils.tsx +++ b/managed/ui/src/components/backupv2/common/BackupUtils.tsx @@ -25,7 +25,7 @@ export const BACKUP_REFETCH_INTERVAL = 20 * 1000; * @param endtime end time * @returns diff between the dates */ -export const calculateDuration = (startTime: number, endtime: number): string => { +export const calculateDuration = (startTime: string, endtime: string): string => { const start = moment(startTime); const end = moment(endtime); diff --git a/managed/ui/src/components/backupv2/common/IBackup.ts b/managed/ui/src/components/backupv2/common/IBackup.ts index 321ee89a474c..46349bcc7c87 100644 --- a/managed/ui/src/components/backupv2/common/IBackup.ts +++ b/managed/ui/src/components/backupv2/common/IBackup.ts @@ -47,15 +47,15 @@ export interface Keyspace_Table { export interface ICommonBackupInfo { backupUUID: string; baseBackupUUID: string; - completionTime: number; - createTime: number; + completionTime: string; + createTime: string; responseList: Keyspace_Table[]; sse: boolean; state: Backup_States; storageConfigUUID: string; taskUUID: string; totalBackupSizeInBytes?: number; - updateTime: number; + updateTime: string; parallelism: number; } @@ -73,8 +73,8 @@ export interface IBackup { isStorageConfigPresent: boolean; isUniversePresent: boolean; onDemand: boolean; - updateTime: number; - expiryTime: number; + updateTime: string; + expiryTime: string; fullChainSizeInBytes: number; kmsConfigUUID?: null | string; } diff --git a/managed/ui/src/components/backupv2/common/RestoreAPI.ts b/managed/ui/src/components/backupv2/common/RestoreAPI.ts index 41d2c580fb34..d22d2188805d 100644 --- a/managed/ui/src/components/backupv2/common/RestoreAPI.ts +++ b/managed/ui/src/components/backupv2/common/RestoreAPI.ts @@ -9,6 +9,7 @@ import axios from 'axios'; import { ROOT_URL } from '../../../config'; +import { convertToISODateString } from '../../../redesign/helpers/DateUtils'; import { TIME_RANGE_STATE } from './IBackup'; export function getRestoreList( @@ -48,8 +49,8 @@ export function getRestoreList( payload.filter['states'] = [states[0].value]; } if (timeRange.startTime && timeRange.endTime) { - payload.filter['dateRangeStart'] = timeRange.startTime.toISOString(); - payload.filter['dateRangeEnd'] = timeRange.endTime.toISOString(); + payload.filter['dateRangeStart'] = convertToISODateString(timeRange.startTime); + payload.filter['dateRangeEnd'] = convertToISODateString(timeRange.endTime); } if (Array.isArray(moreFilters) && moreFilters?.length > 0) { diff --git a/managed/ui/src/components/backupv2/components/AssociatedBackups.tsx b/managed/ui/src/components/backupv2/components/AssociatedBackups.tsx index 65d99db73834..236e26129690 100644 --- a/managed/ui/src/components/backupv2/components/AssociatedBackups.tsx +++ b/managed/ui/src/components/backupv2/components/AssociatedBackups.tsx @@ -16,11 +16,12 @@ import { YBModal } from '../../common/forms/fields'; import { YBSearchInput } from '../../common/forms/fields/YBSearchInput'; import { YBLoading } from '../../common/indicators'; import { getBackupsList } from '../common/BackupAPI'; -import { ENTITY_NOT_AVAILABLE, FormatUnixTimeStampTimeToTimezone } from '../common/BackupUtils'; +import { ENTITY_NOT_AVAILABLE } from '../common/BackupUtils'; import { IBackup } from '../common/IBackup'; import { BackupDeleteModal } from './BackupDeleteModal'; import { BackupDetails } from './BackupDetails'; import './AssociatedBackups.scss'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; interface AssociatedBackupsProps { visible: boolean; @@ -139,11 +140,7 @@ export const AssociatedBackups: FC = ({ ( - - )} + dataFormat={(_, row: IBackup) => ybFormatDate(row.commonBackupInfo.createTime)} > Created At diff --git a/managed/ui/src/components/backupv2/components/BackupDeleteModal.tsx b/managed/ui/src/components/backupv2/components/BackupDeleteModal.tsx index f322ace1d356..bb7687a1127a 100644 --- a/managed/ui/src/components/backupv2/components/BackupDeleteModal.tsx +++ b/managed/ui/src/components/backupv2/components/BackupDeleteModal.tsx @@ -12,10 +12,10 @@ import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table'; import { useMutation, useQueryClient } from 'react-query'; import { toast } from 'react-toastify'; import { cancelBackup, deleteBackup, IBackup, Backup_States } from '..'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; import { StatusBadge } from '../../common/badge/StatusBadge'; import { YBModalForm } from '../../common/forms'; import { YBButton } from '../../common/forms/fields'; -import { FormatUnixTimeStampTimeToTimezone } from '../common/BackupUtils'; interface BackupDeleteProps { backupsList: IBackup[]; @@ -79,9 +79,7 @@ export const BackupDeleteModal: FC = ({ backupsList, visible, ( - - )} + dataFormat={(_, row: IBackup) => ybFormatDate(row.commonBackupInfo.createTime)} > Created At diff --git a/managed/ui/src/components/backupv2/components/BackupDetails.tsx b/managed/ui/src/components/backupv2/components/BackupDetails.tsx index d02c4121bf2f..70ec9b4ede6e 100644 --- a/managed/ui/src/components/backupv2/components/BackupDetails.tsx +++ b/managed/ui/src/components/backupv2/components/BackupDetails.tsx @@ -14,7 +14,6 @@ import { Backup_States, IBackup, Keyspace_Table } from '..'; import { StatusBadge } from '../../common/badge/StatusBadge'; import { YBButton } from '../../common/forms/fields'; import { - FormatUnixTimeStampTimeToTimezone, RevealBadge, calculateDuration } from '../common/BackupUtils'; @@ -35,6 +34,7 @@ import { YBTag } from '../../common/YBTag'; import { YBConfirmModal } from '../../modals'; import { toast } from 'react-toastify'; import { createErrorMessage } from '../../../utils/ObjectUtils'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; import './BackupDetails.scss'; interface BackupDetailsProps { @@ -221,15 +221,17 @@ export const BackupDetails: FC = ({
      Created At
      - + { + ybFormatDate(backupDetails.commonBackupInfo.createTime) + }
      Expiration
      - + { + ybFormatDate(backupDetails.expiryTime) + }
      diff --git a/managed/ui/src/components/backupv2/components/BackupList.tsx b/managed/ui/src/components/backupv2/components/BackupList.tsx index 1abcb6363394..b1aa8e94f90b 100644 --- a/managed/ui/src/components/backupv2/components/BackupList.tsx +++ b/managed/ui/src/components/backupv2/components/BackupList.tsx @@ -26,9 +26,7 @@ import { convertArrayToMap, DATE_FORMAT, ENTITY_NOT_AVAILABLE, - FormatUnixTimeStampTimeToTimezone } from '../common/BackupUtils'; -import './BackupList.scss'; import { BackupCancelModal, BackupDeleteModal } from './BackupDeleteModal'; import { BackupRestoreModal } from './BackupRestoreModal'; import { YBSearchInput } from '../../common/forms/fields/YBSearchInput'; @@ -42,6 +40,8 @@ import { YBTable } from '../../common/YBTable'; import { find } from 'lodash'; import { fetchTablesInUniverse } from '../../../actions/xClusterReplication'; import { TableTypeLabel } from '../../../redesign/helpers/dtos'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; +import './BackupList.scss'; // eslint-disable-next-line @typescript-eslint/no-var-requires const reactWidgets = require('react-widgets'); @@ -149,7 +149,7 @@ export const BackupList: FC = ({ allowTakingBackup, universeU return { label: action.label, - startTime: moment().subtract(action.value[0], action.value[1]), + startTime: moment().subtract(action.value[0], action.value[1]).toDate(), endTime: new Date() }; }; @@ -536,9 +536,7 @@ export const BackupList: FC = ({ allowTakingBackup, universeU ( - - )} + dataFormat={(_, row: IBackup) => ybFormatDate(row.commonBackupInfo.createTime)} width="20%" dataSort > @@ -547,7 +545,7 @@ export const BackupList: FC = ({ allowTakingBackup, universeU - time ? : "Won't Expire" + time ? ybFormatDate(time) : "Won't Expire" } width="20%" > diff --git a/managed/ui/src/components/backupv2/components/BackupRestoreModal.tsx b/managed/ui/src/components/backupv2/components/BackupRestoreModal.tsx index 261b219bb4fe..87d5c927fae2 100644 --- a/managed/ui/src/components/backupv2/components/BackupRestoreModal.tsx +++ b/managed/ui/src/components/backupv2/components/BackupRestoreModal.tsx @@ -30,7 +30,6 @@ import { } from '..'; import { YBModalForm } from '../../common/forms'; import { - FormatUnixTimeStampTimeToTimezone, KEYSPACE_VALIDATION_REGEX, SPINNER_ICON } from '../common/BackupUtils'; @@ -57,6 +56,7 @@ import { TableType } from '../../../redesign/helpers/dtos'; import clsx from 'clsx'; import { isYbcEnabledUniverse } from '../../../utils/UniverseUtils'; import { isDefinedNotNull } from '../../../utils/ObjectUtils'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; import './BackupRestoreModal.scss'; interface RestoreModalProps { @@ -123,9 +123,9 @@ export const BackupRestoreModal: FC = ({ const kmsConfigList = kmsConfigs ? kmsConfigs.map((config: any) => { - const labelName = config.metadata.provider + ' - ' + config.metadata.name; - return { value: config.metadata.configUUID, label: labelName }; - }) + const labelName = config.metadata.provider + ' - ' + config.metadata.name; + return { value: config.metadata.configUUID, label: labelName }; + }) : []; const restore = useMutation( @@ -178,7 +178,7 @@ export const BackupRestoreModal: FC = ({ ); const footerActions = [ - () => {}, + () => { }, () => { setCurrentStep(currentStep - 1); setOverrideSubmitLabel(TEXT_RENAME_DATABASE); @@ -308,11 +308,11 @@ export const BackupRestoreModal: FC = ({ keyspaces: currentStep === 1 ? Yup.array( - Yup.string().matches(KEYSPACE_VALIDATION_REGEX, { - message: 'Invalid keyspace name', - excludeEmptyString: true - }) - ) + Yup.string().matches(KEYSPACE_VALIDATION_REGEX, { + message: 'Invalid keyspace name', + excludeEmptyString: true + }) + ) : Yup.array(Yup.string()), parallelThreads: Yup.number() .min(1, 'Parallel threads should be greater than or equal to 1') @@ -448,9 +448,7 @@ function RestoreChooseUniverseForm({
    Created at
    - + {ybFormatDate(backup_details.commonBackupInfo.createTime)} @@ -553,9 +551,8 @@ function RestoreChooseUniverseForm({ ) => { @@ -631,8 +628,8 @@ export function RenameKeyspace({ values.backup.commonBackupInfo.responseList.map( (keyspace: Keyspace_Table, index: number) => values['searchText'] && - keyspace.keyspace && - !keyspace.keyspace.includes(values['searchText']) ? null : ( + keyspace.keyspace && + !keyspace.keyspace.includes(values['searchText']) ? null : ( // eslint-disable-next-line react/jsx-indent diff --git a/managed/ui/src/components/backupv2/components/BackupTableList.tsx b/managed/ui/src/components/backupv2/components/BackupTableList.tsx index fbec8066c608..3cf86622a05a 100644 --- a/managed/ui/src/components/backupv2/components/BackupTableList.tsx +++ b/managed/ui/src/components/backupv2/components/BackupTableList.tsx @@ -24,13 +24,13 @@ import { YBLoadingCircleIcon } from '../../common/indicators'; import { BACKUP_REFETCH_INTERVAL, calculateDuration, - FormatUnixTimeStampTimeToTimezone } from '../common/BackupUtils'; import { formatBytes } from '../../xcluster/ReplicationUtils'; import { StatusBadge } from '../../common/badge/StatusBadge'; import { TableType } from '../../../redesign/helpers/dtos'; import Timer from '../../universes/images/timer.svg'; import { createErrorMessage } from '../../../utils/ObjectUtils'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; import './BackupTableList.scss'; export enum BackupTypes { @@ -330,7 +330,7 @@ const IncrementalBackupCard = ({ }} > {isExpanded ? EXPANDED_ICON : COLLAPSED_ICON} - + {ybFormatDate(incrementalBackup.createTime)} {backup_type} {formatBytes(incrementalBackup.totalBackupSizeInBytes)} diff --git a/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryDisableModal.tsx b/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryDisableModal.tsx index fedd0c9b2b15..6f49c889a228 100644 --- a/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryDisableModal.tsx +++ b/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryDisableModal.tsx @@ -13,8 +13,8 @@ import { useMutation, useQueryClient } from 'react-query'; import { Col, Row } from 'react-bootstrap'; import { YBModalForm } from '../../common/forms'; import { YBButton } from '../../common/forms/fields'; -import { FormatUnixTimeStampTimeToTimezone } from './PointInTimeRecoveryList'; import { deletePITRConfig } from '../common/PitrAPI'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; import './PointInTimeRecoveryDisableModal.scss'; interface PointInTimeRecoveryDisableModalProps { @@ -102,7 +102,7 @@ export const PointInTimeRecoveryDisableModal: FC - + {ybFormatDate(minTime)} diff --git a/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryList.tsx b/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryList.tsx index 996fc145618d..25d4c3b5dfe8 100644 --- a/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryList.tsx +++ b/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryList.tsx @@ -25,6 +25,7 @@ import { PointInTimeRecoveryModal } from './PointInTimeRecoveryModal'; import { TableTypeLabel } from '../../../redesign/helpers/dtos'; import { PointInTimeRecoveryDisableModal } from './PointInTimeRecoveryDisableModal'; import './PointInTimeRecoveryList.scss'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; const DEFAULT_SORT_COLUMN = 'dbName'; const DEFAULT_SORT_DIRECTION = 'ASC'; @@ -174,7 +175,7 @@ export const PointInTimeRecoveryList = ({ universeUUID }: { universeUUID: string { - return minTime ? : ''; + return minTime ? ybFormatDate(minTime) : ''; }} dataSort > diff --git a/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryModal.tsx b/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryModal.tsx index ee228d461241..d0dffcc1eb5e 100644 --- a/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryModal.tsx +++ b/managed/ui/src/components/backupv2/pitr/PointInTimeRecoveryModal.tsx @@ -18,10 +18,10 @@ import { Col, Row } from 'react-bootstrap'; import { YBModalForm } from '../../common/forms'; import { YBFormSelect, YBNumericInput } from '../../common/forms/fields'; -import { FormatUnixTimeStampTimeToTimezone } from './PointInTimeRecoveryList'; import { restoreSnapShot } from '../common/PitrAPI'; import CautionIcon from '../common/CautionIcon'; import './PointInTimeRecoveryModal.scss'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; // eslint-disable-next-line @typescript-eslint/no-var-requires const reactWidgets = require('react-widgets'); @@ -208,11 +208,11 @@ export const PointInTimeRecoveryModal: FC = ({
    You may recover {config.dbName} to any time between{' '} - + {ybFormatDate(minTime)} {' '} and{' '} - + {ybFormatDate(maxTime)}
    @@ -268,12 +268,12 @@ export const PointInTimeRecoveryModal: FC = ({ ) : (
    Will recover to:{' '} - + })) + }
    )} diff --git a/managed/ui/src/components/backupv2/restore/Restore.tsx b/managed/ui/src/components/backupv2/restore/Restore.tsx index aeb6a5589f5f..7f2c165550f4 100644 --- a/managed/ui/src/components/backupv2/restore/Restore.tsx +++ b/managed/ui/src/components/backupv2/restore/Restore.tsx @@ -21,13 +21,13 @@ import { CALDENDAR_ICON, DATE_FORMAT, ENTITY_NOT_AVAILABLE, - FormatUnixTimeStampTimeToTimezone } from '../common/BackupUtils'; import { TIME_RANGE_STATE } from '../common/IBackup'; import { IRestore, RESTORE_STATUS_OPTIONS } from '../common/IRestore'; import { getRestoreList } from '../common/RestoreAPI'; import { DEFAULT_TIME_STATE, TIME_RANGE_OPTIONS } from '../components/BackupList'; import { RestoreEmpty } from './RestoreEmpty'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; import './Restore.scss'; const reactWidgets = require('react-widgets'); @@ -65,7 +65,7 @@ export const Restore: FC = ({ universeUUID, type }) => { return { label: action.label, - startTime: moment().subtract(action.value[0], action.value[1]), + startTime: moment().subtract(action.value[0], action.value[1]).toDate(), endTime: new Date() }; }; @@ -218,7 +218,7 @@ export const Restore: FC = ({ universeUUID, type }) => { } + dataFormat={(time) => ybFormatDate(time)} width="20%" dataSort > diff --git a/managed/ui/src/components/backupv2/scheduled/ScheduledBackupList.tsx b/managed/ui/src/components/backupv2/scheduled/ScheduledBackupList.tsx index 241a5fb50fc2..006b367fd996 100644 --- a/managed/ui/src/components/backupv2/scheduled/ScheduledBackupList.tsx +++ b/managed/ui/src/components/backupv2/scheduled/ScheduledBackupList.tsx @@ -31,9 +31,9 @@ import { convertScheduleToFormValues, convertMsecToTimeFrame } from './Scheduled import { useSelector } from 'react-redux'; import { Link } from 'react-router'; import { keyBy } from 'lodash'; -import { FormatUnixTimeStampTimeToTimezone } from '../common/BackupUtils'; import { ScheduledBackupEmpty } from '../components/BackupEmpty'; import { fetchTablesInUniverse } from '../../../actions/xClusterReplication'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; import './ScheduledBackupList.scss'; const wrapTableName = (tablesList: string[] | undefined) => { @@ -364,7 +364,7 @@ const ScheduledBackupCard: FC = ({
    Last backup
    {schedule.prevCompletedTask ? ( - + ybFormatDate(schedule.prevCompletedTask) ) : ( '-' )} @@ -374,7 +374,7 @@ const ScheduledBackupCard: FC = ({
    Next backup
    {schedule.nextExpectedTask ? ( - + ybFormatDate(schedule.nextExpectedTask) ) : ( '-' )} diff --git a/managed/ui/src/components/config/Security/ConfigDetails.js b/managed/ui/src/components/config/Security/ConfigDetails.js index 9112fef0322c..cd26759092df 100644 --- a/managed/ui/src/components/config/Security/ConfigDetails.js +++ b/managed/ui/src/components/config/Security/ConfigDetails.js @@ -1,8 +1,8 @@ import React from 'react'; -import moment from 'moment'; import { YBModal } from '../../common/forms/fields'; import { PROTECTION_LEVELS } from './KeyManagementConfiguration'; import { GCP_KMS_REGIONS_FLATTENED } from '../PublicCloud/views/providerRegionsData'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; export const ConfigDetails = ({ data, visible, onHide }) => { const { @@ -89,7 +89,7 @@ export const ConfigDetails = ({ data, visible, onHide }) => { label: 'Expiry', value: HC_VAULT_TTL && HC_VAULT_TTL_EXPIRY - ? moment(HC_VAULT_TTL_EXPIRY).format('DD MMMM YYYY') + ? ybFormatDate(HC_VAULT_TTL_EXPIRY) : 'Wont Expire' } ]; diff --git a/managed/ui/src/components/config/Security/certificates/CertificateDetails.js b/managed/ui/src/components/config/Security/certificates/CertificateDetails.js index 309ef1085eb4..fdb28b0dfd0f 100644 --- a/managed/ui/src/components/config/Security/certificates/CertificateDetails.js +++ b/managed/ui/src/components/config/Security/certificates/CertificateDetails.js @@ -1,15 +1,15 @@ // Copyright (c) YugaByte, Inc. import React, { Fragment } from 'react'; -import moment from 'moment'; import { YBModal } from '../../../common/forms/fields'; +import { ybFormatDate } from '../../../../redesign/helpers/DateUtils'; export const CertificateDetails = ({ certificate, visible, onHide }) => { const certStart = certificate.creationTime - ? moment(certificate.creationTime).format('DD MMMM YYYY') + ? ybFormatDate(certificate.creationTime) : ''; const certExpiry = certificate.expiryDate - ? moment(certificate.expiryDate).format('DD MMMM YYYY') + ? ybFormatDate(certificate.expiryDate) : ''; const isVaultCert = certificate.type === 'HashicorpVault'; @@ -70,7 +70,7 @@ export const CertificateDetails = ({ certificate, visible, onHide }) => { hcVaultCertParams: { vaultAddr, vaultToken, engine, mountPath, role, ttl, ttlExpiry } } = certificate; - const tokenExpiry = ttl && ttlExpiry ? moment(ttlExpiry).format('DD MMMM YYYY') : 'Wont Expire'; + const tokenExpiry = ttl && ttlExpiry ? ybFormatDate(ttlExpiry) : 'Wont Expire'; return ( <> diff --git a/managed/ui/src/components/config/Security/certificates/Certificates.js b/managed/ui/src/components/config/Security/certificates/Certificates.js index ebe82675622b..c848fc7fec62 100644 --- a/managed/ui/src/components/config/Security/certificates/Certificates.js +++ b/managed/ui/src/components/config/Security/certificates/Certificates.js @@ -7,16 +7,16 @@ import * as Yup from 'yup'; import { YBButton , YBFormInput } from '../../../common/forms/fields'; import { YBModalForm } from '../../../common/forms'; import { getPromiseState } from '../../../../utils/PromiseUtils'; -import moment from 'moment'; import { isNotHidden, isDisabled } from '../../../../utils/LayoutUtils'; -import './certificates.scss'; import { AddCertificateFormContainer } from './'; import { CertificateDetails } from './CertificateDetails'; import { api } from '../../../../redesign/helpers/api'; import { AssociatedUniverse } from '../../../common/associatedUniverse/AssociatedUniverse'; import { YBConfirmModal } from '../../../modals'; +import { ybFormatDate } from '../../../../redesign/helpers/DateUtils'; +import './certificates.scss'; const validationSchema = Yup.object().shape({ username: Yup.string().required('Enter username for certificate') @@ -101,7 +101,7 @@ class Certificates extends Component { }; getDateColumn = (key) => (item, row) => { if (key in row) { - return moment.utc(row[key], 'YYYY-MM-DD hh:mm:ss a').local().calendar(); + return ybFormatDate(row[key]); } else { return null; } @@ -173,8 +173,8 @@ class Certificates extends Component { const payload = { name: row.name, uuid: row.uuid, - creationTime: row.creationTime, - expiryDate: row.expiryDate, + creationTime: row.creationTimeIso, + expiryDate: row.expiryDateIso, universeDetails: row.universeDetails }; const disableCertEdit = row.type !== 'HashicorpVault'; @@ -274,9 +274,9 @@ class Certificates extends Component { type: cert.certType, uuid: cert.uuid, name: cert.label, - expiryDate: cert.expiryDate, + expiryDate: cert.expiryDateIso, certificate: cert.certificate, - creationTime: cert.startDate, + creationTime: cert.startDateIso, privateKey: cert.privateKey, customCertInfo: cert.customCertInfo, inUse: cert.inUse, diff --git a/managed/ui/src/components/metrics/MetricsComparisonModal/MetricsComparisonPanel.js b/managed/ui/src/components/metrics/MetricsComparisonModal/MetricsComparisonPanel.js index 053dac6d1599..f3c07e128676 100644 --- a/managed/ui/src/components/metrics/MetricsComparisonModal/MetricsComparisonPanel.js +++ b/managed/ui/src/components/metrics/MetricsComparisonModal/MetricsComparisonPanel.js @@ -12,6 +12,7 @@ import moment from 'moment'; import React from 'react'; import { METRIC_COLORS } from '../MetricsConfig'; import { useSelector } from 'react-redux'; +import { ybFormatDate, YBTimeFormats } from '../../../redesign/helpers/DateUtils'; export const MetricsComparisonPanel = ({ metricsData, metricsKey, metricsLayout, side }) => { const currentUser = useSelector((state) => state.customer.currentUser); @@ -70,7 +71,7 @@ export const MetricsComparisonPanel = ({ metricsData, metricsKey, metricsLayout, width={42} /> timeFormatter(value, 'HH:mm:ss MMM D, YYYY [UTC]ZZ')} + labelFormatter={(value) => ybFormatDate(value, YBTimeFormats.YB_HOURS_FIRST_TIMESTAMP)} /> {chartLines} diff --git a/managed/ui/src/components/panels/UniverseDisplayPanel/UniverseDisplayPanel.js b/managed/ui/src/components/panels/UniverseDisplayPanel/UniverseDisplayPanel.js index b31e3623fbb8..94faec48594f 100644 --- a/managed/ui/src/components/panels/UniverseDisplayPanel/UniverseDisplayPanel.js +++ b/managed/ui/src/components/panels/UniverseDisplayPanel/UniverseDisplayPanel.js @@ -18,7 +18,7 @@ import { getUniverseNodeCount } from '../../../utils/UniverseUtils'; import { isNotHidden, isDisabled } from '../../../utils/LayoutUtils'; -import { TimestampWithTimezone } from '../../common/timestampWithTimezone/TimestampWithTimezone'; +import { ybFormatDate, YBTimeFormats } from '../../../redesign/helpers/DateUtils'; import './UniverseDisplayPanel.scss'; @@ -74,7 +74,7 @@ class UniverseDisplayItem extends Component { ); } const universeCreationDate = universe.creationDate ? ( - + ybFormatDate(universe.creationDate, YBTimeFormats.YB_DATE_ONLY_TIMESTAMP) ) : ( '' ); diff --git a/managed/ui/src/components/universes/UniverseDetail/compounds/UniverseHealthCheckList.js b/managed/ui/src/components/universes/UniverseDetail/compounds/UniverseHealthCheckList.js index b5675747b4b6..d7a4d8a559ae 100644 --- a/managed/ui/src/components/universes/UniverseDetail/compounds/UniverseHealthCheckList.js +++ b/managed/ui/src/components/universes/UniverseDetail/compounds/UniverseHealthCheckList.js @@ -236,9 +236,9 @@ const detailsFormatter = (cell, row) => { // For performance optimization, move this to a Redux reducer, so that it doesn't get run on each render. function prepareData(data, timezone) { return data.map((timeData) => { - let timestampMoment = moment.utc(timeData.timestamp).local(); + let timestampMoment = moment.utc(timeData.timestamp_iso).local(); if (timezone) { - timestampMoment = moment.utc(timeData.timestamp).tz(timezone); + timestampMoment = moment.utc(timeData.timestamp_iso).tz(timezone); } const nodesByIpAddress = {}; timeData.data.forEach((check) => { diff --git a/managed/ui/src/components/universes/UniverseSupportBundle/SecondStep/SecondStep.js b/managed/ui/src/components/universes/UniverseSupportBundle/SecondStep/SecondStep.js index 94681d38ed23..aa3ce6ed51fc 100644 --- a/managed/ui/src/components/universes/UniverseSupportBundle/SecondStep/SecondStep.js +++ b/managed/ui/src/components/universes/UniverseSupportBundle/SecondStep/SecondStep.js @@ -1,10 +1,10 @@ import React, { useRef, useState } from 'react'; import { YBCheckBox } from '../../../common/forms/fields'; import { DropdownButton, MenuItem } from 'react-bootstrap'; -import moment from 'moment'; import { CustomDateRangePicker } from '../DateRangePicker/DateRangePicker'; import { useSelector } from 'react-redux'; import { find } from 'lodash'; +import { convertToISODateString } from '../../../../redesign/helpers/DateUtils'; const filterTypes = [ { label: 'Last 24 hrs', type: 'days', value: '1' }, @@ -36,8 +36,8 @@ export const updateOptions = ( dateType, selectionOptionsValue, setIsDateTypeCustom, - startDate = new moment(new Date()), - endDate = new moment(new Date()) + startDate = new Date(), + endDate = new Date() ) => { if (dateType === 'custom') { setIsDateTypeCustom(true); @@ -45,7 +45,7 @@ export const updateOptions = ( } if (dateType !== 'customWithValue' && dateType !== 'custom') { - startDate = new moment(getBackDateByDay(+dateType)); + startDate = getBackDateByDay(+dateType); setIsDateTypeCustom(false); } @@ -56,8 +56,8 @@ export const updateOptions = ( } }); return { - startDate: startDate.format('yyyy-MM-DD'), - endDate: endDate.format('yyyy-MM-DD'), + startDate: convertToISODateString(startDate), + endDate: convertToISODateString(endDate), components: components }; }; @@ -116,8 +116,8 @@ export const SecondStep = ({ onOptionsChange, isK8sUniverse }) => { 'customWithValue', selectionOptionsValue, setIsDateTypeCustom, - new moment(startEnd.start), - new moment(startEnd.end) + startEnd.start, + startEnd.end ); onOptionsChange(changedOptions); }} diff --git a/managed/ui/src/components/universes/UniverseSupportBundle/ThirdStep/ThirdStep.js b/managed/ui/src/components/universes/UniverseSupportBundle/ThirdStep/ThirdStep.js index 4667238e96a6..88bea6c0542d 100644 --- a/managed/ui/src/components/universes/UniverseSupportBundle/ThirdStep/ThirdStep.js +++ b/managed/ui/src/components/universes/UniverseSupportBundle/ThirdStep/ThirdStep.js @@ -7,6 +7,7 @@ import {DropdownButton, MenuItem} from "react-bootstrap"; import {YBMenuItem} from "../../UniverseDetail/compounds/YBMenuItem"; import {YBButton, YBModal} from "../../../common/forms/fields"; +import { ybFormatDate, YBTimeFormats } from "../../../../redesign/helpers/DateUtils"; import {formatBytes} from "../../../xcluster/ReplicationUtils"; import "./ThirdStep.scss"; @@ -114,7 +115,7 @@ export const ThirdStep = withRouter(({ closeModal={() => { setIsConfirmDeleteOpen(false); }} - createdOn={deleteBundleObj.creationDate} + createdOn={ybFormatDate(deleteBundleObj.creationDate)} confirmDelete={() => { handleDeleteBundle(deleteBundleObj.bundleUUID); setDeleteBundleObj({}); @@ -152,6 +153,7 @@ export const ThirdStep = withRouter(({ ybFormatDate(creationDate, YBTimeFormats.YB_DATE_ONLY_TIMESTAMP)} isKey={true} className={'node-name-field'} columnClassName={'node-name-field'} @@ -160,6 +162,7 @@ export const ThirdStep = withRouter(({ ybFormatDate(expirationDate, YBTimeFormats.YB_DATE_ONLY_TIMESTAMP)} className={'yb-node-status-cell'} columnClassName={'yb-node-status-cell'} > diff --git a/managed/ui/src/components/xcluster/XClusterConfigCard.tsx b/managed/ui/src/components/xcluster/XClusterConfigCard.tsx index 8e5ad483c4f8..ec5402380afc 100644 --- a/managed/ui/src/components/xcluster/XClusterConfigCard.tsx +++ b/managed/ui/src/components/xcluster/XClusterConfigCard.tsx @@ -6,7 +6,6 @@ import clsx from 'clsx'; import { fetchUniversesList } from '../../actions/xClusterReplication'; import { - convertToLocalTime, findUniverseName, MaxAcceptableLag, CurrentReplicationLag @@ -16,6 +15,7 @@ import { XClusterConfig } from './XClusterTypes'; import RightArrow from './ArrowIcon'; import { ReplicationParticipantCard } from './ReplicationParticipantCard'; import { XClusterConfigStatus } from './constants'; +import { ybFormatDate } from '../../redesign/helpers/DateUtils'; import styles from './XClusterConfigCard.module.scss'; @@ -51,11 +51,11 @@ export const XClusterConfigCard = ({
    Started
    -
    {convertToLocalTime(xClusterConfig.createTime, currentUserTimezone)}
    +
    {ybFormatDate(xClusterConfig.createTime)}
    Last modified
    -
    {convertToLocalTime(xClusterConfig.modifyTime, currentUserTimezone)}
    +
    {ybFormatDate(xClusterConfig.modifyTime)}
    diff --git a/managed/ui/src/components/xcluster/configDetails/ReplicationOverview.tsx b/managed/ui/src/components/xcluster/configDetails/ReplicationOverview.tsx index 8ab95e04b8f1..d9ee845ee117 100644 --- a/managed/ui/src/components/xcluster/configDetails/ReplicationOverview.tsx +++ b/managed/ui/src/components/xcluster/configDetails/ReplicationOverview.tsx @@ -1,9 +1,9 @@ import React from 'react'; import { Col, Row } from 'react-bootstrap'; -import { useSelector } from 'react-redux'; import { Link } from 'react-router'; import { XClusterConfig } from '../XClusterTypes'; -import { convertToLocalTime, getMasterNodeAddress } from '../ReplicationUtils'; +import { getMasterNodeAddress } from '../ReplicationUtils'; +import { ybFormatDate } from '../../../redesign/helpers/DateUtils'; export function ReplicationOverview({ replication, @@ -15,7 +15,6 @@ export function ReplicationOverview({ const { universeDetails: { nodeDetailsSet } } = destinationUniverse; - const currentUserTimezone = useSelector((state: any) => state.customer.currentUser.data.timezone); return ( <> @@ -25,14 +24,14 @@ export function ReplicationOverview({
    Replication started - {convertToLocalTime(replication.createTime, currentUserTimezone)} + {ybFormatDate(replication.createTime)}
    Replication last modified - {convertToLocalTime(replication.modifyTime, currentUserTimezone)} + {ybFormatDate(replication.modifyTime)} diff --git a/managed/ui/src/redesign/helpers/DateUtils.ts b/managed/ui/src/redesign/helpers/DateUtils.ts new file mode 100644 index 000000000000..822148faa545 --- /dev/null +++ b/managed/ui/src/redesign/helpers/DateUtils.ts @@ -0,0 +1,46 @@ +/* + * Created on Fri Feb 03 2023 + * + * Copyright 2021 YugaByte, Inc. and Contributors + * Licensed under the Polyform Free Trial License 1.0.0 (the "License") + * You may not use this file except in compliance with the License. You may obtain a copy of the License at + * http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt + */ + +import moment from "moment" + +export const YBTimeFormats = { + YB_DEFAULT_TIMESTAMP: 'MMM-DD-YYYY hh:mm:ss ZZ', + YB_DATE_ONLY_TIMESTAMP: 'MMM-DD-YYYY', + YB_HOURS_FIRST_TIMESTAMP: 'hh:mm:ss MMM-DD-YYYY [UTC]ZZ', + YB_ISO8601_TIMESTAMP: 'YYYY-MM-DD[T]H:mm:ssZZ' +} as const; + +/** + * Converts date to RFC3339 format("yyyy-MM-dd'T'HH:mm:ss'Z'") + * @param d Date + * @returns RFC3339 format string + */ +export const convertToISODateString = (d: Date) => { + const pad = (n: number) => { return n < 10 ? '0' + n : n } + try { + return d.getUTCFullYear() + '-' + + pad(d.getUTCMonth() + 1) + '-' + + pad(d.getUTCDate()) + 'T' + + pad(d.getUTCHours()) + ':' + + pad(d.getUTCMinutes()) + ':' + + pad(d.getUTCSeconds()) + 'Z' + } + catch (e) { + console.error(e); + return '-'; + } +} + +export const ybFormatDate = (d: Date | string | number, timeFormat = YBTimeFormats.YB_DEFAULT_TIMESTAMP) => { + return moment(d).format(timeFormat as any); +} + +export const dateStrToMoment = (str: string) => { + return moment(str); +} diff --git a/managed/ui/src/utils/TableFormatters.js b/managed/ui/src/utils/TableFormatters.js index 758aec5fc6b2..19c86c22f90d 100644 --- a/managed/ui/src/utils/TableFormatters.js +++ b/managed/ui/src/utils/TableFormatters.js @@ -1,27 +1,23 @@ // Copyright (c) YugaByte, Inc. import React from 'react'; -import moment from 'moment'; import { isValidObject } from './ObjectUtils'; import { YBFormattedNumber } from '../components/common/descriptors'; import { YBLoadingCircleIcon } from '../components/common/indicators'; -import { TimestampWithTimezone } from '../components/common/timestampWithTimezone/TimestampWithTimezone'; +import { ybFormatDate, YBTimeFormats } from '../redesign/helpers/DateUtils'; export function timeFormatter(cell) { if (!isValidObject(cell)) { return -; } else { - return ; + return ybFormatDate(cell); } } -export function timeFormatterISO8601(cell, _, timezone) { +export function timeFormatterISO8601(cell) { if (!isValidObject(cell)) { return '-'; } else { - if (timezone) { - return moment(cell).tz(timezone).format('YYYY-MM-DD[T]H:mm:ssZZ'); - } - return moment(cell).format('YYYY-MM-DD[T]H:mm:ssZZ'); + return ybFormatDate(cell, YBTimeFormats.YB_ISO8601_TIMESTAMP); } } From 90be6620dc35c1fda9a3ad55a4145a97d544e009 Mon Sep 17 00:00:00 2001 From: Lingeshwar S Date: Sat, 4 Mar 2023 15:28:11 +0530 Subject: [PATCH 75/81] [PLAT-6609][PLAT-7618][PLAT-7633] - fix : Error handling, alignment issues , code cleanup and RR does not inherit tags from primary Summary: **[PLAT-6609][PLAT-7633] - Error handling, code clean up and alignment issues** 1. Added page not found if url id improper 2. Shows error and restrict form submission if universe_configure api fails 3. Added data-testid as per conventions. 4. Fixed alignment issues across the form. 5. Removed wrong usage of Grid in code. 6. Fixed unhandled guard conditions 7. Updated close button used in modal as per old ui modal. 8. Prefetch runtime configs to avoid flickering/movements in form. 9. Added placeholder text for required ui elements. 10. Added proper padding, margin and spacing as per figma. 11. Handled responsiveness till medium device. **[PLAT-7618] - Read replica does not inherit tags from primary in one of the flow** 1. Handled RR does not inherit flags from primary in Create + RR flow 2. Fixed showing empty textbox iin RR if user tags does not exist Test Plan: Tested Manually with below flows 1. Create Primary 2. Create Primary + RR 3. Edit Primary 4. Edit RR 3. Add RR **Screenshots** {F35100} {F35101} {F35104} {F35105} Reviewers: kkannan, asathyan Reviewed By: asathyan Subscribers: jenkins-bot, ui, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23331 --- .../common/indicators/YBErrorIndicator.js | 7 +- .../redesign/components/YBModal/YBModal.tsx | 5 +- .../redesign/components/YBSelect/YBSelect.tsx | 8 +- .../universe/universe-form/CreateUniverse.tsx | 4 +- .../universe-form/UniverseFormContainer.tsx | 34 +++++- .../universe-form/form/UniverseForm.tsx | 33 ++++-- .../form/fields/ARNField/ARNField.tsx | 2 +- .../AccessKeysField/AccessKeysField.tsx | 4 +- .../fields/DBVersionField/DBVersionField.tsx | 2 +- .../PlacementsField/PlacementsField.tsx | 51 +++++---- .../PlacementsField/PlacementsFieldHelper.ts | 24 +++- .../fields/ProvidersField/ProvidersField.tsx | 3 +- .../form/fields/RegionsField/RegionsField.tsx | 5 +- .../ReplicationFactorField.tsx | 28 +++-- .../TotalNodesField/TotalNodesField.tsx | 26 +++-- .../UniverseNameField/UniverseNameField.tsx | 1 + .../fields/UserTagsField/UserTagsField.tsx | 104 +++++++++-------- .../form/fields/YCQLField/YCQLField.tsx | 4 + .../form/fields/YSQLField/YSQLField.tsx | 4 + .../advanced/AdvancedConfiguration.tsx | 107 ++++++------------ .../sections/cloud/CloudConfiguration.tsx | 104 ++++++++--------- .../form/sections/gflags/Gflags.tsx | 8 +- .../sections/helmoverrides/HelmOverrides.tsx | 2 +- .../instance/InstanceConfiguration.tsx | 10 +- .../security/SecurityConfiguration.tsx | 54 +++------ .../form/sections/tags/UserTags.tsx | 32 ++++-- .../universe-form/universeMainStyle.ts | 7 +- .../universe/universe-form/utils/dto.ts | 4 +- managed/ui/src/translations/en.json | 23 +++- 29 files changed, 368 insertions(+), 332 deletions(-) diff --git a/managed/ui/src/components/common/indicators/YBErrorIndicator.js b/managed/ui/src/components/common/indicators/YBErrorIndicator.js index d03b7f792178..10de0dc97058 100644 --- a/managed/ui/src/components/common/indicators/YBErrorIndicator.js +++ b/managed/ui/src/components/common/indicators/YBErrorIndicator.js @@ -7,7 +7,7 @@ import { Link } from 'react-router'; export default class YBErrorIndicator extends Component { render() { - const { type, uuid, customErrorMessage } = this.props; + const { type, uuid, customErrorMessage, showRecoveryMsg } = this.props; let errorDisplayMessage = ; let errorRecoveryMessage = ; @@ -15,6 +15,9 @@ export default class YBErrorIndicator extends Component { errorDisplayMessage = (
    Seems like universe with UUID {uuid} has issues or does not exist.
    ); + } + + if (type === 'universe' || showRecoveryMsg) { errorRecoveryMessage = (
    Click here to go back to home page. @@ -27,8 +30,8 @@ export default class YBErrorIndicator extends Component { Aww Snap. sad face
    {errorDisplayMessage}
    + {customErrorMessage &&
    {customErrorMessage}
    }
    {errorRecoveryMessage}
    - {customErrorMessage && (
    {customErrorMessage}
    )} ); } diff --git a/managed/ui/src/redesign/components/YBModal/YBModal.tsx b/managed/ui/src/redesign/components/YBModal/YBModal.tsx index 360b500b5e40..9550d219d24a 100644 --- a/managed/ui/src/redesign/components/YBModal/YBModal.tsx +++ b/managed/ui/src/redesign/components/YBModal/YBModal.tsx @@ -14,7 +14,6 @@ import { } from '@material-ui/core'; import type { TransitionProps } from '@material-ui/core/transitions'; import { YBTooltip, YBButton, YBButtonProps } from '../../components/'; -import { isSubmitting } from 'redux-form'; export interface OverrideButtonProps { primary?: YBButtonProps; @@ -102,7 +101,7 @@ const useStyles = makeStyles>((theme) => ({ }, closeBtnText: { color: theme.palette.orange[500], - fontSize: theme.spacing(4), + fontSize: theme.spacing(3), lineHeight: '26px' }, actionsInfo: { @@ -274,7 +273,7 @@ export const YBModal: FC = (props: YBModalProps) => { )} {!hideCloseBtn && ( - x + )} diff --git a/managed/ui/src/redesign/components/YBSelect/YBSelect.tsx b/managed/ui/src/redesign/components/YBSelect/YBSelect.tsx index 286f2d1a4e69..97c9493a915a 100644 --- a/managed/ui/src/redesign/components/YBSelect/YBSelect.tsx +++ b/managed/ui/src/redesign/components/YBSelect/YBSelect.tsx @@ -1,6 +1,5 @@ import React, { FC, ReactNode } from 'react'; import { TextField, StandardTextFieldProps } from '@material-ui/core'; -import { YBTooltip } from '../YBTooltip/YBTooltip'; export type YBSelectProps = { tooltip?: ReactNode; @@ -17,14 +16,9 @@ export type YBSelectProps = { | 'SelectProps' >; -export const YBSelect: FC = ({ label, tooltip, renderValue, ...props }) => ( +export const YBSelect: FC = ({ renderValue, ...props }) => ( - {label} {tooltip && } - - } select SelectProps={{ IconComponent: undefined, diff --git a/managed/ui/src/redesign/features/universe/universe-form/CreateUniverse.tsx b/managed/ui/src/redesign/features/universe/universe-form/CreateUniverse.tsx index ac0a39f11c56..996619e3d9dd 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/CreateUniverse.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/CreateUniverse.tsx @@ -123,7 +123,9 @@ export const CreateUniverse: FC = () => { if (asyncData) { configurePayload.clusters?.push({ clusterType: ClusterType.ASYNC, - userIntent: getUserIntent({ formData: asyncData }), + userIntent: getUserIntent({ + formData: { ...asyncData, instanceTags: primaryData.instanceTags } //copy primary user tags to RR while creation + }), placementInfo: { cloudList: [ { diff --git a/managed/ui/src/redesign/features/universe/universe-form/UniverseFormContainer.tsx b/managed/ui/src/redesign/features/universe/universe-form/UniverseFormContainer.tsx index e92c823eac04..68c83413e534 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/UniverseFormContainer.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/UniverseFormContainer.tsx @@ -1,9 +1,11 @@ import React, { createContext, FC } from 'react'; import { useQuery } from 'react-query'; import { useMethods } from 'react-use'; +import { useSelector } from 'react-redux'; +import { useTranslation } from 'react-i18next'; import { RouteComponentProps } from 'react-router-dom'; import { Box } from '@material-ui/core'; -import { YBLoading } from '../../../../components/common/indicators'; +import { YBErrorIndicator, YBLoading } from '../../../../components/common/indicators'; import { CreateReadReplica } from './CreateRR'; import { CreateUniverse } from './CreateUniverse'; import { EditReadReplica } from './EditRR'; @@ -27,6 +29,7 @@ export interface UniverseFormContextState { isLoading: boolean; // To safeguard against bad defaults newUniverse: boolean; // Fresh Universe ( set to true only in Primary + RR flow ) universeResourceTemplate: UniverseResource | null; + universeConfigureError: string | null; } const initialState: UniverseFormContextState = { @@ -37,7 +40,8 @@ const initialState: UniverseFormContextState = { mode: ClusterModes.CREATE, isLoading: true, newUniverse: false, - universeResourceTemplate: null + universeResourceTemplate: null, + universeConfigureError: null }; //Avoiding using global state since we are using react-query @@ -61,6 +65,10 @@ const createFormMethods = (contextState: UniverseFormContextState) => ({ ...contextState, asyncFormData: data }), + setConfigureError: (data: string | null): UniverseFormContextState => ({ + ...contextState, + universeConfigureError: data + }), toggleClusterType: (type: ClusterType): UniverseFormContextState => ({ ...contextState, clusterType: type @@ -93,6 +101,9 @@ export const UniverseFormContainer: FC state.customer.currentCustomer); + const customerUUID = currentCustomer?.data?.uuid; //route has it in lower case & enum has it in upper case const mode = MODE?.toUpperCase(); @@ -104,11 +115,16 @@ export const UniverseFormContainer: FC + //Prefetch Global and Customer scope runtime configs + const { isLoading: isGlobalConfigsLoading } = useQuery(QUERY_KEY.fetchGlobalRunTimeConfigs, () => api.fetchRunTimeConfigs(true) ); + const { isLoading: isCustomerConfigsLoading } = useQuery( + [QUERY_KEY.fetchCustomerRunTimeConfigs, customerUUID], + () => api.fetchRunTimeConfigs(true, customerUUID) + ); + const switchInternalRoutes = () => { //Create Primary + RR if (location.pathname === '/universes/create') return ; @@ -122,10 +138,16 @@ export const UniverseFormContainer: FC; //Page not found - else return
    Page not found
    ; + else + return ( + + ); }; - if (isProviderLoading || isRuntimeConfigsLoading) return ; + if (isProviderLoading || isGlobalConfigsLoading || isCustomerConfigsLoading) return ; else return ( diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/UniverseForm.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/UniverseForm.tsx index 676b4cd19bb0..87e415f08048 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/UniverseForm.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/UniverseForm.tsx @@ -1,8 +1,10 @@ import React, { useContext, FC } from 'react'; +import _ from 'lodash'; import { useTranslation } from 'react-i18next'; import { useForm, FormProvider } from 'react-hook-form'; import { useQuery } from 'react-query'; import { useSelector } from 'react-redux'; +import { toast } from 'react-toastify'; import { Typography, Grid, Box, Link } from '@material-ui/core'; import { YBButton } from '../../../../components'; import { @@ -18,7 +20,7 @@ import { import { UniverseFormContext } from '../UniverseFormContainer'; import { api, QUERY_KEY } from '../utils/api'; import { UniverseFormData, ClusterType, ClusterModes } from '../utils/dto'; -import { UNIVERSE_NAME_FIELD } from '../utils/constants'; +import { UNIVERSE_NAME_FIELD, TOAST_AUTO_DISMISS_INTERVAL } from '../utils/constants'; import { useFormMainStyles } from '../universeMainStyle'; // ! How to add new form field ? @@ -57,18 +59,23 @@ export const UniverseForm: FC = ({ const { t } = useTranslation(); //context state - const { asyncFormData, clusterType, mode, universeResourceTemplate } = useContext( - UniverseFormContext - )[0]; + const { + asyncFormData, + clusterType, + mode, + universeResourceTemplate, + universeConfigureError + } = useContext(UniverseFormContext)[0]; const isPrimary = clusterType === ClusterType.PRIMARY; const isEditMode = mode === ClusterModes.EDIT; const isEditRR = isEditMode && !isPrimary; - // Get customer scope runtime configs + // Fetch customer scope runtime configs const currentCustomer = useSelector((state: any) => state.customer.currentCustomer); const customerUUID = currentCustomer?.data?.uuid; - const { data: runtimeConfigs } = useQuery(QUERY_KEY.fetchCustomerRunTimeConfigs, () => - api.fetchRunTimeConfigs(true, customerUUID) + const { data: runtimeConfigs } = useQuery( + [QUERY_KEY.fetchCustomerRunTimeConfigs, customerUUID], + () => api.fetchRunTimeConfigs(true, customerUUID) ); //init form @@ -82,13 +89,21 @@ export const UniverseForm: FC = ({ //methods const triggerValidation = () => trigger(undefined, { shouldFocus: true }); //Trigger validation and focus on fields with errors , undefined = validate all fields - const onSubmit = (formData: UniverseFormData) => onFormSubmit(formData); + + const onSubmit = (formData: UniverseFormData) => { + if (!_.isEmpty(universeConfigureError)) + // Do not allow for form submission incase error exists in universe configure response + toast.error(universeConfigureError, { autoClose: TOAST_AUTO_DISMISS_INTERVAL }); + else onFormSubmit(formData); + }; const switchClusterType = () => onClusterTypeChange && onClusterTypeChange(getValues()); //switching from primary to RR and vice versa (Create Primary + RR flow) const handleClusterChange = async () => { if (isPrimary) { // Validate primary form before switching to async + if (!_.isEmpty(universeConfigureError)) + toast.error(universeConfigureError, { autoClose: TOAST_AUTO_DISMISS_INTERVAL }); let isValid = await triggerValidation(); isValid && switchClusterType(); } else { @@ -237,7 +252,7 @@ export const UniverseForm: FC = ({ // const isPrimary = [clusterModes.NEW_PRIMARY, clusterModes.EDIT_PRIMARY].includes(mode); return ( - +
    {renderHeader()} diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/ARNField/ARNField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/ARNField/ARNField.tsx index 2ad2aa872508..d95d0fa71f85 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/ARNField/ARNField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/ARNField/ARNField.tsx @@ -21,7 +21,7 @@ export const ARNField = ({ disabled }: ARNFieldProps): ReactElement => { {t('universeForm.advancedConfig.instanceProfileARN')} - + {t('universeForm.advancedConfig.accessKey')} - + {t('universeForm.advancedConfig.dbVersion')} - + ({ '& .MuiFormControl-root': { flexDirection: 'inherit' } + }, + nameColumn: { + width: 400, + display: 'flex', + alignItems: 'center', + justifyContent: 'flex-start' + }, + nodesColumn: { + width: 100, + display: 'flex', + alignItems: 'center', + justifyContent: 'flex-start', + margin: theme.spacing(0, 1) + }, + preferredColumn: { + width: 100, + display: 'flex', + alignItems: 'center', + justifyContent: 'flex-start' } })); @@ -58,19 +77,13 @@ export const PlacementsField = ({ disabled, isPrimary }: PlacementsFieldProps): const isDedicatedNodes = masterPlacement === MasterPlacementMode.DEDICATED; const renderHeader = ( - - + + {t('universeForm.cloudConfig.azNameLabel')} - + {provider?.code === CloudType.kubernetes ? t('universeForm.cloudConfig.azPodsLabel') @@ -80,7 +93,7 @@ export const PlacementsField = ({ disabled, isPrimary }: PlacementsFieldProps): {isPrimary && ( - + {t('universeForm.cloudConfig.preferredAZLabel')} @@ -111,12 +124,7 @@ export const PlacementsField = ({ disabled, isPrimary }: PlacementsFieldProps): return ( - + - + {isPrimary && ( - + { @@ -188,9 +196,9 @@ export const PlacementsField = ({ disabled, isPrimary }: PlacementsFieldProps): flexDirection="column" data-testid="PlacementsField-Container" > - + - {t('universeForm.cloudConfig.azHeader')} + {t('universeForm.cloudConfig.azHeader')} setValue(RESET_AZ_FIELD, true)}> {t('universeForm.cloudConfig.resetAZLabel')} @@ -227,9 +235,8 @@ export const PlacementsField = ({ disabled, isPrimary }: PlacementsFieldProps): return ( diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/PlacementsField/PlacementsFieldHelper.ts b/managed/ui/src/redesign/features/universe/universe-form/form/fields/PlacementsField/PlacementsFieldHelper.ts index cbad03c2703f..5d8eb4326e85 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/PlacementsField/PlacementsFieldHelper.ts +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/PlacementsField/PlacementsFieldHelper.ts @@ -109,7 +109,6 @@ export const useGetAllZones = () => { useEffect(() => { const selectedRegions = new Set(regionList); - const zones = (allRegions || []) .filter((region) => selectedRegions.has(region.uuid)) .flatMap((region: any) => { @@ -147,7 +146,7 @@ export const useNodePlacements = () => { const { setValue, getValues } = useFormContext(); const [ { universeConfigureTemplate, clusterType, mode }, - { setUniverseConfigureTemplate, setUniverseResourceTemplate } + { setUniverseConfigureTemplate, setUniverseResourceTemplate, setConfigureError } ]: any = useContext(UniverseFormContext); //watchers @@ -161,8 +160,8 @@ export const useNodePlacements = () => { const defaultRegion = useWatch({ name: DEFAULT_REGION_FIELD }); const defaultMasterRegion = useWatch({ name: MASTERS_IN_DEFAULT_REGION_FIELD }); const masterPlacement = useWatch({ name: MASTER_PLACEMENT_FIELD }); - const masterDeviceInfo = useWatch({ name: MASTER_DEVICE_INFO_FIELD }); - const masterInstanceType = useWatch({ name: MASTER_INSTANCE_TYPE_FIELD }); + const masterDeviceInfo = useWatch({ name: MASTER_DEVICE_INFO_FIELD }); + const masterInstanceType = useWatch({ name: MASTER_INSTANCE_TYPE_FIELD }); const tserverK8SNodeResourceSpec = useWatch({ name: TSERVER_K8_NODE_SPEC_FIELD }); const masterK8SNodeResourceSpec = useWatch({ name: MASTER_K8_NODE_SPEC_FIELD }); const resetAZ = useWatch({ name: RESET_AZ_FIELD }); @@ -253,7 +252,7 @@ export const useNodePlacements = () => { setUniverseConfigureTemplate(data); setRegionsChanged(false); setNeedPlacement(false); - + setConfigureError(null); try { let resource = await api.universeResource(data); // set Universe resource template whenever configure is called setUniverseResourceTemplate(resource); @@ -262,6 +261,7 @@ export const useNodePlacements = () => { } }, onError: (error) => { + setConfigureError(createErrorMessage(error)); toast.error(createErrorMessage(error), { autoClose: TOAST_AUTO_DISMISS_INTERVAL }); } } @@ -296,6 +296,18 @@ export const useNodePlacements = () => { } prevPropsCombination.current = propsCombination; - }, [instanceType, regionList, totalNodes, replicationFactor, deviceInfo, masterPlacement, masterDeviceInfo, masterInstanceType, resetAZ, tserverK8SNodeResourceSpec, masterK8SNodeResourceSpec]); + }, [ + instanceType, + regionList, + totalNodes, + replicationFactor, + deviceInfo, + masterPlacement, + masterDeviceInfo, + masterInstanceType, + resetAZ, + tserverK8SNodeResourceSpec, + masterK8SNodeResourceSpec + ]); return { isLoading: isFetching }; }; diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/ProvidersField/ProvidersField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/ProvidersField/ProvidersField.tsx index 7d92bcfb5daf..82d775db4fe5 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/ProvidersField/ProvidersField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/ProvidersField/ProvidersField.tsx @@ -73,7 +73,8 @@ export const ProvidersField = ({ ybInputProps={{ error: !!fieldState.error, helperText: fieldState.error?.message, - 'data-testid': 'ProvidersField-AutoComplete' + 'data-testid': 'ProvidersField-AutoComplete', + placeholder: t('universeForm.cloudConfig.placeholder.selectProvider') }} /> diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/RegionsField/RegionsField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/RegionsField/RegionsField.tsx index d3d044a2407d..14596d9bbbf7 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/RegionsField/RegionsField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/RegionsField/RegionsField.tsx @@ -84,7 +84,10 @@ export const RegionsField = ({ disabled }: RegionsFieldProps): ReactElement => { ybInputProps={{ error: !!fieldState.error, helperText: fieldState.error?.message, - 'data-testid': 'RegionsField-AutoComplete' + 'data-testid': 'RegionsField-AutoComplete', + placeholder: _.isEmpty(value) + ? t('universeForm.cloudConfig.placeholder.selectRegions') + : '' }} /> diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/ReplicationFactorField/ReplicationFactorField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/ReplicationFactorField/ReplicationFactorField.tsx index ee127e560630..9f990585b1cc 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/ReplicationFactorField/ReplicationFactorField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/ReplicationFactorField/ReplicationFactorField.tsx @@ -7,6 +7,7 @@ import { YBButton, YBLabel, YBInputField } from '../../../../../../components'; import { UniverseFormData } from '../../../utils/dto'; import { REPLICATION_FACTOR_FIELD, TOAST_AUTO_DISMISS_INTERVAL } from '../../../utils/constants'; import { themeVariables } from '../../../../../../theme/variables'; +import { useFormFieldStyles } from '../../../universeMainStyle'; interface ReplicationFactorProps { disabled?: boolean; @@ -31,6 +32,7 @@ export const ReplicationFactor = ({ const { control, setValue } = useFormContext(); const { t } = useTranslation(); const classes = useStyles(); + const fieldClasses = useFormFieldStyles(); const { field: { value } @@ -44,18 +46,20 @@ export const ReplicationFactor = ({ const handleChange = (e: ChangeEvent) => { //reset field value - const fieldValue = e.target.value as unknown as number; + const fieldValue = (e.target.value as unknown) as number; - if(!fieldValue || fieldValue < ASYNC_RF_MIN) { - setValue(REPLICATION_FACTOR_FIELD, ASYNC_RF_MIN, {shouldValidate:true}); - toast.error(t('universeForm.cloudConfig.minRFValue', {rfValue: ASYNC_RF_MIN}), {autoClose: TOAST_AUTO_DISMISS_INTERVAL}) - } - else if(fieldValue > ASYNC_RF_MAX) { - setValue(REPLICATION_FACTOR_FIELD, ASYNC_RF_MAX, {shouldValidate:true}); - toast.error(t('universeForm.cloudConfig.maxRFvalue', {rfValue: ASYNC_RF_MAX}), {autoClose: TOAST_AUTO_DISMISS_INTERVAL}) - } - else setValue(REPLICATION_FACTOR_FIELD, fieldValue, {shouldValidate:true}); - } + if (!fieldValue || fieldValue < ASYNC_RF_MIN) { + setValue(REPLICATION_FACTOR_FIELD, ASYNC_RF_MIN, { shouldValidate: true }); + toast.error(t('universeForm.cloudConfig.minRFValue', { rfValue: ASYNC_RF_MIN }), { + autoClose: TOAST_AUTO_DISMISS_INTERVAL + }); + } else if (fieldValue > ASYNC_RF_MAX) { + setValue(REPLICATION_FACTOR_FIELD, ASYNC_RF_MAX, { shouldValidate: true }); + toast.error(t('universeForm.cloudConfig.maxRFvalue', { rfValue: ASYNC_RF_MAX }), { + autoClose: TOAST_AUTO_DISMISS_INTERVAL + }); + } else setValue(REPLICATION_FACTOR_FIELD, fieldValue, { shouldValidate: true }); + }; return ( @@ -64,7 +68,7 @@ export const ReplicationFactor = ({ ? t('universeForm.cloudConfig.replicationField') : t('universeForm.cloudConfig.numReadReplicas')} - + {isPrimary ? ( {PRIMARY_RF.map((factor) => { diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/TotalNodesField/TotalNodesField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/TotalNodesField/TotalNodesField.tsx index 5dca40646d45..4b66aeab1667 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/TotalNodesField/TotalNodesField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/TotalNodesField/TotalNodesField.tsx @@ -41,16 +41,18 @@ export const TotalNodesField = ({ disabled }: TotalNodesFieldProps): ReactElemen const placements = useWatch({ name: PLACEMENTS_FIELD }); const currentTotalNodes = getValues(TOTAL_NODES_FIELD); - const fieldLabel = provider?.code === CloudType.kubernetes - ? t('universeForm.cloudConfig.totalPodsField') - : t('universeForm.cloudConfig.totalNodesField'); + const fieldLabel = + provider?.code === CloudType.kubernetes + ? t('universeForm.cloudConfig.totalPodsField') + : t('universeForm.cloudConfig.totalNodesField'); - const handleChange = (e:FocusEvent) => { + const handleChange = (e: FocusEvent) => { //reset field value to replication factor if field is empty or less than RF - const fieldValue = e.target.value as unknown as number; - if(!fieldValue || fieldValue < replicationFactor) setValue(TOTAL_NODES_FIELD, replicationFactor, {shouldValidate:true}); - else setValue(TOTAL_NODES_FIELD, fieldValue, {shouldValidate:true}); - } + const fieldValue = (e.target.value as unknown) as number; + if (!fieldValue || fieldValue < replicationFactor) + setValue(TOTAL_NODES_FIELD, replicationFactor, { shouldValidate: true }); + else setValue(TOTAL_NODES_FIELD, fieldValue, { shouldValidate: true }); + }; //set TotalNodes to RF Value when totalNodes < RF useUpdateEffect(() => { @@ -83,7 +85,9 @@ export const TotalNodesField = ({ disabled }: TotalNodesFieldProps): ReactElemen type="number" disabled={disabled} rules={{ - required: !disabled ? (t('universeForm.validation.required', {field: fieldLabel}) as string) : '' + required: !disabled + ? (t('universeForm.validation.required', { field: fieldLabel }) as string) + : '' }} inputProps={{ 'data-testid': 'TotalNodesField-TServer-Input', @@ -121,9 +125,7 @@ export const TotalNodesField = ({ disabled }: TotalNodesFieldProps): ReactElemen return ( - - {fieldLabel} - + {fieldLabel} {numNodesElement} ); diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/UniverseNameField/UniverseNameField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/UniverseNameField/UniverseNameField.tsx index b8cb5cfb3923..d80238366da1 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/UniverseNameField/UniverseNameField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/UniverseNameField/UniverseNameField.tsx @@ -55,6 +55,7 @@ export const UniverseNameField = ({ disabled }: UniverseNameFieldProps): ReactEl autoFocus: true, 'data-testid': 'UniverseNameField-Input' }} + placeholder={t('universeForm.cloudConfig.placeholder.enterUniverseName')} /> diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/UserTagsField/UserTagsField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/UserTagsField/UserTagsField.tsx index 32b9120fb705..18d45318d07c 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/UserTagsField/UserTagsField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/UserTagsField/UserTagsField.tsx @@ -1,19 +1,23 @@ import React, { ReactElement } from 'react'; +import _ from 'lodash'; import { useTranslation } from 'react-i18next'; import { useFormContext, useFieldArray, FieldArrayPath } from 'react-hook-form'; -import { Box, Grid, IconButton } from '@material-ui/core'; +import { Box, IconButton } from '@material-ui/core'; import { YBButton, YBInputField } from '../../../../../../components'; import { UniverseFormData, InstanceTag } from '../../../utils/dto'; import { USER_TAGS_FIELD } from '../../../utils/constants'; //Icons -import { ReactComponent as CloseIcon } from '../../../../../../assets/close.svg'; +import { CloseSharp } from '@material-ui/icons'; +import { useFormFieldStyles } from '../../../universeMainStyle'; interface UserTagsFieldProps { disabled: boolean; + isAsyncCluster?: boolean; } -export const UserTagsField = ({ disabled }: UserTagsFieldProps): ReactElement => { +export const UserTagsField = ({ disabled, isAsyncCluster }: UserTagsFieldProps): ReactElement => { const { t } = useTranslation(); + const classes = useFormFieldStyles(); const { control } = useFormContext(); const { fields, append, remove } = useFieldArray({ @@ -22,54 +26,62 @@ export const UserTagsField = ({ disabled }: UserTagsFieldProps): ReactElement => }); return ( - - - {fields.map((field, index) => { - return ( - - - } - control={control} - fullWidth - disabled={disabled} - inputProps={{ - 'data-testid': `UniverseNameField-NameInput${index}` - }} - /> - - - } - control={control} - fullWidth - disabled={disabled} - inputProps={{ - 'data-testid': `UniverseNameField-ValueInput${index}` - }} - /> - - {!disabled && ( - - remove(index)} - > - - - - )} - - ); - })} - + + {fields.map((field, index) => { + if (isAsyncCluster && (_.isEmpty(field.name) || _.isEmpty(field.value))) return null; + return ( + + + } + control={control} + fullWidth + disabled={disabled} + placeholder={t('universeForm.userTags.tagName')} + inputProps={{ + 'data-testid': `UniverseNameField-NameInput${index}` + }} + /> + + + } + control={control} + fullWidth + disabled={disabled} + placeholder={t('universeForm.userTags.tagValue')} + inputProps={{ + 'data-testid': `UniverseNameField-ValueInput${index}` + }} + /> + + {!disabled && ( + + remove(index)} + > + + + + )} + + ); + })} {!disabled && ( append({ name: '', value: '' })} + size="medium" disabled={disabled} > @@ -77,6 +89,6 @@ export const UserTagsField = ({ disabled }: UserTagsFieldProps): ReactElement => )} - + ); }; diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/YCQLField/YCQLField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/YCQLField/YCQLField.tsx index 6f028228ce74..8010807e283c 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/YCQLField/YCQLField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/YCQLField/YCQLField.tsx @@ -134,6 +134,7 @@ export const YCQLField = ({ disabled, enforceAuth }: YCQLFieldProps): ReactEleme }} error={!!errors?.instanceConfig?.ycqlPassword} helperText={errors?.instanceConfig?.ycqlPassword?.message} + placeholder={t('universeForm.securityConfig.placeholder.enterYCQLPassword')} /> @@ -164,6 +165,9 @@ export const YCQLField = ({ disabled, enforceAuth }: YCQLFieldProps): ReactEleme }} error={!!errors?.instanceConfig?.ycqlConfirmPassword} helperText={errors?.instanceConfig?.ycqlConfirmPassword?.message ?? ''} + placeholder={t( + 'universeForm.securityConfig.placeholder.confirmYCQLPassword' + )} /> diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/YSQLField/YSQLField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/YSQLField/YSQLField.tsx index d13ce9e1754e..35808dbcd108 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/YSQLField/YSQLField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/YSQLField/YSQLField.tsx @@ -133,6 +133,7 @@ export const YSQLField = ({ disabled, enforceAuth }: YSQLFieldProps): ReactEleme }} error={!!errors?.instanceConfig?.ysqlPassword} helperText={errors?.instanceConfig?.ysqlPassword?.message} + placeholder={t('universeForm.securityConfig.placeholder.enterYSQLPassword')} /> @@ -163,6 +164,9 @@ export const YSQLField = ({ disabled, enforceAuth }: YSQLFieldProps): ReactEleme }} error={!!errors?.instanceConfig?.ysqlConfirmPassword} helperText={errors?.instanceConfig?.ysqlConfirmPassword?.message} + placeholder={t( + 'universeForm.securityConfig.placeholder.confirmYSQLPassword' + )} /> diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/sections/advanced/AdvancedConfiguration.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/sections/advanced/AdvancedConfiguration.tsx index c1f12c258446..7eacc74ab822 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/sections/advanced/AdvancedConfiguration.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/sections/advanced/AdvancedConfiguration.tsx @@ -1,7 +1,7 @@ import React, { FC, useContext } from 'react'; import { useTranslation } from 'react-i18next'; import { useWatch } from 'react-hook-form'; -import { Box, Grid, Typography } from '@material-ui/core'; +import { Box, Typography } from '@material-ui/core'; import { AccessKeysField, ARNField, @@ -34,78 +34,45 @@ export const AdvancedConfiguration: FC = () => { if (!provider?.code) return null; return ( - - - {t('universeForm.advancedConfig.title')} - - - - - - - - + + {t('universeForm.advancedConfig.title')} + + + + {provider.code !== CloudType.kubernetes && ( + + - - {provider.code !== CloudType.kubernetes && ( - - - - - - + )} + {provider.code === CloudType.aws && ( + + + + )} + {provider.code === CloudType.kubernetes && ( + <> + + - )} - - {provider.code === CloudType.aws && ( - - - - - - + + - )} - - {provider.code === CloudType.kubernetes && ( - <> - - - - - - - - - - - - - - - - - )} - - {provider.code !== CloudType.kubernetes && ( - <> - - - - - - - - - - - - - - - - )} - + + )} + {provider.code !== CloudType.kubernetes && ( + <> + + + + + + + + )} ); }; diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx index 4f18cfed427b..dcb93fb0fc29 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx @@ -2,7 +2,7 @@ import React, { useContext } from 'react'; import _ from 'lodash'; import { useSelector } from 'react-redux'; import { useTranslation } from 'react-i18next'; -import { Box, Grid, makeStyles, Typography } from '@material-ui/core'; +import { Box, Typography, useMediaQuery } from '@material-ui/core'; import { DefaultRegionField, MasterPlacementField, @@ -23,16 +23,10 @@ import { } from '../../../utils/dto'; import { useSectionStyles } from '../../../universeMainStyle'; -const useStyles = makeStyles((theme) => ({ - placementFieldContainer: { - width: theme.spacing(70) - } -})); - export const CloudConfiguration = ({ runtimeConfigs }: UniverseFormConfigurationProps) => { const classes = useSectionStyles(); - const helperClasses = useStyles(); const { t } = useTranslation(); + const isLargeDevice = useMediaQuery('(min-width:1400px)'); //feature flagging const featureFlags = useSelector((state: any) => state.featureFlags); @@ -62,59 +56,51 @@ export const CloudConfiguration = ({ runtimeConfigs }: UniverseFormConfiguration : null; return ( - - - - - - - - {t('universeForm.cloudConfig.title')} - - - {isPrimary && ( - - - - )} - - - - - - - {isDedicatedNodesEnabled && ( - - - - )} - - - - - - - {isCreatePrimary && isGeoPartitionEnabled && ( - - - - )} - - + + + + {t('universeForm.cloudConfig.title')} + + {isPrimary && ( + + + + )} + + + + + - - - - - - + {isPrimary && isDedicatedNodesEnabled && ( + + + + )} + + + + + + {isCreatePrimary && isGeoPartitionEnabled && ( + + + + )} + + + ); diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/sections/gflags/Gflags.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/sections/gflags/Gflags.tsx index 95e64fd98613..f916f85c0287 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/sections/gflags/Gflags.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/sections/gflags/Gflags.tsx @@ -26,11 +26,9 @@ export const GFlags: FC = () => { if (!isPrimary || (isEditPrimary && _.isEmpty(getValues(GFLAGS_FIELD)))) return null; return ( - - - {t('universeForm.gFlags.title')} - - + + {t('universeForm.gFlags.title')} + { if (isCreatePrimary && provider?.code === CloudType.kubernetes) return ( - + {t('universeForm.helmOverrides.title')} diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/sections/instance/InstanceConfiguration.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/sections/instance/InstanceConfiguration.tsx index f91a74143fbe..d2d826b25923 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/sections/instance/InstanceConfiguration.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/sections/instance/InstanceConfiguration.tsx @@ -112,10 +112,12 @@ export const InstanceConfiguration = ({ runtimeConfigs }: UniverseFormConfigurat }; return ( - - - {t('universeForm.instanceConfig.title')} - + + {t('universeForm.instanceConfig.title')} diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/sections/security/SecurityConfiguration.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/sections/security/SecurityConfiguration.tsx index 33178da62301..6024aee07a34 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/sections/security/SecurityConfiguration.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/sections/security/SecurityConfiguration.tsx @@ -2,7 +2,7 @@ import React, { useContext } from 'react'; import { useSelector } from 'react-redux'; import { useTranslation } from 'react-i18next'; import { useWatch } from 'react-hook-form'; -import { Box, Grid, Typography, makeStyles } from '@material-ui/core'; +import { Box, Typography, makeStyles } from '@material-ui/core'; import { AssignPublicIPField, ClientToNodeTLSField, @@ -87,10 +87,12 @@ export const SecurityConfiguration = ({ runtimeConfigs }: UniverseFormConfigurat if (!provider?.code) return null; return ( - - - {t('universeForm.securityConfig.title')} - + + {t('universeForm.securityConfig.title')} {[CloudType.aws, CloudType.gcp, CloudType.azu].includes(provider?.code) && ( <> @@ -100,21 +102,13 @@ export const SecurityConfiguration = ({ runtimeConfigs }: UniverseFormConfigurat - - - - - + {currentAccessKeyInfo?.keyInfo?.showSetUpChrony === false && ( - - - - - + )} @@ -148,11 +142,7 @@ export const SecurityConfiguration = ({ runtimeConfigs }: UniverseFormConfigurat - - - - - + @@ -170,19 +160,11 @@ export const SecurityConfiguration = ({ runtimeConfigs }: UniverseFormConfigurat {t('universeForm.securityConfig.encryptionSettings.encryptionInTransit')} - - - - - + - - - - - + {(clientNodeTLSEnabled || nodeNodeTLSEnabled) && ( @@ -200,20 +182,12 @@ export const SecurityConfiguration = ({ runtimeConfigs }: UniverseFormConfigurat {t('universeForm.securityConfig.encryptionSettings.encryptionAtRest')} - - - - - + {encryptionEnabled && isPrimary && ( - - - - - + )} diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/sections/tags/UserTags.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/sections/tags/UserTags.tsx index 7e853a4a2e74..cc257f0d066e 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/sections/tags/UserTags.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/sections/tags/UserTags.tsx @@ -1,10 +1,11 @@ import React, { FC, useContext } from 'react'; +import _ from 'lodash'; import { useTranslation } from 'react-i18next'; -import { useWatch } from 'react-hook-form'; -import { Box, Grid, Typography } from '@material-ui/core'; +import { useWatch, useFormContext } from 'react-hook-form'; +import { Box, Typography } from '@material-ui/core'; import { UserTagsField } from '../../fields'; -import { CloudType, ClusterType } from '../../../utils/dto'; -import { PROVIDER_FIELD } from '../../../utils/constants'; +import { CloudType, ClusterType, UniverseFormData } from '../../../utils/dto'; +import { PROVIDER_FIELD, USER_TAGS_FIELD } from '../../../utils/constants'; import { useSectionStyles } from '../../../universeMainStyle'; import { UniverseFormContext } from '../../../UniverseFormContainer'; @@ -16,19 +17,26 @@ export const UserTags: FC = () => { const { clusterType } = useContext(UniverseFormContext)[0]; const isAsyncCluster = clusterType === ClusterType.ASYNC; + //form Data + const { getValues } = useFormContext>(); + const userTagsValue = getValues(USER_TAGS_FIELD); + const isUserTagExist = userTagsValue?.some((ut) => !_.isEmpty(ut.name) && !_.isEmpty(ut.value)); + //field data const provider = useWatch({ name: PROVIDER_FIELD }); + if (!isUserTagExist && isAsyncCluster) return null; + if ([CloudType.aws, CloudType.gcp, CloudType.azu].includes(provider?.code)) return ( - - - {t('universeForm.userTags.title')} - - - - - + + {t('universeForm.userTags.title')} + + ); diff --git a/managed/ui/src/redesign/features/universe/universe-form/universeMainStyle.ts b/managed/ui/src/redesign/features/universe/universe-form/universeMainStyle.ts index 3a0bdbd2fb96..6c9892afdb11 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/universeMainStyle.ts +++ b/managed/ui/src/redesign/features/universe/universe-form/universeMainStyle.ts @@ -62,7 +62,7 @@ export const useFormMainStyles = makeStyles((theme) => ({ display: 'flex', flexDirection: 'column', height: '100%', - padding: theme.spacing(0, 3), + padding: theme.spacing(0, 5), width: '100%', flexGrow: 1 }, @@ -95,7 +95,6 @@ export const useSectionStyles = makeStyles((theme) => ({ sectionContainer: { display: 'flex', padding: theme.spacing(5, 0), - flexDirection: 'column', borderBottom: `1px solid ${theme.palette.ybacolors.ybGrayHover}` }, sectionHeaderFont: { @@ -106,7 +105,7 @@ export const useSectionStyles = makeStyles((theme) => ({ subsectionHeaderFont: { fontFamily: 'Inter', fontWeight: 600, - fontSize: "15px" + fontSize: '15px' } })); @@ -117,7 +116,7 @@ export const useFormFieldStyles = makeStyles((theme) => ({ }, labelFont: { fontFamily: 'Inter', - fontSize: "13px", + fontSize: '13px', fontWeight: theme.typography.fontWeightMedium as number }, defaultTextBox: { diff --git a/managed/ui/src/redesign/features/universe/universe-form/utils/dto.ts b/managed/ui/src/redesign/features/universe/universe-form/utils/dto.ts index 4104c9422b0b..63d26c416b70 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/utils/dto.ts +++ b/managed/ui/src/redesign/features/universe/universe-form/utils/dto.ts @@ -279,7 +279,7 @@ export interface DeviceInfo { export interface K8NodeSpec { memory: number; - cpu: number + cpu: number; } //-------------------------------------------------------- Most Used OR Common Types - Ends -------------------------------------------------------- @@ -564,7 +564,7 @@ export const DEFAULT_INSTANCE_CONFIG: InstanceConfigFormValue = { ycqlPassword: '', ycqlConfirmPassword: '', enableYEDIS: false, - kmsConfig: null, + kmsConfig: null }; export const DEFAULT_ADVANCED_CONFIG: AdvancedConfigFormValue = { diff --git a/managed/ui/src/translations/en.json b/managed/ui/src/translations/en.json index bdb880a6f246..3cfcc8f299a6 100644 --- a/managed/ui/src/translations/en.json +++ b/managed/ui/src/translations/en.json @@ -12,6 +12,10 @@ "validateAndSave": "Validate and Save", "yes": "Yes" }, + "commonErrors": { + "pageNotExist": "Seems like this Page does not exist", + "pageNotFound": "Page not found" + }, "universeForm": { "master": "Master", "tserver": "TServer", @@ -95,7 +99,12 @@ "masterNumNodesHelper": "Number of nodes dedicated to Masters is equal to the Universe’s replication factor and can not be modified.", "whenToUseDedicatedHelper": "When to use this setting?", "masterPlacementHelper": "Select this option if you plan to use this universe for multi-tenancy use cases -or- you expect to create Databases with a very large number of tables", - "resetAZLabel": "Reset Config" + "resetAZLabel": "Reset Config", + "placeholder": { + "enterUniverseName": "Enter Universe name", + "selectProvider": "Select Provider", + "selectRegions": "Select Regions" + } }, "deleteClusterModal": { "deleteRRMessage": "Are you sure you want to delete this read replica cluster ?", @@ -191,11 +200,19 @@ "enableNodeToNodeTLSHelper": "Enable Node-to-Node encryption to use TLS enabled connections for communication between different Universe nodes.", "rootCertificate": "Root Certificate", "rootCertificatePlaceHolder": "Create New Certificate" + }, + "placeholder": { + "enterYSQLPassword": "Enter YSQL auth password", + "confirmYSQLPassword": "Confirm YSQL auth password", + "enterYCQLPassword": "Enter YCQL auth password", + "confirmYCQLPassword": "Confirm YCQL auth password" } }, "userTags": { - "addRow": "Add Row", - "title": "User Tags" + "addRow": "Add Tag", + "title": "User Tags", + "tagName": "Tag Name", + "tagValue": "Value" }, "resizeNodeModal": { "confirmResizeCheckbox": "Confirm Resize Nodes", From 133a6273fa91c018e21909a13758589cd202989b Mon Sep 17 00:00:00 2001 From: Karan Date: Sat, 4 Mar 2023 18:00:05 +0000 Subject: [PATCH 76/81] [#15767] DocDB: Fix some yb_build issues seen with --no-tcmalloc flag Summary: This change fixes a bunch of undefined symbols/dependencies related issues. These errors are only visible in no-tcmalloc build because we don't allow any undefined symbols in libs. With tcmalloc (default option for CI and local tests), we allow undefined symbols, therefore these issues were not caught during initial review. Test Plan: Jenkins Reviewers: rthallam, asrivastava, mbautin, smishra, mihnea Reviewed By: mbautin Subscribers: amartsinchyk, rsami, rahuldesirazu, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D22425 --- src/postgres/src/backend/exports-common.txt | 1 + src/yb/bfcommon/CMakeLists.txt | 7 +++++-- src/yb/cdc/CMakeLists.txt | 2 +- src/yb/client/CMakeLists.txt | 3 ++- src/yb/common/CMakeLists.txt | 3 ++- src/yb/consensus/CMakeLists.txt | 1 + src/yb/docdb/CMakeLists.txt | 20 +++++++++++++++++++- src/yb/docdb/doc_key.cc | 2 +- src/yb/master/CMakeLists.txt | 3 ++- src/yb/server/CMakeLists.txt | 1 + 10 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/postgres/src/backend/exports-common.txt b/src/postgres/src/backend/exports-common.txt index d16b61592f7d..0a2ba07eacac 100644 --- a/src/postgres/src/backend/exports-common.txt +++ b/src/postgres/src/backend/exports-common.txt @@ -5,6 +5,7 @@ YbgInit # YbgStatus API YbgStatusErrorReportingError YbgStatusCreate +YbgStatusCreateError YbgStatusDestroy YbgStatusIsError YbgStatusGetContext diff --git a/src/yb/bfcommon/CMakeLists.txt b/src/yb/bfcommon/CMakeLists.txt index 317b658289cd..1fa66da62b5f 100644 --- a/src/yb/bfcommon/CMakeLists.txt +++ b/src/yb/bfcommon/CMakeLists.txt @@ -18,12 +18,15 @@ set(YB_PCH_DEP_LIBS gutil yb_test_util yb_util) set(YB_BFCOMMON_SRCS bfdecl.cc) -set(YB_BFPG_LIBS +set(YB_BFCOMMON_LIBS gutil - yb_common_base yb_common_proto yb_util) +if (NOT APPLE) + list(APPEND YB_BFCOMMON_LIBS libuuid) +endif() + ADD_YB_LIBRARY(yb_bfcommon SRCS ${YB_BFCOMMON_SRCS} DEPS ${YB_BFCOMMON_LIBS}) diff --git a/src/yb/cdc/CMakeLists.txt b/src/yb/cdc/CMakeLists.txt index 46e8e6df80e4..62fb021eec89 100644 --- a/src/yb/cdc/CMakeLists.txt +++ b/src/yb/cdc/CMakeLists.txt @@ -66,7 +66,7 @@ set(CDC_YRPC_LIBS opid_proto) ADD_YB_LIBRARY(cdc_service_proto SRCS ${CDC_YRPC_SRCS} cdc_error.cc - DEPS ${CDC_YRPC_LIBS} + DEPS yb_ql_common ${CDC_YRPC_LIBS} NONLINK_DEPS ${CDC_YRPC_TGTS}) ######################################### diff --git a/src/yb/client/CMakeLists.txt b/src/yb/client/CMakeLists.txt index 35ed4e64d3ec..c5429194775b 100644 --- a/src/yb/client/CMakeLists.txt +++ b/src/yb/client/CMakeLists.txt @@ -87,7 +87,8 @@ set(CLIENT_LIBS yb_util gutil yrpc - yb_docdb_encoding) + yb_docdb_encoding + yb_docdb_encoding_util) # We customize the output name/directory of the exported library so that we can # call it "yb_client" without it colliding with the regular library. diff --git a/src/yb/common/CMakeLists.txt b/src/yb/common/CMakeLists.txt index 9a044327be8b..64acd0d455e1 100644 --- a/src/yb/common/CMakeLists.txt +++ b/src/yb/common/CMakeLists.txt @@ -54,6 +54,7 @@ ADD_YB_LIBRARY(yb_common_proto NONLINK_DEPS ${COMMON_PROTO_TGTS}) set(COMMON_BASE_SRCS + column_id.cc json_util.cc jsonb.cc ql_datatype.cc @@ -65,6 +66,7 @@ set(COMMON_BASE_SRCS set(COMMON_BASE_LIBS yb_util yb_common_proto + yb_bfcommon ) ADD_YB_LIBRARY(yb_common_base @@ -75,7 +77,6 @@ set(COMMON_SRCS clock.cc common_flags.cc common_types_util.cc - column_id.cc consistent_read_point.cc entity_ids.cc hybrid_time.cc diff --git a/src/yb/consensus/CMakeLists.txt b/src/yb/consensus/CMakeLists.txt index df92c2b59656..cf7bdf772d36 100644 --- a/src/yb/consensus/CMakeLists.txt +++ b/src/yb/consensus/CMakeLists.txt @@ -116,6 +116,7 @@ target_link_libraries(log server_common gutil yb_common + yb_ql_common yb_fs consensus_proto log_proto diff --git a/src/yb/docdb/CMakeLists.txt b/src/yb/docdb/CMakeLists.txt index cdb6ab39d90a..a8cdbdd798b4 100644 --- a/src/yb/docdb/CMakeLists.txt +++ b/src/yb/docdb/CMakeLists.txt @@ -36,7 +36,6 @@ set(DOCDB_ENCODING_SRCS intent.cc key_bytes.cc primitive_value.cc - primitive_value_util.cc value.cc ) @@ -52,6 +51,24 @@ ADD_YB_LIBRARY(yb_docdb_encoding DEPS ${DOCDB_ENCODING_DEPS} ) +set(DOCDB_ENCODING_UTIL_SRCS + primitive_value_util.cc + ) + +# DocDB Encoding util lib is shared with PG process as well. Avoid +# including tserver specific dependencies for it. +set(DOCDB_ENCODING_UTIL_DEPS + yb_common + yb_ql_common + yb_util + ) + +ADD_YB_LIBRARY(yb_docdb_encoding_util + SRCS ${DOCDB_ENCODING_UTIL_SRCS} + DEPS ${DOCDB_ENCODING_UTIL_DEPS} + ) + + include_directories(${YB_BUILD_ROOT}/postgres/include/server) include_directories("${YB_BUILD_ROOT}/postgres_build/src/include/catalog") @@ -110,6 +127,7 @@ set(DOCDB_SRCS set(DOCDB_DEPS yb_docdb_encoding + yb_docdb_encoding_util rocksdb yb_util yb_rocksutil diff --git a/src/yb/docdb/doc_key.cc b/src/yb/docdb/doc_key.cc index 502e705e6c41..7793f3c42c39 100644 --- a/src/yb/docdb/doc_key.cc +++ b/src/yb/docdb/doc_key.cc @@ -441,7 +441,7 @@ yb::DocKeyOffsets DocKey::ComputeKeyColumnOffsets(const Schema& schema) { KeyEntryValue::GetEncodedKeyEntryValueSize(schema.column(i).type()->main()); LOG_IF(DFATAL, encoded_size == 0) << "Encountered a varlength column when computing Key offsets. Column " - << schema.column(i).ToString(); + << schema.column(i).name(); offset += encoded_size; } diff --git a/src/yb/master/CMakeLists.txt b/src/yb/master/CMakeLists.txt index 1dbcb61e0be5..986437623c20 100644 --- a/src/yb/master/CMakeLists.txt +++ b/src/yb/master/CMakeLists.txt @@ -169,6 +169,7 @@ set(MASTER_RPC_SRCS master_rpc.cc) set(MASTER_RPC_LIBS yb_common + yb_ql_common yrpc gutil yb_util @@ -188,7 +189,7 @@ set(MASTER_UTIL_SRCS master_util.cc) ADD_YB_LIBRARY(master_util SRCS ${MASTER_UTIL_SRCS} - DEPS master_error ${MASTER_RPC_LIBS}) + DEPS yb_ql_common master_error ${MASTER_RPC_LIBS}) # A library with common code shared between master tests. set(MASTER_TEST_COMMON_SRCS diff --git a/src/yb/server/CMakeLists.txt b/src/yb/server/CMakeLists.txt index 20c88664b356..c4cbef6c9256 100644 --- a/src/yb/server/CMakeLists.txt +++ b/src/yb/server/CMakeLists.txt @@ -110,6 +110,7 @@ target_link_libraries(server_process server_base_proto server_common yb_common + yb_ql_common yb_fs gutil yrpc From 16a8580b505ecbfab2f8b3d87d6ff100963e0b3a Mon Sep 17 00:00:00 2001 From: Karan Date: Thu, 2 Mar 2023 16:23:58 -0800 Subject: [PATCH 77/81] [#13718] DocDB: DocRowwiseIterator tests - validate Iterate and HasNext/DoNextRow APIs for all tests Summary: This change extends all existing DocRowwiseIterator tests and adds validation for Iterate API in addition to existing validation for HasNext/DoNextRow APIs. Also adding validation for max seen HT and couple of new tests: - DeleteMarkerWithPackedRow -- validates delete of a packed row key. - UpdatePackedRow -- validates case where initial insert and update both use packed row. Other changes: - a minor cleanup for ScanChoices, removing the special initial seek path since there is no user anymore. - removed the generic metadata/system key handling for ScanChoices key parsing failure to currently known non-parsable key i.e. table tombstone key. Test Plan: ./yb_build.sh --skip-java --cxx-test docrowwiseiterator-test Reviewers: rthallam, tnayak, rsami, timur, mbautin Reviewed By: timur, mbautin Subscribers: ybase, lnguyen, rskannan Differential Revision: https://phabricator.dev.yugabyte.com/D22532 --- src/yb/docdb/doc_key.cc | 13 + src/yb/docdb/doc_key.h | 2 + src/yb/docdb/doc_rowwise_iterator.cc | 81 +- src/yb/docdb/doc_rowwise_iterator.h | 11 +- src/yb/docdb/docdb_fwd.h | 1 + src/yb/docdb/docrowwiseiterator-test.cc | 1345 ++++++----------- src/yb/docdb/ql_rowwise_iterator_interface.cc | 6 + src/yb/docdb/ql_rowwise_iterator_interface.h | 3 + src/yb/docdb/scan_choices-test.cc | 3 +- src/yb/docdb/scan_choices.cc | 10 +- src/yb/docdb/scan_choices.h | 6 +- 11 files changed, 511 insertions(+), 970 deletions(-) diff --git a/src/yb/docdb/doc_key.cc b/src/yb/docdb/doc_key.cc index 7793f3c42c39..e4d205a76ac6 100644 --- a/src/yb/docdb/doc_key.cc +++ b/src/yb/docdb/doc_key.cc @@ -1390,6 +1390,19 @@ bool DocKeyBelongsTo(Slice doc_key, const Schema& schema) { } } +Result IsColocatedTableTombstoneKey(Slice doc_key) { + DocKeyDecoder decoder(doc_key); + if (VERIFY_RESULT(decoder.DecodeColocationId())) { + RETURN_NOT_OK(decoder.ConsumeGroupEnd()); + + if (decoder.left_input().size() == 0) { + return true; + } + } + + return false; +} + Result> DecodeDocKeyHash(const Slice& encoded_key) { DocKey key; RETURN_NOT_OK(key.DecodeFrom(encoded_key, DocKeyPart::kUpToHashCode)); diff --git a/src/yb/docdb/doc_key.h b/src/yb/docdb/doc_key.h index 21a0068c556d..d98bfa0ddc02 100644 --- a/src/yb/docdb/doc_key.h +++ b/src/yb/docdb/doc_key.h @@ -439,6 +439,8 @@ Result HashedOrFirstRangeComponentsEqual(const Slice& lhs, const Slice& rh bool DocKeyBelongsTo(Slice doc_key, const Schema& schema); +Result IsColocatedTableTombstoneKey(Slice doc_key); + // Consumes single primitive value from start of slice. // Returns true when value was consumed, false when group end is found. The group end byte is // consumed in the latter case. diff --git a/src/yb/docdb/doc_rowwise_iterator.cc b/src/yb/docdb/doc_rowwise_iterator.cc index 628f7773a9f1..70a7c1858945 100644 --- a/src/yb/docdb/doc_rowwise_iterator.cc +++ b/src/yb/docdb/doc_rowwise_iterator.cc @@ -184,33 +184,17 @@ Status DocRowwiseIterator::Init(TableType table_type, const Slice& sub_doc_key) return Status::OK(); } -Result DocRowwiseIterator::InitScanChoices( +void DocRowwiseIterator::InitScanChoices( const DocQLScanSpec& doc_spec, const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key) { - scan_choices_ = ScanChoices::Create(doc_read_context_.schema, doc_spec, lower_doc_key, - upper_doc_key, doc_spec.prefix_length()); - - if (scan_choices_ && scan_choices_->IsInitialPositionKnown()) { - // Let's not seek to the lower doc key or upper doc key. We know exactly what we want. - RETURN_NOT_OK(AdvanceIteratorToNextDesiredRow()); - return true; - } - - return false; + scan_choices_ = + ScanChoices::Create(doc_read_context_.schema, doc_spec, lower_doc_key, upper_doc_key); } -Result DocRowwiseIterator::InitScanChoices( +void DocRowwiseIterator::InitScanChoices( const DocPgsqlScanSpec& doc_spec, const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key) { - scan_choices_ = ScanChoices::Create(doc_read_context_.schema, doc_spec, lower_doc_key, - upper_doc_key, doc_spec.prefix_length()); - - if (scan_choices_ && scan_choices_->IsInitialPositionKnown()) { - // Let's not seek to the lower doc key or upper doc key. We know exactly what we want. - RETURN_NOT_OK(AdvanceIteratorToNextDesiredRow()); - return true; - } - - return false; + scan_choices_ = + ScanChoices::Create(doc_read_context_.schema, doc_spec, lower_doc_key, upper_doc_key); } template @@ -251,19 +235,19 @@ Status DocRowwiseIterator::DoInit(const T& doc_spec) { } } - if (!VERIFY_RESULT(InitScanChoices(doc_spec, - !is_forward_scan_ && has_bound_key_ ? bound_key_ : lower_doc_key, - is_forward_scan_ && has_bound_key_ ? bound_key_ : upper_doc_key))) { - if (is_forward_scan_) { - VLOG(3) << __PRETTY_FUNCTION__ << " Seeking to " << DocKey::DebugSliceToString(lower_doc_key); - db_iter_->Seek(lower_doc_key); + InitScanChoices( + doc_spec, + !is_forward_scan_ && has_bound_key_ ? bound_key_ : lower_doc_key, + is_forward_scan_ && has_bound_key_ ? bound_key_ : upper_doc_key); + if (is_forward_scan_) { + VLOG(3) << __PRETTY_FUNCTION__ << " Seeking to " << DocKey::DebugSliceToString(lower_doc_key); + db_iter_->Seek(lower_doc_key); + } else { + // TODO consider adding an operator bool to DocKey to use instead of empty() here. + if (!upper_doc_key.empty()) { + db_iter_->PrevDocKey(upper_doc_key); } else { - // TODO consider adding an operator bool to DocKey to use instead of empty() here. - if (!upper_doc_key.empty()) { - db_iter_->PrevDocKey(upper_doc_key); - } else { - db_iter_->SeekToLastDocKey(); - } + db_iter_->SeekToLastDocKey(); } } @@ -407,11 +391,11 @@ Result DocRowwiseIterator::HasNext() { // skip all scan targets between the current target and row key (excluding row_key_ itself). // Update the target key and iterator and call HasNext again to try the next target. if (!VERIFY_RESULT(scan_choices_->SkipTargetsUpTo(row_key_))) { - // SkipTargetsUpTo returns false when it fails to decode the key. ValidateSystemKey() - // checks if current key is a known system key. This is a temporary fix until we address - // GH15304 (https://github.com/yugabyte/yugabyte-db/issues/15304) which will remove key - // decoding from ScanChoices completely. - RETURN_NOT_OK(ValidateSystemKey()); + // SkipTargetsUpTo returns false when it fails to decode the key. + if (!VERIFY_RESULT(IsColocatedTableTombstoneKey(row_key_))) { + return STATUS_FORMAT( + Corruption, "Key $0 is not table tombstone key.", row_key_.ToDebugHexString()); + } if (is_forward_scan_) { db_iter_->SeekOutOfSubDoc(&iter_key_); } else { @@ -510,6 +494,10 @@ HybridTime DocRowwiseIterator::RestartReadHt() { return HybridTime::kInvalid; } +HybridTime DocRowwiseIterator::TEST_MaxSeenHt() { + return db_iter_->max_seen_ht(); +} + bool DocRowwiseIterator::IsNextStaticColumn() const { return doc_read_context_.schema.has_statics() && row_hash_key_.end() + 1 == row_key_.end(); } @@ -605,21 +593,6 @@ Status DocRowwiseIterator::DoNextRow( return Status::OK(); } -Status DocRowwiseIterator::ValidateSystemKey() { - // Currently we only have Table tombstone key as system key. - DocKeyDecoder decoder(row_key_); - if (VERIFY_RESULT(decoder.DecodeColocationId())) { - RETURN_NOT_OK(decoder.ConsumeGroupEnd()); - - if (decoder.left_input().size() == 0) { - return Status::OK(); - } - } - - return STATUS_FORMAT( - Corruption, "Key parsing failed for non-system key $0", row_key_.ToDebugHexString()); -} - bool DocRowwiseIterator::LivenessColumnExists() const { if (is_flat_doc_) { const auto& type = (*values_)[0].type(); diff --git a/src/yb/docdb/doc_rowwise_iterator.h b/src/yb/docdb/doc_rowwise_iterator.h index 9cbb14c95668..6567023fdd04 100644 --- a/src/yb/docdb/doc_rowwise_iterator.h +++ b/src/yb/docdb/doc_rowwise_iterator.h @@ -107,6 +107,8 @@ class DocRowwiseIterator : public YQLRowwiseIteratorIf { HybridTime RestartReadHt() override; + HybridTime TEST_MaxSeenHt() override; + // Returns the tuple id of the current tuple. The tuple id returned is the serialized DocKey // and without the cotable id. Result GetTupleId() const override; @@ -128,18 +130,16 @@ class DocRowwiseIterator : public YQLRowwiseIteratorIf { // Used only in debug mode to ensure that generated key offsets are correct for provided key. bool ValidateDocKeyOffsets(const Slice& iter_key); - static bool is_hybrid_scan_enabled(); - private: template Status DoInit(const T& spec); void ConfigureForYsql(); void InitResult(); - Result InitScanChoices( + void InitScanChoices( const DocQLScanSpec& doc_spec, const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key); - Result InitScanChoices( + void InitScanChoices( const DocPgsqlScanSpec& doc_spec, const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key); @@ -151,9 +151,6 @@ class DocRowwiseIterator : public YQLRowwiseIteratorIf { // Read next row into a value map using the specified projection. Status DoNextRow(boost::optional projection, QLTableRow* table_row) override; - // Returns OK if row_key_ is pointing to a system key. - Status ValidateSystemKey(); - const std::unique_ptr projection_owner_; // Used to maintain ownership of projection_. // Separate field is used since ownership could be optional. diff --git a/src/yb/docdb/docdb_fwd.h b/src/yb/docdb/docdb_fwd.h index df140fe6b162..cb941c3947aa 100644 --- a/src/yb/docdb/docdb_fwd.h +++ b/src/yb/docdb/docdb_fwd.h @@ -37,6 +37,7 @@ class ExternalTxnIntentsState; class HistoryRetentionPolicy; class IntentAwareIterator; class IntentAwareIteratorIf; +class IntentIterator; class KeyBytes; class KeyEntryValue; class ManualHistoryRetentionPolicy; diff --git a/src/yb/docdb/docrowwiseiterator-test.cc b/src/yb/docdb/docrowwiseiterator-test.cc index ca100c45c24f..157ce869053d 100644 --- a/src/yb/docdb/docrowwiseiterator-test.cc +++ b/src/yb/docdb/docrowwiseiterator-test.cc @@ -51,10 +51,6 @@ class DocRowwiseIteratorTest : public DocDBTestBase { } ~DocRowwiseIteratorTest() override {} - // TODO Could define them out of class, so one line would be enough for them. - static const KeyBytes kEncodedDocKey1; - static const KeyBytes kEncodedDocKey2; - void SetUp() override { FLAGS_TEST_docdb_sort_weak_intents = true; DocDBTestBase::SetUp(); @@ -74,10 +70,14 @@ class DocRowwiseIteratorTest : public DocDBTestBase { void InsertTestRangeData(); + void InsertPackedRow( + SchemaVersion version, std::reference_wrapper schema_packing, + const KeyBytes &doc_key, HybridTime ht, + std::initializer_list> columns); + virtual Result CreateIterator( const Schema &projection, - std::reference_wrapper - doc_read_context, + std::reference_wrapper doc_read_context, const TransactionOperationContext &txn_op_context, const DocDB &doc_db, CoarseTimePoint deadline, @@ -94,8 +94,7 @@ class DocRowwiseIteratorTest : public DocDBTestBase { virtual Result CreateIterator( const Schema &projection, - std::reference_wrapper - doc_read_context, + std::reference_wrapper doc_read_context, const TransactionOperationContext &txn_op_context, const DocDB &doc_db, CoarseTimePoint deadline, @@ -112,8 +111,7 @@ class DocRowwiseIteratorTest : public DocDBTestBase { virtual Result CreateIterator( const Schema &projection, - std::reference_wrapper - doc_read_context, + std::reference_wrapper doc_read_context, const TransactionOperationContext &txn_op_context, const DocDB &doc_db, CoarseTimePoint deadline, @@ -127,6 +125,32 @@ class DocRowwiseIteratorTest : public DocDBTestBase { return iter; } + // CreateIteratorAndValidate functions create iterator and validate output of both Iterate() and + // HasNext/DoNextRow access patterns. + template + void CreateIteratorAndValidate( + const Schema &schema, + const ReadHybridTime &read_time, + const T &spec, + const std::string &expected, + const HybridTime &max_seen_ht = HybridTime::kInvalid, + const Schema *projection = nullptr, + const TransactionOperationContext &txn_op_context = kNonTransactionalOperationContext); + + void CreateIteratorAndValidate( + const Schema &schema, + const ReadHybridTime &read_time, + const std::string &expected, + const HybridTime &max_seen_ht = HybridTime::kInvalid, + const Schema *projection = nullptr, + const TransactionOperationContext &txn_op_context = kNonTransactionalOperationContext); + + void CreateIteratorAndValidate( + const ReadHybridTime &read_time, + const std::string &expected, + const HybridTime &max_seen_ht = HybridTime::kInvalid, + const TransactionOperationContext &txn_op_context = kNonTransactionalOperationContext); + // Test case implementation. void TestClusteredFilterRange(); void TestClusteredFilterRangeWithTableTombstone(); @@ -157,6 +181,8 @@ class DocRowwiseIteratorTest : public DocDBTestBase { void TestScanWithinTheSameTxn(); void TestLargeKeys(); void TestPackedRow(); + void TestDeleteMarkerWithPackedRow(); + void TestUpdatePackedRow(); // Restore doesn't use delete tombstones for rows, instead marks all columns // as deleted. void TestDeletedDocumentUsingLivenessColumnDelete(); @@ -164,15 +190,15 @@ class DocRowwiseIteratorTest : public DocDBTestBase { std::optional projection_; }; -const std::string kStrKey1 = "row1"; -constexpr int64_t kIntKey1 = 11111; -const std::string kStrKey2 = "row2"; -constexpr int64_t kIntKey2 = 22222; +static const std::string kStrKey1 = "row1"; +static constexpr int64_t kIntKey1 = 11111; +static const std::string kStrKey2 = "row2"; +static constexpr int64_t kIntKey2 = 22222; -const KeyBytes DocRowwiseIteratorTest::kEncodedDocKey1( +static const KeyBytes kEncodedDocKey1( DocKey(KeyEntryValues(kStrKey1, kIntKey1)).Encode()); -const KeyBytes DocRowwiseIteratorTest::kEncodedDocKey2( +static const KeyBytes kEncodedDocKey2( DocKey(KeyEntryValues(kStrKey2, kIntKey2)).Encode()); Schema DocRowwiseIteratorTest::CreateSchema() { @@ -191,7 +217,6 @@ Schema DocRowwiseIteratorTest::CreateSchema() { 50_ColId }, 2); } - constexpr int32_t kFixedHashCode = 0; const KeyBytes GetKeyBytes( @@ -324,9 +349,143 @@ void DocRowwiseIteratorTest::InsertTestRangeData() { } } +void DocRowwiseIteratorTest::InsertPackedRow( + SchemaVersion version, std::reference_wrapper schema_packing, + const KeyBytes &doc_key, HybridTime ht, + std::initializer_list> columns) { + RowPacker packer( + version, schema_packing, /* packed_size_limit= */ std::numeric_limits::max(), + /* value_control_fields= */ Slice()); + + for (auto &column : columns) { + ASSERT_OK(packer.AddValue(column.first, column.second)); + } + auto packed_row = ASSERT_RESULT(packer.Complete()); + + ASSERT_OK(SetPrimitive( + DocPath(doc_key), ValueControlFields(), ValueRef(packed_row), ht)); +} + +Result QLTableRowToString( + const Schema &schema, const QLTableRow &row, const Schema *projection) { + QLValue value; + std::stringstream buffer; + buffer << "{"; + for (size_t idx = 0; idx < schema.num_columns(); idx++) { + if (idx != 0) { + buffer << ","; + } + if (projection && + projection->find_column_by_id(schema.column_id(idx)) == Schema::kColumnNotFound) { + buffer << "missing"; + } else { + RETURN_NOT_OK(row.GetValue(schema.column_id(idx), &value)); + buffer << value.ToString(); + } + } + buffer << "}"; + return buffer.str(); +} + +Result ConvertIteratorRowsToString( + YQLRowwiseIteratorIf *iter, + const Schema &schema, + bool use_iterate_callback = false, + const Schema *projection = nullptr) { + std::stringstream buffer; + if (use_iterate_callback) { + RETURN_NOT_OK(iter->Iterate([&](const QLTableRow &row) -> Result { + buffer << VERIFY_RESULT(QLTableRowToString(schema, row, projection)); + buffer << "\n"; + + return ContinueScan::kTrue; + })); + } else { + QLTableRow row; + while (VERIFY_RESULT(iter->HasNext())) { + RETURN_NOT_OK(iter->NextRow(&row)); + buffer << VERIFY_RESULT(QLTableRowToString(schema, row, projection)); + buffer << "\n"; + } + } + + return buffer.str(); +} + +void ValidateIterator( + YQLRowwiseIteratorIf *iter, + const Schema &schema, + bool use_iterate_callback, + const Schema *projection, + const std::string &expected, + const HybridTime &expected_max_seen_ht) { + ASSERT_STR_EQ_VERBOSE_TRIMMED( + expected, + ASSERT_RESULT( + ConvertIteratorRowsToString(iter, schema, use_iterate_callback, projection))); + + ASSERT_EQ(expected_max_seen_ht, iter->TEST_MaxSeenHt()); +} + +template +void DocRowwiseIteratorTest::CreateIteratorAndValidate( + const Schema &schema, + const ReadHybridTime &read_time, + const T &spec, + const std::string &expected, + const HybridTime &expected_max_seen_ht, + const Schema *projection, + const TransactionOperationContext &txn_op_context) { + auto doc_read_context = DocReadContext::TEST_Create(schema); + + for (bool use_iterate_callback : {false, true}) { + auto iter = ASSERT_RESULT(CreateIterator( + projection ? *projection : schema, doc_read_context, txn_op_context, doc_db(), + CoarseTimePoint::max() /* deadline */, read_time, spec)); + + ValidateIterator( + iter.get(), schema, use_iterate_callback, projection, expected, expected_max_seen_ht); + } +} + +void DocRowwiseIteratorTest::CreateIteratorAndValidate( + const Schema &schema, + const ReadHybridTime &read_time, + const std::string &expected, + const HybridTime &expected_max_seen_ht, + const Schema *projection, + const TransactionOperationContext &txn_op_context) { + + for (bool use_iterate_callback : {false, true}) { + auto iter = ASSERT_RESULT(CreateIterator( + projection ? *projection : schema, doc_read_context(), txn_op_context, doc_db(), + CoarseTimePoint::max() /* deadline */, read_time)); + + ValidateIterator( + iter.get(), schema, use_iterate_callback, projection, expected, expected_max_seen_ht); + } +} + +void DocRowwiseIteratorTest::CreateIteratorAndValidate( + const ReadHybridTime &read_time, + const std::string &expected, + const HybridTime &expected_max_seen_ht, + const TransactionOperationContext &txn_op_context) { + auto& projection = this->projection(); + + for (bool use_iterate_callback : {false, true}) { + auto iter = ASSERT_RESULT(CreateIterator( + projection, doc_read_context(), txn_op_context, doc_db(), + CoarseTimePoint::max() /* deadline */, read_time)); + + ValidateIterator( + iter.get(), doc_read_context().schema, use_iterate_callback, &doc_read_context().schema, + expected, expected_max_seen_ht); + } +} + void DocRowwiseIteratorTest::TestClusteredFilterRange() { InsertTestRangeData(); - auto doc_read_context = DocReadContext::TEST_Create(test_range_schema); const std::vector hashed_components{KeyEntryValue::Int32(5)}; @@ -345,32 +504,12 @@ void DocRowwiseIteratorTest::TestClusteredFilterRange() { test_range_schema, kFixedHashCode, kFixedHashCode, hashed_components, &cond, nullptr, rocksdb::kDefaultQueryId); - auto iter = ASSERT_RESULT(CreateIterator( - test_range_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - QLTableRow row; - QLValue value; - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(test_range_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(5, value.int32_value()); - - ASSERT_OK(row.GetValue(test_range_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(5, value.int32_value()); - - ASSERT_OK(row.GetValue(test_range_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(6, value.int32_value()); - - ASSERT_OK(row.GetValue(test_range_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(6, value.int32_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + test_range_schema, ReadHybridTime::FromMicros(2000), spec, + R"#( + {int32:5,int32:5,int32:6,int32:6} + )#", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestClusteredFilterRangeWithTableTombstone() { @@ -400,8 +539,6 @@ void DocRowwiseIteratorTest::TestClusteredFilterRangeWithTableTombstone() { DocKey colocation_key(colocation_id); ASSERT_OK(DeleteSubDoc(DocPath(colocation_key.Encode()), HybridTime::FromMicros(500))); - auto doc_read_context = DocReadContext::TEST_Create(test_schema); - PgsqlConditionPB cond; auto ids = cond.add_operands()->mutable_tuple(); ids->add_elems()->set_column_id(12_ColId); @@ -419,28 +556,12 @@ void DocRowwiseIteratorTest::TestClusteredFilterRangeWithTableTombstone() { test_schema, rocksdb::kDefaultQueryId, empty_key_components, empty_key_components, &cond, empty_hash_code, empty_hash_code, nullptr); - auto iter = ASSERT_RESULT(CreateIterator( - test_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - QLTableRow row; - QLValue value; - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(test_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(5, value.int32_value()); - - ASSERT_OK(row.GetValue(test_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(6, value.int32_value()); - - ASSERT_OK(row.GetValue(test_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int32_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + test_schema, ReadHybridTime::FromMicros(2000), spec, + R"#( + {int32:5,int32:6,int32:10} + )#", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestClusteredFilterRangeWithTableTombstoneReverseScan() { @@ -470,8 +591,6 @@ void DocRowwiseIteratorTest::TestClusteredFilterRangeWithTableTombstoneReverseSc DocKey colocation_key(colocation_id); ASSERT_OK(DeleteSubDoc(DocPath(colocation_key.Encode()), HybridTime::FromMicros(500))); - auto doc_read_context = DocReadContext::TEST_Create(test_schema); - PgsqlConditionPB cond; auto ids = cond.add_operands()->mutable_tuple(); ids->add_elems()->set_column_id(12_ColId); @@ -488,33 +607,16 @@ void DocRowwiseIteratorTest::TestClusteredFilterRangeWithTableTombstoneReverseSc test_schema, rocksdb::kDefaultQueryId, empty_key_components, empty_key_components, &cond, empty_hash_code, empty_hash_code, nullptr, default_doc_key, /* is_forward_scan */ false); - auto iter = ASSERT_RESULT(CreateIterator( - test_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - QLTableRow row; - QLValue value; - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(test_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(5, value.int32_value()); - - ASSERT_OK(row.GetValue(test_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(6, value.int32_value()); - - ASSERT_OK(row.GetValue(test_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int32_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + test_schema, ReadHybridTime::FromMicros(2000), spec, + R"#( + {int32:5,int32:6,int32:10} + )#", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestClusteredFilterHybridScan() { InsertPopulationData(); - auto doc_read_context = DocReadContext::TEST_Create(population_schema); const std::vector hashed_components{KeyEntryValue(INDIA)}; @@ -540,64 +642,17 @@ void DocRowwiseIteratorTest::TestClusteredFilterHybridScan() { population_schema, kFixedHashCode, kFixedHashCode, hashed_components, &cond, nullptr, rocksdb::kDefaultQueryId); - auto iter = ASSERT_RESULT(CreateIterator( - population_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - QLTableRow row; - QLValue value; - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(CG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(DURG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(KA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(MYSORE, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + population_schema, ReadHybridTime::FromMicros(2000), spec, + R"#( + {string:"INDIA",string:"CG",string:"DURG",string:"AREA1",int64:10} + {string:"INDIA",string:"KA",string:"MYSORE",string:"AREA1",int64:10} + )#", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestClusteredFilterSubsetCol() { InsertPopulationData(); - auto doc_read_context = DocReadContext::TEST_Create(population_schema); const std::vector hashed_components{KeyEntryValue(INDIA)}; @@ -619,110 +674,20 @@ void DocRowwiseIteratorTest::TestClusteredFilterSubsetCol() { DocQLScanSpec spec( population_schema, kFixedHashCode, kFixedHashCode, hashed_components, &cond, nullptr, rocksdb::kDefaultQueryId); - auto iter = ASSERT_RESULT(CreateIterator( - population_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - QLTableRow row; - QLValue value; - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(CG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(DURG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(CG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(DURG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA2, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(KA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(MYSORE, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(KA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(MYSORE, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA2, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + population_schema, ReadHybridTime::FromMicros(2000), spec, + R"#( + {string:"INDIA",string:"CG",string:"DURG",string:"AREA1",int64:10} + {string:"INDIA",string:"CG",string:"DURG",string:"AREA2",int64:10} + {string:"INDIA",string:"KA",string:"MYSORE",string:"AREA1",int64:10} + {string:"INDIA",string:"KA",string:"MYSORE",string:"AREA2",int64:10} + )#", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestClusteredFilterSubsetCol2() { InsertPopulationData(); - auto doc_read_context = DocReadContext::TEST_Create(population_schema); const std::vector hashed_components{KeyEntryValue(INDIA)}; @@ -744,64 +709,18 @@ void DocRowwiseIteratorTest::TestClusteredFilterSubsetCol2() { DocQLScanSpec spec( population_schema, kFixedHashCode, kFixedHashCode, hashed_components, &cond, nullptr, rocksdb::kDefaultQueryId); - auto iter = ASSERT_RESULT(CreateIterator( - population_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - QLTableRow row; - QLValue value; - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(CG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(DURG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(KA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(MYSORE, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + population_schema, ReadHybridTime::FromMicros(2000), spec, + R"#( + {string:"INDIA",string:"CG",string:"DURG",string:"AREA1",int64:10} + {string:"INDIA",string:"KA",string:"MYSORE",string:"AREA1",int64:10} + )#", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestClusteredFilterMultiIn() { InsertPopulationData(); - auto doc_read_context = DocReadContext::TEST_Create(population_schema); const std::vector hashed_components{KeyEntryValue(INDIA)}; @@ -832,64 +751,17 @@ void DocRowwiseIteratorTest::TestClusteredFilterMultiIn() { population_schema, kFixedHashCode, kFixedHashCode, hashed_components, &cond, nullptr, rocksdb::kDefaultQueryId); - auto iter = ASSERT_RESULT(CreateIterator( - population_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - QLTableRow row; - QLValue value; - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(CG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(DURG, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(population_schema.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(INDIA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(KA, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(MYSORE, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(3), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(AREA1, value.string_value()); - - ASSERT_OK(row.GetValue(population_schema.column_id(4), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10, value.int64_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + population_schema, ReadHybridTime::FromMicros(2000), spec, + R"#( + {string:"INDIA",string:"CG",string:"DURG",string:"AREA1",int64:10} + {string:"INDIA",string:"KA",string:"MYSORE",string:"AREA1",int64:10} + )#", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestClusteredFilterEmptyIn() { InsertPopulationData(); - auto doc_read_context = DocReadContext::TEST_Create(population_schema); const std::vector hashed_components{KeyEntryValue(INDIA)}; @@ -914,11 +786,10 @@ void DocRowwiseIteratorTest::TestClusteredFilterEmptyIn() { population_schema, kFixedHashCode, kFixedHashCode, hashed_components, &cond, nullptr, rocksdb::kDefaultQueryId); - auto iter = ASSERT_RESULT(CreateIterator( - population_schema, doc_read_context, kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000), spec)); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + CreateIteratorAndValidate( + population_schema, ReadHybridTime::FromMicros(2000), spec, + "", + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::SetupDocRowwiseIteratorData() { @@ -976,89 +847,22 @@ void DocRowwiseIteratorTest::SetupDocRowwiseIteratorData() { void DocRowwiseIteratorTest::TestDocRowwiseIterator() { SetupDocRowwiseIteratorData(); - const auto& projection = this->projection(); - QLTableRow row; - QLValue value; - - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()) << "Value: " << value.ToString(); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(20000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_e", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(2000), + R"#( + {string:"row1",int64:11111,string:"row1_c",int64:10000,string:"row1_e"} + {string:"row2",int64:22222,null,int64:20000,string:"row2_e"} + )#", + HybridTime::FromMicros(2000)); // Scan at a later hybrid_time. - - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(5000))); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - // This row is exactly the same as in the previous case. TODO: deduplicate. - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - - // These two rows have different values compared to the previous case. - ASSERT_EQ(30000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_e_prime", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(5000), + R"#( + {string:"row1",int64:11111,string:"row1_c",int64:10000,string:"row1_e"} + {string:"row2",int64:22222,null,int64:30000,string:"row2_e_prime"} + )#", + HybridTime::FromMicros(4000)); } void DocRowwiseIteratorTest::TestDocRowwiseIteratorCallbackAPI() { @@ -1220,31 +1024,12 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorDeletedDocument() { SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(40); HT{ physical: 2000 }]) -> 20000 )#"); - const auto& projection = this->projection(); - - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2500))); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(20000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(5000), + R"#( + {string:"row2",int64:22222,null,int64:20000,null} + )#", + HybridTime::FromMicros(2500)); } void DocRowwiseIteratorTest::TestDocRowwiseIteratorWithRowDeletes() { @@ -1275,58 +1060,28 @@ SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 2800 }]) -> SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(40); HT{ physical: 2800 w: 1 }]) -> 20000 )#"); - const auto& projection = this->projection(); + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(5000), + R"#( + {string:"row1",int64:11111,null,null,string:"row1_e"} + {string:"row2",int64:22222,null,int64:20000,null} + )#", + HybridTime::FromMicros(2800)); +} - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - // ColumnId 30, 40 should be hidden whereas ColumnId 50 should be visible. - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(20000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_TRUE(value.IsNull()); - } -} - -void VerifyOldestRecordTime(IntentAwareIterator *iter, const DocKey &doc_key, - const SubDocKey &subkey, HybridTime min_hybrid_time, - HybridTime expected_oldest_record_time) { - iter->Seek(doc_key); - const KeyBytes subkey_bytes = subkey.EncodeWithoutHt(); - const Slice subkey_slice = subkey_bytes.AsSlice(); - Slice read_value; - HybridTime oldest_past_min_ht = - ASSERT_RESULT(iter->FindOldestRecord(subkey_slice, min_hybrid_time)); - LOG(INFO) << "iter->FindOldestRecord returned " << oldest_past_min_ht - << " for " << SubDocKey::DebugSliceToString(subkey_slice); - ASSERT_EQ(oldest_past_min_ht, expected_oldest_record_time); -} +void VerifyOldestRecordTime(IntentAwareIterator *iter, const DocKey &doc_key, + const SubDocKey &subkey, HybridTime min_hybrid_time, + HybridTime expected_oldest_record_time) { + iter->Seek(doc_key); + const KeyBytes subkey_bytes = subkey.EncodeWithoutHt(); + const Slice subkey_slice = subkey_bytes.AsSlice(); + Slice read_value; + HybridTime oldest_past_min_ht = + ASSERT_RESULT(iter->FindOldestRecord(subkey_slice, min_hybrid_time)); + LOG(INFO) << "iter->FindOldestRecord returned " << oldest_past_min_ht + << " for " << SubDocKey::DebugSliceToString(subkey_slice); + ASSERT_EQ(oldest_past_min_ht, expected_oldest_record_time); +} void VerifyOldestRecordTime(IntentAwareIterator *iter, const DocKey &doc_key, const SubDocKey &subkey, uint64_t min_hybrid_time, @@ -1487,8 +1242,8 @@ SubDocKey(DocKey([], ["row1", 11111]), [HT{ physical: 2500 }]) -> DEL SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(40); HT{ physical: 1000 }]) -> 10000 SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 2800 }]) -> "row1_e" )#"); - const auto& projection = this->projection(); + const auto& projection = this->projection(); { auto iter = ASSERT_RESULT(CreateIterator( projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), @@ -1536,37 +1291,13 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorIncompleteProjection() { Schema projection; ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"c", "d"}, &projection)); - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); - - // Now find next row. - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(20000, value.int64_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + doc_read_context().schema, ReadHybridTime::FromMicros(5000), + R"#( + {missing,missing,null,int64:10000,missing} + {missing,missing,null,int64:20000,missing} + )#", + HybridTime::FromMicros(1000), &projection); } void DocRowwiseIteratorTest::TestColocatedTableTombstone() { @@ -1667,28 +1398,12 @@ SubDocKey(DocKey([], ["row2", 22222]), [ColumnId(50); HT{ physical: 2800 w: 3 }] Schema projection; ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"c", "e"}, &projection)); - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, read_time)); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - // Ensure Idempotency. - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_e", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + doc_read_context().schema, read_time, + R"#( + {missing,missing,null,missing,string:"row2_e"} + )#", + HybridTime::FromMicros(2800), &projection); } void DocRowwiseIteratorTest::TestDocRowwiseIteratorValidColumnNotInProjection() { @@ -1731,42 +1446,22 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorValidColumnNotInProjection() Schema projection; ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"c", "d"}, &projection)); - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_c", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(20000, value.int64_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + doc_read_context().schema, ReadHybridTime::FromMicros(2800), + R"#( + {missing,missing,null,null,missing} + {missing,missing,string:"row2_c",int64:20000,missing} + )#", + HybridTime::FromMicros(2800), &projection); } void DocRowwiseIteratorTest::TestDocRowwiseIteratorKeyProjection() { auto dwb = MakeDocWriteBatch(); // Row 1 + ASSERT_OK(dwb.SetPrimitive( + DocPath(kEncodedDocKey1, KeyEntryValue::kLivenessColumn), + ValueRef(ValueEntryType::kNullLow))); ASSERT_OK(dwb.SetPrimitive( DocPath(kEncodedDocKey1, KeyEntryValue::MakeColumnId(40_ColId)), ValueRef(QLValue::PrimitiveInt64(10000)))); @@ -1777,32 +1472,19 @@ void DocRowwiseIteratorTest::TestDocRowwiseIteratorKeyProjection() { ASSERT_OK(WriteToRocksDB(dwb, HybridTime::FromMicros(1000))); ASSERT_DOCDB_DEBUG_DUMP_STR_EQ(R"#( -SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(40); HT{ physical: 1000 }]) -> 10000 -SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 1000 w: 1 }]) -> "row1_e" +SubDocKey(DocKey([], ["row1", 11111]), [SystemColumnId(0); HT{ physical: 1000 }]) -> null +SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(40); HT{ physical: 1000 w: 1 }]) -> 10000 +SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 1000 w: 2 }]) -> "row1_e" )#"); Schema projection; ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"a", "b"}, &projection, 2)); - - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2800))); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_EQ("row1", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_EQ(kIntKey1, value.int64_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + doc_read_context().schema, ReadHybridTime::FromMicros(2800), + R"#( + {string:"row1",int64:11111,missing,missing,missing} + )#", + HybridTime::FromMicros(1000), &projection); } void DocRowwiseIteratorTest::TestDocRowwiseIteratorResolveWriteIntents() { @@ -1937,119 +1619,37 @@ TXN REV 30303030-3030-3030-3030-303030303032 HT{ physical: 4000 w: 3 } -> \ SubDocKey(DocKey([], ["row2", 22222]), []) [kWeakRead, kWeakWrite] HT{ physical: 4000 w: 3 } )#"); - const auto& projection = this->projection(); const auto txn_context = TransactionOperationContext( TransactionId::GenerateRandom(), &txn_status_manager); LOG(INFO) << "=============================================== ReadTime-2000"; - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), txn_context, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(20000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_e", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(2000), + R"#( + {string:"row1",int64:11111,string:"row1_c",int64:10000,string:"row1_e"} + {string:"row2",int64:22222,null,int64:20000,string:"row2_e"} + )#", + HybridTime::FromMicros(2000), txn_context); // Scan at a later hybrid_time. LOG(INFO) << "=============================================== ReadTime-5000"; - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), txn_context, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(5000))); - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c_t1", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(40000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e_t1", value.string_value()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(42000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_e_prime", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(5000), + R"#( + {string:"row1",int64:11111,string:"row1_c_t1",int64:40000,string:"row1_e_t1"} + {string:"row2",int64:22222,null,int64:42000,string:"row2_e_prime"} + )#", + HybridTime::FromMicros(4000), txn_context); // Scan at a later hybrid_time. LOG(INFO) << "=============================================== ReadTime-6000"; - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), txn_context, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(6000))); - - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(42000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_e_t2", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(6000), + R"#( + {string:"row2",int64:22222,null,int64:42000,string:"row2_e_t2"} + )#", + HybridTime::FromMicros(6000), txn_context); } void DocRowwiseIteratorTest::TestIntentAwareIteratorSeek() { @@ -2186,42 +1786,19 @@ void DocRowwiseIteratorTest::TestScanWithinTheSameTxn() { LOG(INFO) << "Dump:\n" << DocDBDebugDumpToStr(); const auto txn_context = TransactionOperationContext(*txn, &txn_status_manager); - const auto& projection = this->projection(); + Schema projection; + ASSERT_OK(doc_read_context().schema.CreateProjectionByNames({"c", "d", "e"}, &projection)); auto iter = ASSERT_RESULT(CreateIterator( projection, doc_read_context(), txn_context, doc_db(), CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1000))); - QLTableRow row; - QLValue value; - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c_t1", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_c_t1", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); + ASSERT_STR_EQ_VERBOSE_TRIMMED( + ASSERT_RESULT(ConvertIteratorRowsToString(iter.get(), doc_read_context().schema)), + R"#( + {string:"row1",int64:11111,string:"row1_c_t1",null,null} + {string:"row2",int64:22222,string:"row2_c_t1",null,null} + )#"); // Empirically we require 3 seeks to perform this test. // If this number increased, then something got broken and should be fixed. @@ -2231,7 +1808,7 @@ void DocRowwiseIteratorTest::TestScanWithinTheSameTxn() { void DocRowwiseIteratorTest::TestLargeKeys() { constexpr size_t str_key_size = 0x100; - auto str_key = RandomString(str_key_size); + std::string str_key(str_key_size, 't'); KeyBytes kEncodedKey( DocKey(KeyEntryValues(str_key, kIntKey1)).Encode()); @@ -2249,32 +1826,14 @@ void DocRowwiseIteratorTest::TestLargeKeys() { DocDBDebugDumpToConsole(); - const auto& projection = this->projection(); - QLTableRow row; - QLValue value; - - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(2000), + Format( + R"#( + {string:"$0",int64:$1,string:"row1_c",int64:10000,string:"row1_e"} + )#", + str_key, kIntKey1), + HybridTime::FromMicros(1000)); } void DocRowwiseIteratorTest::TestPackedRow() { @@ -2282,82 +1841,66 @@ void DocRowwiseIteratorTest::TestPackedRow() { auto& schema_packing = ASSERT_RESULT( doc_read_context().schema_packing_storage.GetPacking(kVersion)).get(); - { - Slice row1_packed_row; - RowPacker packer( - kVersion, schema_packing, /* packed_size_limit= */ std::numeric_limits::max(), - /* value_control_fields= */ Slice()); - ASSERT_OK(packer.AddValue(30_ColId, QLValue::Primitive("row1_c"))); - ASSERT_OK(packer.AddValue(40_ColId, QLValue::PrimitiveInt64(10000))); - ASSERT_OK(packer.AddValue(50_ColId, QLValue::Primitive("row1_e"))); - row1_packed_row = ASSERT_RESULT(packer.Complete()); - LOG(INFO) << "Row1 Packed: " << row1_packed_row.ToDebugHexString(); - - ASSERT_OK(SetPrimitive( - DocPath(kEncodedDocKey1), - ValueControlFields(), - ValueRef(row1_packed_row), - HybridTime::FromMicros(1000))); - } + InsertPackedRow( + kVersion, schema_packing, kEncodedDocKey1, HybridTime::FromMicros(1000), + { + {30_ColId, QLValue::Primitive("row1_c")}, + {40_ColId, QLValue::PrimitiveInt64(10000)}, + {50_ColId, QLValue::Primitive("row1_e")}, + }); // Add row2 with missing columns. - { - Slice row2_packed_row; - RowPacker packer( - kVersion, schema_packing, /* packed_size_limit= */ std::numeric_limits::max(), - /* value_control_fields= */ Slice()); - ASSERT_OK(packer.AddValue(30_ColId, QLValue::Primitive("row2_c"))); - row2_packed_row = ASSERT_RESULT(packer.Complete()); - LOG(INFO) << "Row2 Packed: " << row2_packed_row.ToDebugHexString(); - - ASSERT_OK(SetPrimitive( - DocPath(kEncodedDocKey2), - ValueControlFields(), - ValueRef(row2_packed_row), - HybridTime::FromMicros(1000))); - } + InsertPackedRow( + kVersion, schema_packing, kEncodedDocKey2, HybridTime::FromMicros(1000), + { + {30_ColId, QLValue::Primitive("row2_c")}, + }); DocDBDebugDumpToConsole(); - const Schema &projection = this->projection(); - QLTableRow row; - QLValue value; - - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(2000))); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c", value.string_value()); + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(2000), + R"#( + {string:"row1",int64:11111,string:"row1_c",int64:10000,string:"row1_e"} + {string:"row2",int64:22222,string:"row2_c",null,null} + )#", + HybridTime::FromMicros(1000)); +} - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); +void DocRowwiseIteratorTest::TestDeleteMarkerWithPackedRow() { + constexpr int kVersion = 0; + auto& schema_packing = ASSERT_RESULT( + doc_read_context().schema_packing_storage.GetPacking(kVersion)).get(); - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); + InsertPackedRow( + kVersion, schema_packing, kEncodedDocKey1, HybridTime::FromMicros(1000), + { + {30_ColId, QLValue::Primitive("row1_c")}, + {40_ColId, QLValue::PrimitiveInt64(10000)}, + {50_ColId, QLValue::Primitive("row1_e")}, + }); - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); + DocDBDebugDumpToConsole(); - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row2_c", value.string_value()); + // Test delete marker with lower timestamp than packed row. + ASSERT_OK(DeleteSubDoc( + DocPath(kEncodedDocKey1), HybridTime::FromMicros(800))); - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_TRUE(value.IsNull()); + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(2000), + R"#( + {string:"row1",int64:11111,string:"row1_c",int64:10000,string:"row1_e"} + )#", + HybridTime::FromMicros(1000)); - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_TRUE(value.IsNull()); + // Delete document with higher timestamp than packed row. + ASSERT_OK(DeleteSubDoc( + DocPath(kEncodedDocKey1), HybridTime::FromMicros(1100))); - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(2000), + R"#()#", + HybridTime::FromMicros(1100)); } void DocRowwiseIteratorTest::TestDeletedDocumentUsingLivenessColumnDelete() { @@ -2404,64 +1947,64 @@ void DocRowwiseIteratorTest::TestDeletedDocumentUsingLivenessColumnDelete() { SubDocKey(DocKey([], ["row1", 11111]), [ColumnId(50); HT{ physical: 1000 }]) -> "row1_e" )#"); - const auto& projection = this->projection(); - QLTableRow row; - QLValue value; - - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1000), nullptr)); - - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_c", value.string_value()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); - - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); - - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(1000), + R"#( + {string:"row1",int64:11111,string:"row1_c",int64:10000,string:"row1_e"} + )#", + HybridTime::FromMicros(1000)); LOG(INFO) << "Validate one deleted column is removed"; - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1100), nullptr)); + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(1100), + R"#( + {string:"row1",int64:11111,null,int64:10000,string:"row1_e"} + )#", + HybridTime::FromMicros(1100)); - ASSERT_TRUE(ASSERT_RESULT(iter->HasNext())); - ASSERT_OK(iter->NextRow(&row)); - - ASSERT_OK(row.GetValue(projection.column_id(0), &value)); - ASSERT_TRUE(value.IsNull()); - - ASSERT_OK(row.GetValue(projection.column_id(1), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ(10000, value.int64_value()); + LOG(INFO) << "Validate that row is not visible when liveness column is tombstoned"; + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(1500), + "", + HybridTime::FromMicros(1500)); +} - ASSERT_OK(row.GetValue(projection.column_id(2), &value)); - ASSERT_FALSE(value.IsNull()); - ASSERT_EQ("row1_e", value.string_value()); +void DocRowwiseIteratorTest::TestUpdatePackedRow() { + constexpr int kVersion = 0; + auto& schema_packing = ASSERT_RESULT( + doc_read_context().schema_packing_storage.GetPacking(kVersion)).get(); - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + InsertPackedRow( + kVersion, schema_packing, kEncodedDocKey1, HybridTime::FromMicros(1000), + { + {30_ColId, QLValue::Primitive("row1_c")}, + {40_ColId, QLValue::PrimitiveInt64(10000)}, + {50_ColId, QLValue::Primitive("row1_e")}, + }); + + InsertPackedRow( + kVersion, schema_packing, kEncodedDocKey1, HybridTime::FromMicros(1500), + { + {30_ColId, QLValue::Primitive("row1_c_prime")}, + {40_ColId, QLValue::PrimitiveInt64(20000)}, + {50_ColId, QLValue::Primitive("row1_e_prime")}, + }); - LOG(INFO) << "Validate that row is not visible when liveness column is tombstoned"; - { - auto iter = ASSERT_RESULT(CreateIterator( - projection, doc_read_context(), kNonTransactionalOperationContext, doc_db(), - CoarseTimePoint::max() /* deadline */, ReadHybridTime::FromMicros(1500), nullptr)); + DocDBDebugDumpToConsole(); - ASSERT_FALSE(ASSERT_RESULT(iter->HasNext())); - } + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(1000), + R"#( + {string:"row1",int64:11111,string:"row1_c",int64:10000,string:"row1_e"} + )#", + HybridTime::FromMicros(1000)); + + CreateIteratorAndValidate( + ReadHybridTime::FromMicros(2000), + R"#( + {string:"row1",int64:11111,string:"row1_c_prime",int64:20000,string:"row1_e_prime"} + )#", + HybridTime::FromMicros(1500)); } TEST_F(DocRowwiseIteratorTest, ClusteredFilterTestRange) { @@ -2501,7 +2044,7 @@ TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorTest) { } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorTestCallbackAPI) { -TestDocRowwiseIteratorCallbackAPI(); + TestDocRowwiseIteratorCallbackAPI(); } TEST_F(DocRowwiseIteratorTest, DocRowwiseIteratorDeletedDocumentTest) { @@ -2564,6 +2107,14 @@ TEST_F(DocRowwiseIteratorTest, BasicPackedRowTest) { TestPackedRow(); } +TEST_F(DocRowwiseIteratorTest, DeleteMarkerWithPackedRow) { + TestDeleteMarkerWithPackedRow(); +} + +TEST_F(DocRowwiseIteratorTest, UpdatePackedRow) { + TestUpdatePackedRow(); +} + TEST_F(DocRowwiseIteratorTest, DeletedDocumentUsingLivenessColumnDeleteTest) { TestDeletedDocumentUsingLivenessColumnDelete(); } diff --git a/src/yb/docdb/ql_rowwise_iterator_interface.cc b/src/yb/docdb/ql_rowwise_iterator_interface.cc index 3fdcc2e4710b..99aea3d5b0f9 100644 --- a/src/yb/docdb/ql_rowwise_iterator_interface.cc +++ b/src/yb/docdb/ql_rowwise_iterator_interface.cc @@ -13,6 +13,8 @@ #include "yb/docdb/ql_rowwise_iterator_interface.h" +#include "yb/common/hybrid_time.h" + #include "yb/util/result.h" namespace yb { @@ -42,5 +44,9 @@ Status YQLRowwiseIteratorIf::Iterate(const YQLScanCallback& callback) { return STATUS(NotSupported, "This iterator does not support iterate with callback."); } +HybridTime YQLRowwiseIteratorIf::TEST_MaxSeenHt() { + return HybridTime::kInvalid; +} + } // namespace docdb } // namespace yb diff --git a/src/yb/docdb/ql_rowwise_iterator_interface.h b/src/yb/docdb/ql_rowwise_iterator_interface.h index 50a10483c049..5ccc2af1468c 100644 --- a/src/yb/docdb/ql_rowwise_iterator_interface.h +++ b/src/yb/docdb/ql_rowwise_iterator_interface.h @@ -51,6 +51,9 @@ class YQLRowwiseIteratorIf { // Otherwise returns invalid hybrid time. virtual HybridTime RestartReadHt() = 0; + // Returns max seen hybrid time. Only used by tests for validation. + virtual HybridTime TEST_MaxSeenHt(); + virtual std::string ToString() const = 0; // Could be subset of actual table schema. diff --git a/src/yb/docdb/scan_choices-test.cc b/src/yb/docdb/scan_choices-test.cc index 4ac05c7d8c4f..f2146bed805f 100644 --- a/src/yb/docdb/scan_choices-test.cc +++ b/src/yb/docdb/scan_choices-test.cc @@ -199,8 +199,7 @@ void ScanChoicesTest::InitializeScanChoicesInstance(const Schema &schema, PgsqlC const auto &upper_bound = spec.UpperBound(); EXPECT_OK(upper_bound); auto base_choices = - ScanChoices::Create(schema, spec, lower_bound.get(), upper_bound.get(), - 0 /* prefix_length */).release(); + ScanChoices::Create(schema, spec, lower_bound.get(), upper_bound.get()).release(); choices_ = std::unique_ptr(down_cast(base_choices)); } diff --git a/src/yb/docdb/scan_choices.cc b/src/yb/docdb/scan_choices.cc index 8d232a01cb16..a3b9a9c6fd47 100644 --- a/src/yb/docdb/scan_choices.cc +++ b/src/yb/docdb/scan_choices.cc @@ -771,11 +771,10 @@ Status HybridScanChoices::SeekToCurrentTarget(IntentAwareIteratorIf* db_iter) { ScanChoicesPtr ScanChoices::Create( const Schema& schema, const DocQLScanSpec& doc_spec, - const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key, - const size_t prefix_length) { + const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key) { if (doc_spec.range_options() || doc_spec.range_bounds()) { return std::make_unique( - schema, doc_spec, lower_doc_key, upper_doc_key, prefix_length); + schema, doc_spec, lower_doc_key, upper_doc_key, doc_spec.prefix_length()); } return nullptr; @@ -783,11 +782,10 @@ ScanChoicesPtr ScanChoices::Create( ScanChoicesPtr ScanChoices::Create( const Schema& schema, const DocPgsqlScanSpec& doc_spec, - const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key, - const size_t prefix_length) { + const KeyBytes& lower_doc_key, const KeyBytes& upper_doc_key) { if (doc_spec.range_options() || doc_spec.range_bounds()) { return std::make_unique( - schema, doc_spec, lower_doc_key, upper_doc_key, prefix_length); + schema, doc_spec, lower_doc_key, upper_doc_key, doc_spec.prefix_length()); } return nullptr; diff --git a/src/yb/docdb/scan_choices.h b/src/yb/docdb/scan_choices.h index c856d48298ff..4ec45fed104d 100644 --- a/src/yb/docdb/scan_choices.h +++ b/src/yb/docdb/scan_choices.h @@ -37,8 +37,6 @@ class ScanChoices { // Returns false if there are still target keys we need to scan, and true if we are done. virtual bool FinishedWithScanChoices() const { return finished_; } - virtual bool IsInitialPositionKnown() const { return false; } - // Go to the next scan target if any. virtual Status DoneWithCurrentTarget() = 0; @@ -60,11 +58,11 @@ class ScanChoices { static ScanChoicesPtr Create( const Schema& schema, const DocQLScanSpec& doc_spec, const KeyBytes& lower_doc_key, - const KeyBytes& upper_doc_key, const size_t prefix_length); + const KeyBytes& upper_doc_key); static ScanChoicesPtr Create( const Schema& schema, const DocPgsqlScanSpec& doc_spec, const KeyBytes& lower_doc_key, - const KeyBytes& upper_doc_key, const size_t prefix_length); + const KeyBytes& upper_doc_key); protected: const bool is_forward_scan_; From ffcf4791e1f073c49a749785dc505ab6f186375b Mon Sep 17 00:00:00 2001 From: Lingeshwar S Date: Sat, 4 Mar 2023 22:52:14 +0530 Subject: [PATCH 78/81] fix : Removed empty spaces Summary: Removed empty spaces Test Plan: Tested Manually Reviewers: asathyan Reviewed By: asathyan Subscribers: jenkins-bot, ui, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23332 --- .../form/fields/MasterPlacementField/MasterPlacementField.tsx | 2 +- .../universe-form/form/sections/cloud/CloudConfiguration.tsx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/fields/MasterPlacementField/MasterPlacementField.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/fields/MasterPlacementField/MasterPlacementField.tsx index ffa70d74e245..8aae00d2f295 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/fields/MasterPlacementField/MasterPlacementField.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/fields/MasterPlacementField/MasterPlacementField.tsx @@ -102,7 +102,7 @@ export const MasterPlacementField = ({ ) : ( - + <> )} ); diff --git a/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx b/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx index dcb93fb0fc29..395b187c94bd 100644 --- a/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx +++ b/managed/ui/src/redesign/features/universe/universe-form/form/sections/cloud/CloudConfiguration.tsx @@ -79,8 +79,8 @@ export const CloudConfiguration = ({ runtimeConfigs }: UniverseFormConfiguration - {isPrimary && isDedicatedNodesEnabled && ( - + {isDedicatedNodesEnabled && ( + Date: Sat, 25 Feb 2023 00:13:22 -0800 Subject: [PATCH 79/81] [#13535] xCluster: Add ability to pause and resume replication from the source cluster Summary: Fixes #13535 Added the pause_producer_xcluster_streams operation to yb-admin to pause/resume replication over cdc streams from the producer side. Test Plan: TwoDCTest.PausingAndResumingReplicationFromProducerSingleTable TwoDCTest.PausingAndResumingReplicationFromProducerMultiTable Reviewers: hsunder, jhe Reviewed By: hsunder, jhe Subscribers: ybase, bogdan Differential Revision: https://phabricator.dev.yugabyte.com/D22720 --- ent/src/yb/master/catalog_manager.h | 8 + ent/src/yb/master/catalog_manager_ent.cc | 1 + ent/src/yb/master/xrepl_catalog_manager.cc | 86 +++++++++ src/yb/cdc/CMakeLists.txt | 19 ++ src/yb/cdc/cdc_service.cc | 36 +++- src/yb/cdc/cdc_service.h | 12 ++ src/yb/cdc/xcluster_producer.proto | 24 +++ src/yb/integration-tests/xcluster-test.cc | 184 +++++++++++++++++++- src/yb/master/CMakeLists.txt | 3 +- src/yb/master/catalog_entity_info.h | 19 ++ src/yb/master/catalog_entity_info.proto | 6 + src/yb/master/catalog_loaders.cc | 23 +++ src/yb/master/catalog_loaders.h | 2 + src/yb/master/catalog_manager.cc | 47 +++++ src/yb/master/catalog_manager.h | 8 + src/yb/master/master_heartbeat.proto | 7 + src/yb/master/master_replication.proto | 11 ++ src/yb/master/master_replication_service.cc | 1 + src/yb/master/master_types.proto | 1 + src/yb/tools/yb-admin_cli.cc | 23 +++ src/yb/tools/yb-admin_client.cc | 24 +++ src/yb/tools/yb-admin_client.h | 3 + src/yb/tserver/CMakeLists.txt | 3 +- src/yb/tserver/heartbeater.cc | 12 ++ src/yb/tserver/tablet_server.cc | 16 ++ src/yb/tserver/tablet_server.h | 6 + src/yb/yql/pgwrapper/CMakeLists.txt | 1 + 27 files changed, 577 insertions(+), 9 deletions(-) create mode 100644 src/yb/cdc/xcluster_producer.proto diff --git a/ent/src/yb/master/catalog_manager.h b/ent/src/yb/master/catalog_manager.h index b6a2a2cc2329..e8168cc551e6 100644 --- a/ent/src/yb/master/catalog_manager.h +++ b/ent/src/yb/master/catalog_manager.h @@ -216,6 +216,11 @@ class CatalogManager : public yb::master::CatalogManager, SnapshotCoordinatorCon SetUniverseReplicationEnabledResponsePB* resp, rpc::RpcContext* rpc); + Status PauseResumeXClusterProducerStreams( + const PauseResumeXClusterProducerStreamsRequestPB* req, + PauseResumeXClusterProducerStreamsResponsePB* resp, + rpc::RpcContext* rpc); + // Get Universe Replication. Status GetUniverseReplication(const GetUniverseReplicationRequestPB* req, GetUniverseReplicationResponsePB* resp, @@ -290,6 +295,9 @@ class CatalogManager : public yb::master::CatalogManager, SnapshotCoordinatorCon std::shared_ptr session, const TabletId& tablet_id, const CDCStreamId& stream_id); + // Remove deleted xcluster stream IDs from producer stream Id map. + Status RemoveStreamFromXClusterProducerConfig(const std::vector& streams); + // Delete specified CDC streams metadata. Status CleanUpCDCStreamsMetadata(const std::vector>& streams); diff --git a/ent/src/yb/master/catalog_manager_ent.cc b/ent/src/yb/master/catalog_manager_ent.cc index c2f61400d8b2..033c69d10170 100644 --- a/ent/src/yb/master/catalog_manager_ent.cc +++ b/ent/src/yb/master/catalog_manager_ent.cc @@ -911,6 +911,7 @@ Status CatalogManager::ImportSnapshotPreprocess(const SnapshotInfoPB& snapshot_p case SysRowEntryType::DDL_LOG_ENTRY: FALLTHROUGH_INTENDED; case SysRowEntryType::SNAPSHOT_RESTORATION: FALLTHROUGH_INTENDED; case SysRowEntryType::XCLUSTER_SAFE_TIME: FALLTHROUGH_INTENDED; + case SysRowEntryType::XCLUSTER_CONFIG: FALLTHROUGH_INTENDED; case SysRowEntryType::UNKNOWN: FATAL_INVALID_ENUM_VALUE(SysRowEntryType, entry.type()); } diff --git a/ent/src/yb/master/xrepl_catalog_manager.cc b/ent/src/yb/master/xrepl_catalog_manager.cc index 0aceccbc4d1a..5763f7a7d7b2 100644 --- a/ent/src/yb/master/xrepl_catalog_manager.cc +++ b/ent/src/yb/master/xrepl_catalog_manager.cc @@ -1345,6 +1345,24 @@ Status CatalogManager::CleanUpCDCStreamsMetadata( return CleanUpCDCMetadataFromSystemCatalog(drop_stream_tablelist); } +Status CatalogManager::RemoveStreamFromXClusterProducerConfig( + const std::vector& streams) { + auto xcluster_config = XClusterConfig(); + auto l = xcluster_config->LockForWrite(); + auto* data = l.mutable_data(); + auto paused_producer_stream_ids = + data->pb.mutable_xcluster_producer_registry()->mutable_paused_producer_stream_ids(); + for (const auto& stream : streams) { + paused_producer_stream_ids->erase(stream->id()); + } + data->pb.set_version(data->pb.version() + 1); + RETURN_NOT_OK(CheckStatus( + sys_catalog_->Upsert(leader_ready_term(), xcluster_config.get()), + "updating xcluster config in sys-catalog")); + l.Commit(); + return Status::OK(); +} + Status CatalogManager::CleanUpDeletedCDCStreams( const std::vector>& streams) { auto ybclient = master_->cdc_state_client_initializer().client(); @@ -1454,6 +1472,9 @@ Status CatalogManager::CleanUpDeletedCDCStreams( } } + // Remove the stream ID from the cluster config CDC stream replication enabled/disabled map. + RETURN_NOT_OK(RemoveStreamFromXClusterProducerConfig(streams_to_delete)); + // The mutation will be aborted when 'l' exits the scope on early return. RETURN_NOT_OK(CheckStatus( sys_catalog_->Delete(leader_ready_term(), streams_to_delete), @@ -3334,6 +3355,64 @@ Status CatalogManager::SetUniverseReplicationEnabled( return Status::OK(); } +Status CatalogManager::PauseResumeXClusterProducerStreams( + const PauseResumeXClusterProducerStreamsRequestPB* req, + PauseResumeXClusterProducerStreamsResponsePB* resp, + rpc::RpcContext* rpc) { + LOG(INFO) << "Servicing PauseXCluster request from " << RequestorString(rpc) << "."; + SCHECK(req->has_is_paused(), InvalidArgument, "is_paused must be set in the request"); + bool paused = req->is_paused(); + string action = paused ? "Pausing" : "Resuming"; + if (req->stream_ids_size() == 0) { + LOG(INFO) << action << " replication for all XCluster streams."; + } + + auto xcluster_config = XClusterConfig(); + auto l = xcluster_config->LockForWrite(); + { + SharedLock lock(mutex_); + auto paused_producer_stream_ids = l.mutable_data() + ->pb.mutable_xcluster_producer_registry() + ->mutable_paused_producer_stream_ids(); + // If an empty stream_ids list is given, then pause replication for all streams. Presence in + // paused_producer_stream_ids indicates that a stream is paused. + if (req->stream_ids().empty()) { + if (paused) { + for (auto& stream : cdc_stream_map_) { + // If the stream id is not already in paused_producer_stream_ids, then insert it into + // paused_producer_stream_ids to pause it. + if (!paused_producer_stream_ids->count(stream.first)) { + paused_producer_stream_ids->insert({stream.first, true}); + } + } + } else { + // Clear paused_producer_stream_ids to resume replication for all streams. + paused_producer_stream_ids->clear(); + } + } else { + // Pause or resume the user-provided list of streams. + for (const auto& stream_id : req->stream_ids()) { + bool contains_stream_id = paused_producer_stream_ids->count(stream_id); + bool stream_exists = cdc_stream_map_.contains(stream_id); + SCHECK(stream_exists, NotFound, "XCluster Stream: $0 does not exists", stream_id); + if (paused && !contains_stream_id) { + // Insert stream id to pause replication on that stream. + paused_producer_stream_ids->insert({stream_id, true}); + } else if (!paused && contains_stream_id) { + // Erase stream id to resume replication on that stream. + paused_producer_stream_ids->erase(stream_id); + } + } + } + } + l.mutable_data()->pb.set_version(l.mutable_data()->pb.version() + 1); + RETURN_NOT_OK(CheckStatus( + sys_catalog_->Upsert(leader_ready_term(), xcluster_config.get()), + "updating xcluster config in sys-catalog")); + l.Commit(); + return Status::OK(); +} + Status CatalogManager::AlterUniverseReplication( const AlterUniverseReplicationRequestPB* req, AlterUniverseReplicationResponsePB* resp, @@ -5463,6 +5542,13 @@ Status CatalogManager::FillHeartbeatResponseCDC( *resp->mutable_consumer_registry() = consumer_registry; } } + if (req->has_xcluster_config_version() && + req->xcluster_config_version() < VERIFY_RESULT(GetXClusterConfigVersion())) { + auto xcluster_config = XClusterConfig(); + auto l = xcluster_config->LockForRead(); + resp->set_xcluster_config_version(l->pb.version()); + *resp->mutable_xcluster_producer_registry() = l->pb.xcluster_producer_registry(); + } return Status::OK(); } diff --git a/src/yb/cdc/CMakeLists.txt b/src/yb/cdc/CMakeLists.txt index 62fb021eec89..8ca24632abf5 100644 --- a/src/yb/cdc/CMakeLists.txt +++ b/src/yb/cdc/CMakeLists.txt @@ -69,6 +69,25 @@ ADD_YB_LIBRARY(cdc_service_proto DEPS yb_ql_common ${CDC_YRPC_LIBS} NONLINK_DEPS ${CDC_YRPC_TGTS}) +######################################### +# xcluster_producer_proto +######################################### + +YRPC_GENERATE( + XCLUSTER_PRODUCER_YRPC_SRCS XCLUSTER_PRODUCER_YRPC_HDRS XCLUSTER_PRODUCER_YRPC_TGTS + SOURCE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../.. + BINARY_ROOT ${CMAKE_CURRENT_BINARY_DIR}/../.. + NO_SERVICE_MESSAGES_PROTO_FILES xcluster_producer.proto) + +set(XCLUSTER_PRODUCER_YRPC_LIBS + yrpc + protobuf) + +ADD_YB_LIBRARY(xcluster_producer_proto + SRCS ${XCLUSTER_PRODUCER_YRPC_SRCS} + DEPS ${XCLUSTER_PRODUCER_YRPC_LIBS} + NONLINK_DEPS ${XCLUSTER_PRODUCER_YRPC_TGTS}) + ######################################### # cdc_service ######################################### diff --git a/src/yb/cdc/cdc_service.cc b/src/yb/cdc/cdc_service.cc index fdaae9d2f27f..28a0ae195d46 100644 --- a/src/yb/cdc/cdc_service.cc +++ b/src/yb/cdc/cdc_service.cc @@ -1479,6 +1479,11 @@ Result CDCServiceImpl::TEST_GetTabletInfoFromCache( return impl_->TEST_GetTabletInfoFromCache(producer_tablet); } +bool CDCServiceImpl::IsReplicationPausedForStream(const std::string& stream_id) const { + SharedLock l(mutex_); + return paused_xcluster_producer_streams_.contains(stream_id); +} + void CDCServiceImpl::GetChanges( const GetChangesRequestPB* req, GetChangesResponsePB* resp, RpcContext context) { RPC_CHECK_AND_RETURN_ERROR( @@ -1602,8 +1607,14 @@ void CDCServiceImpl::GetChanges( } } - if (PREDICT_FALSE(FLAGS_TEST_block_get_changes)) { - // Early exit for testing purpose. + bool is_replication_paused_for_stream = IsReplicationPausedForStream(req->stream_id()); + if (is_replication_paused_for_stream || PREDICT_FALSE(FLAGS_TEST_block_get_changes)) { + if (is_replication_paused_for_stream && VLOG_IS_ON(1)) { + YB_LOG_EVERY_N_SECS(INFO, 300) + << "Replication is paused from the producer for stream: " << req->stream_id(); + } + // Returning success to slow down polling on the consumer side while replication is paused or + // early exit for testing purpose. from_op_id.ToPB(resp->mutable_checkpoint()->mutable_op_id()); context.RespondSuccess(); return; @@ -2068,6 +2079,27 @@ bool CDCServiceImpl::CDCEnabled() { return cdc_enabled_.load(std::memory_order_a void CDCServiceImpl::SetCDCServiceEnabled() { cdc_enabled_.store(true, std::memory_order_release); } +void CDCServiceImpl::SetPausedXClusterProducerStreams( + const ::google::protobuf::Map& paused_producer_stream_ids, + uint32_t xcluster_config_version) { + std::lock_guard l(mutex_); + if (xcluster_config_version_ < xcluster_config_version) { + paused_xcluster_producer_streams_.clear(); + for (const auto& stream_id : paused_producer_stream_ids) { + paused_xcluster_producer_streams_.insert(stream_id.first); + } + xcluster_config_version_ = xcluster_config_version; + const auto list_str = JoinStrings(paused_xcluster_producer_streams_, ","); + LOG(INFO) << "Updating xCluster paused producer streams: " << list_str + << " Config version: " << xcluster_config_version_; + } +} + +uint32_t CDCServiceImpl::GetXClusterConfigVersion() const { + SharedLock l(mutex_); + return xcluster_config_version_; +} + Result> CDCServiceImpl::GetCdcStateTable() { bool use_cache = GetAtomicFlag(&FLAGS_enable_cdc_state_table_caching); { diff --git a/src/yb/cdc/cdc_service.h b/src/yb/cdc/cdc_service.h index b2174dae8122..cb67fc301de2 100644 --- a/src/yb/cdc/cdc_service.h +++ b/src/yb/cdc/cdc_service.h @@ -114,6 +114,7 @@ class CDCServiceImpl : public CDCServiceIf { const ListTabletsRequestPB* req, ListTabletsResponsePB* resp, rpc::RpcContext rpc) override; void GetChanges( const GetChangesRequestPB* req, GetChangesResponsePB* resp, rpc::RpcContext rpc) override; + bool IsReplicationPausedForStream(const std::string& stream_id) const EXCLUDES(mutex_); void GetCheckpoint( const GetCheckpointRequestPB* req, GetCheckpointResponsePB* resp, @@ -202,6 +203,13 @@ class CDCServiceImpl : public CDCServiceIf { static bool IsCDCSDKSnapshotBootstrapRequest(const CDCSDKCheckpointPB& req_checkpoint); + // Sets paused producer XCluster streams. + void SetPausedXClusterProducerStreams( + const ::google::protobuf::Map& paused_producer_stream_ids, + uint32_t xcluster_config_version); + + uint32_t GetXClusterConfigVersion() const; + private: FRIEND_TEST(CDCServiceTest, TestMetricsOnDeletedReplication); FRIEND_TEST(CDCServiceTestMultipleServersOneTablet, TestMetricsAfterServerFailure); @@ -465,6 +473,10 @@ class CDCServiceImpl : public CDCServiceIf { // True when the server is a producer of a valid replication stream. std::atomic cdc_enabled_{false}; + + std::unordered_set paused_xcluster_producer_streams_ GUARDED_BY(mutex_); + + uint32_t xcluster_config_version_ GUARDED_BY(mutex_) = 0; }; } // namespace cdc diff --git a/src/yb/cdc/xcluster_producer.proto b/src/yb/cdc/xcluster_producer.proto new file mode 100644 index 000000000000..659737bf715c --- /dev/null +++ b/src/yb/cdc/xcluster_producer.proto @@ -0,0 +1,24 @@ +// Copyright (c) YugaByte, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. + +syntax = "proto3"; + +package yb.xcluster; + +option java_package = "org.yb.xcluster"; + +message ProducerRegistryPB { + // Map from producer cdc stream ids to whether replication is paused on that stream. + // The paused_producer_stream_ids map is currently being used as a set so the second + // value will always be set to true. + map paused_producer_stream_ids = 1; +} diff --git a/src/yb/integration-tests/xcluster-test.cc b/src/yb/integration-tests/xcluster-test.cc index b2286c4fe146..aae20bcb3300 100644 --- a/src/yb/integration-tests/xcluster-test.cc +++ b/src/yb/integration-tests/xcluster-test.cc @@ -261,11 +261,20 @@ class XClusterTest : public XClusterTestBase, Status VerifyWrittenRecords(const YBTableName& producer_table, const YBTableName& consumer_table, int timeout_secs = kRpcTimeout) { - return LoggedWaitFor([this, producer_table, consumer_table]() -> Result { - auto producer_results = ScanTableToStrings(producer_table, producer_client()); - auto consumer_results = ScanTableToStrings(consumer_table, consumer_client()); - return producer_results == consumer_results; - }, MonoDelta::FromSeconds(timeout_secs), "Verify written records"); + std::vector producer_results, consumer_results; + const auto s = LoggedWaitFor( + [this, producer_table, consumer_table, &producer_results, + &consumer_results]() -> Result { + producer_results = ScanTableToStrings(producer_table, producer_client()); + consumer_results = ScanTableToStrings(consumer_table, consumer_client()); + return producer_results == consumer_results; + }, + MonoDelta::FromSeconds(timeout_secs), "Verify written records"); + if (!s.ok()) { + LOG(ERROR) << "Producer records: " << JoinStrings(producer_results, ",") + << ";Consumer records: " << JoinStrings(consumer_results, ","); + } + return s; } Status VerifyNumRecords(const YBTableName& table, YBClient* client, size_t expected_size) { @@ -520,6 +529,65 @@ class XClusterTest : public XClusterTestBase, return Status::OK(); } + Status PauseResumeXClusterProducerStreams( + const std::vector& stream_ids, bool is_paused) { + master::PauseResumeXClusterProducerStreamsRequestPB req; + master::PauseResumeXClusterProducerStreamsResponsePB resp; + + auto master_proxy = std::make_shared( + &producer_client()->proxy_cache(), + VERIFY_RESULT(producer_cluster()->GetLeaderMiniMaster())->bound_rpc_addr()); + + rpc::RpcController rpc; + rpc.set_timeout(MonoDelta::FromSeconds(kRpcTimeout)); + for (const auto& stream_id : stream_ids) { + req.add_stream_ids(stream_id); + } + req.set_is_paused(is_paused); + RETURN_NOT_OK(master_proxy->PauseResumeXClusterProducerStreams(req, &resp, &rpc)); + SCHECK( + !resp.has_error(), IllegalState, + Format( + "PauseResumeXClusterProducerStreams returned error: $0", resp.error().DebugString())); + return Status::OK(); + } + + void WriteWorkloadAndVerifyWrittenRows( + const std::shared_ptr& producer_table, + const std::shared_ptr& consumer_table, uint32_t start, uint32_t end, + bool replication_enabled = true, int timeout = kRpcTimeout) { + LOG(INFO) << "Writing records for table " << producer_table->name().ToString(); + WriteWorkload(start, end, producer_client(), producer_table->name()); + if (replication_enabled) { + ASSERT_OK(VerifyWrittenRecords(producer_table->name(), consumer_table->name(), timeout)); + } else { + ASSERT_NOK(VerifyWrittenRecords(producer_table->name(), consumer_table->name(), timeout)); + } + } + + // Empty stream_ids will pause all streams. + void SetPauseAndVerifyWrittenRows( + const std::vector& stream_ids, + const std::vector>& producer_tables, + const std::vector>& consumer_tables, uint32_t start, + uint32_t end, bool pause = true) { + ASSERT_OK(PauseResumeXClusterProducerStreams(stream_ids, pause)); + // Needs to sleep to wait for heartbeat to propogate. + SleepFor(3s * kTimeMultiplier); + + // If stream_ids is empty, then write and test replication on all streams, otherwise write and + // test on only those selected in stream_ids. + size_t size = stream_ids.size() ? stream_ids.size() : producer_tables.size(); + for (size_t i = 0; i < size; i++) { + const auto& producer_table = producer_tables[i]; + const auto& consumer_table = consumer_tables[i]; + // Reduce the timeout time when we don't expect replication to be successful. + int timeout = pause ? 10 : kRpcTimeout; + WriteWorkloadAndVerifyWrittenRows( + producer_table, consumer_table, start, end, !pause, timeout); + } + } + private: server::ClockPtr clock_{new server::HybridClock()}; @@ -3611,4 +3679,110 @@ TEST_P(XClusterTest, YB_DISABLE_TEST_IN_TSAN(TestPrematureLogGC)) { consumer_table->id(), stream_id, ReplicationErrorPb::REPLICATION_MISSING_OP_ID); } +TEST_P(XClusterTest, PausingAndResumingReplicationFromProducerSingleTable) { + constexpr int kNTabletsPerTable = 3; + constexpr int kNumTables = 1; + uint32_t kReplicationFactor = NonTsanVsTsan(3, 1); + std::vector tables_vector(kNumTables); + for (size_t i = 0; i < kNumTables; i++) { + tables_vector[i] = kNTabletsPerTable; + } + auto tables = ASSERT_RESULT(SetUpWithParams(tables_vector, tables_vector, kReplicationFactor)); + std::vector> producer_tables; + std::vector> consumer_tables; + + producer_tables.push_back(tables[0]); + consumer_tables.push_back(tables[1]); + + // Empty case. + ASSERT_OK(PauseResumeXClusterProducerStreams({}, true)); + ASSERT_OK(PauseResumeXClusterProducerStreams({}, false)); + + // Invalid ID case. + ASSERT_NOK(PauseResumeXClusterProducerStreams({"invalid_id"}, true)); + ASSERT_NOK(PauseResumeXClusterProducerStreams({"invalid_id"}, false)); + + ASSERT_OK(SetupUniverseReplication(producer_tables)); + master::IsSetupUniverseReplicationDoneResponsePB is_resp; + ASSERT_OK(WaitForSetupUniverseReplication( + consumer_cluster(), consumer_client(), kUniverseId, &is_resp)); + + std::vector stream_ids; + + WriteWorkloadAndVerifyWrittenRows(producer_tables[0], consumer_tables[0], 0, 10); + + // Test pausing all streams (by passing empty streams list) when there is only one stream. + SetPauseAndVerifyWrittenRows({}, producer_tables, consumer_tables, 10, 20); + + // Test resuming all streams (by passing empty streams list) when there is only one stream. + SetPauseAndVerifyWrittenRows({}, producer_tables, consumer_tables, 20, 30, false); + + // Test pausing stream by ID when there is only one stream by passing non-empty stream_ids. + auto stream_id = ASSERT_RESULT(GetCDCStreamID(producer_tables[0]->id())); + stream_ids.push_back(stream_id); + SetPauseAndVerifyWrittenRows(stream_ids, producer_tables, consumer_tables, 30, 40); + + // Test resuming stream by ID when there is only one stream by passing non-empty stream_ids. + SetPauseAndVerifyWrittenRows(stream_ids, producer_tables, consumer_tables, 40, 50, false); + + ASSERT_OK(DeleteUniverseReplication()); +} + +TEST_P(XClusterTest, PausingAndResumingReplicationFromProducerMultiTable) { + constexpr int kNTabletsPerTable = 3; + constexpr int kNumTables = 3; + uint32_t kReplicationFactor = NonTsanVsTsan(3, 1); + std::vector tables_vector(kNumTables); + for (size_t i = 0; i < kNumTables; i++) { + tables_vector[i] = kNTabletsPerTable; + } + auto tables = ASSERT_RESULT(SetUpWithParams(tables_vector, tables_vector, kReplicationFactor)); + + std::vector> producer_tables; + std::vector> consumer_tables; + // Testing on multiple tables. + producer_tables.reserve(tables.size() / 2); + consumer_tables.reserve(tables.size() / 2); + for (size_t i = 0; i < tables.size(); i++) { + if (i % 2 == 0) { + producer_tables.push_back(tables[i]); + } else { + consumer_tables.push_back(tables[i]); + } + } + + std::vector stream_ids; + + ASSERT_OK(SetupUniverseReplication(producer_tables)); + master::IsSetupUniverseReplicationDoneResponsePB is_resp; + ASSERT_OK(WaitForSetupUniverseReplication( + consumer_cluster(), consumer_client(), kUniverseId, &is_resp)); + + SetPauseAndVerifyWrittenRows({}, producer_tables, consumer_tables, 0, 10, false); + // Test pausing all streams (by passing empty streams list) when there are multiple streams. + SetPauseAndVerifyWrittenRows({}, producer_tables, consumer_tables, 10, 20); + // Test resuming all streams (by passing empty streams list) when there are multiple streams. + SetPauseAndVerifyWrittenRows({}, producer_tables, consumer_tables, 20, 30, false); + // Add the stream IDs of the first two tables to test pausing and resuming just those ones. + for (size_t i = 0; i < producer_tables.size() - 1; ++i) { + auto stream_id = (string)ASSERT_RESULT(GetCDCStreamID(producer_tables[i]->id())); + stream_ids.push_back(stream_id); + } + // Test pausing replication on the first two tables by passing stream_ids containing their + // corresponding xcluster streams. + SetPauseAndVerifyWrittenRows(stream_ids, producer_tables, consumer_tables, 30, 40); + // Verify that the remaining streams are unpaused still. + for (size_t i = stream_ids.size(); i < producer_tables.size(); ++i) { + WriteWorkloadAndVerifyWrittenRows(producer_tables[i], consumer_tables[i], 30, 40); + } + // Test resuming replication on the first two tables by passing stream_ids containing their + // corresponding xcluster streams. + SetPauseAndVerifyWrittenRows(stream_ids, producer_tables, consumer_tables, 40, 50, false); + // Verify that the previously unpaused streams are still replicating. + for (size_t i = stream_ids.size(); i < producer_tables.size(); ++i) { + WriteWorkloadAndVerifyWrittenRows(producer_tables[i], consumer_tables[i], 40, 50); + } + ASSERT_OK(DeleteUniverseReplication()); +} + } // namespace yb diff --git a/src/yb/master/CMakeLists.txt b/src/yb/master/CMakeLists.txt index 986437623c20..9ec89f96ba04 100644 --- a/src/yb/master/CMakeLists.txt +++ b/src/yb/master/CMakeLists.txt @@ -54,7 +54,8 @@ set(MASTER_YRPC_LIBS rpc_header_proto tablet_proto tserver_proto - cdc_consumer_proto) + cdc_consumer_proto + xcluster_producer_proto) ADD_YB_LIBRARY(master_proto SRCS ${MASTER_YRPC_SRCS} DEPS ${MASTER_YRPC_LIBS} diff --git a/src/yb/master/catalog_entity_info.h b/src/yb/master/catalog_entity_info.h index dd5c5055c8b5..35891c808f8b 100644 --- a/src/yb/master/catalog_entity_info.h +++ b/src/yb/master/catalog_entity_info.h @@ -904,6 +904,25 @@ class ClusterConfigInfo : public MetadataCowWrapper const std::string fake_id_; }; +// This wraps around the proto containing xcluster cluster level config information. It will be used +// for CowObject managed access. +struct PersistentXClusterConfigInfo + : public Persistent {}; + +// This is the in memory representation of the xcluster config information serialized proto +// data, using metadata() for CowObject access. +class XClusterConfigInfo : public MetadataCowWrapper { + public: + XClusterConfigInfo() {} + ~XClusterConfigInfo() = default; + + virtual const std::string& id() const override { return fake_id_; } + + private: + // We do not use the ID field in the sys_catalog table. + const std::string fake_id_; +}; + struct PersistentRedisConfigInfo : public Persistent {}; diff --git a/src/yb/master/catalog_entity_info.proto b/src/yb/master/catalog_entity_info.proto index 52fc8ea408b6..931ceed350b7 100644 --- a/src/yb/master/catalog_entity_info.proto +++ b/src/yb/master/catalog_entity_info.proto @@ -17,6 +17,7 @@ package yb.master; option java_package = "org.yb.master"; import "yb/cdc/cdc_consumer.proto"; +import "yb/cdc/xcluster_producer.proto"; import "yb/common/common.proto"; import "yb/common/common_net.proto"; import "yb/common/common_types.proto"; @@ -349,6 +350,11 @@ message SysClusterConfigEntryPB { optional cdc.ConsumerRegistryPB consumer_registry = 6; } +message SysXClusterConfigEntryPB { + optional uint32 version = 1; + optional xcluster.ProducerRegistryPB xcluster_producer_registry = 2; +} + // The data part of a SysRowEntry in the sys.catalog table for Redis Config. message SysRedisConfigEntryPB { optional string key = 1; diff --git a/src/yb/master/catalog_loaders.cc b/src/yb/master/catalog_loaders.cc index 12db0d97cdfa..13bf5e540d34 100644 --- a/src/yb/master/catalog_loaders.cc +++ b/src/yb/master/catalog_loaders.cc @@ -520,6 +520,29 @@ Status ClusterConfigLoader::Visit( return Status::OK(); } +//////////////////////////////////////////////////////////// +// XCluster Config Loader +//////////////////////////////////////////////////////////// + +Status XClusterConfigLoader::Visit( + const std::string& unused_id, const SysXClusterConfigEntryPB& metadata) { + // Debug confirm that there is no xcluster_config_ set. + DCHECK(!catalog_manager_->xcluster_config_) << "Already have config data!"; + + // Prepare the config object. + std::shared_ptr config = std::make_shared(); + { + auto l = config->LockForWrite(); + l.mutable_data()->pb.CopyFrom(metadata); + + // Update in memory state. + catalog_manager_->xcluster_config_ = config; + l.Commit(); + } + + return Status::OK(); +} + //////////////////////////////////////////////////////////// // Redis Config Loader //////////////////////////////////////////////////////////// diff --git a/src/yb/master/catalog_loaders.h b/src/yb/master/catalog_loaders.h index cb498e122b77..7375b63d01f7 100644 --- a/src/yb/master/catalog_loaders.h +++ b/src/yb/master/catalog_loaders.h @@ -98,6 +98,8 @@ DECLARE_LOADER_CLASS(Tablet, TabletId, SysTabletsEntryPB, catalo DECLARE_LOADER_CLASS(Namespace, NamespaceId, SysNamespaceEntryPB, catalog_manager_->mutex_); DECLARE_LOADER_CLASS(UDType, UDTypeId, SysUDTypeEntryPB, catalog_manager_->mutex_); DECLARE_LOADER_CLASS(ClusterConfig, std::string, SysClusterConfigEntryPB, catalog_manager_->mutex_); +DECLARE_LOADER_CLASS( + XClusterConfig, std::string, SysXClusterConfigEntryPB, catalog_manager_->mutex_); DECLARE_LOADER_CLASS(RedisConfig, std::string, SysRedisConfigEntryPB, catalog_manager_->mutex_); DECLARE_LOADER_CLASS(Role, RoleName, SysRoleEntryPB, catalog_manager_->permissions_manager()->mutex()); diff --git a/src/yb/master/catalog_manager.cc b/src/yb/master/catalog_manager.cc index afa10f38317e..72c51d6ddbd9 100644 --- a/src/yb/master/catalog_manager.cc +++ b/src/yb/master/catalog_manager.cc @@ -1213,6 +1213,10 @@ Status CatalogManager::VisitSysCatalog(int64_t term) { cluster_config_.reset(); RETURN_NOT_OK(PrepareDefaultClusterConfig(term)); + LOG_WITH_PREFIX(INFO) << "Re-initializing xcluster config"; + xcluster_config_.reset(); + RETURN_NOT_OK(PrepareDefaultXClusterConfig(term)); + LOG_WITH_PREFIX(INFO) << "Restoring snapshot completed, considering initdb finished"; RETURN_NOT_OK(InitDbFinished(Status::OK(), term)); RETURN_NOT_OK(RunLoaders(term)); @@ -1239,6 +1243,8 @@ Status CatalogManager::VisitSysCatalog(int64_t term) { // empty version 0. RETURN_NOT_OK(PrepareDefaultClusterConfig(term)); + RETURN_NOT_OK(PrepareDefaultXClusterConfig(term)); + permissions_manager_->BuildRecursiveRoles(); if (FLAGS_enable_ysql) { @@ -1319,6 +1325,9 @@ Status CatalogManager::RunLoaders(int64_t term) { // Clear the current cluster config. cluster_config_.reset(); + // Clear the current xcluster config. + xcluster_config_.reset(); + // Clear redis config mapping. redis_config_map_.clear(); @@ -1363,6 +1372,7 @@ Status CatalogManager::RunLoaders(int64_t term) { RETURN_NOT_OK(Load("cluster configuration", &state, term)); RETURN_NOT_OK(Load("Redis config", &state, term)); RETURN_NOT_OK(Load("XCluster safe time", &state, term)); + RETURN_NOT_OK(Load("xcluster configuration", &state, term)); if (!transaction_tables_config_) { RETURN_NOT_OK(InitializeTransactionTablesConfig(term)); @@ -1447,6 +1457,31 @@ Status CatalogManager::PrepareDefaultClusterConfig(int64_t term) { return Status::OK(); } +Status CatalogManager::PrepareDefaultXClusterConfig(int64_t term) { + if (xcluster_config_) { + LOG_WITH_PREFIX(INFO) + << "Cluster configuration has already been set up, skipping re-initialization."; + return Status::OK(); + } + + // Create default. + SysXClusterConfigEntryPB config; + config.set_version(0); + + // Create in memory object. + xcluster_config_ = std::make_shared(); + + // Prepare write. + auto l = xcluster_config_->LockForWrite(); + l.mutable_data()->pb = std::move(config); + + // Write to sys_catalog and in memory. + RETURN_NOT_OK(sys_catalog_->Upsert(term, xcluster_config_.get())); + l.Commit(); + + return Status::OK(); +} + std::vector CatalogManager::GetMasterAddresses() { std::vector result; consensus::ConsensusStatePB state; @@ -5031,6 +5066,7 @@ std::string CatalogManager::GenerateIdUnlocked( case SysRowEntryType::DDL_LOG_ENTRY: FALLTHROUGH_INTENDED; case SysRowEntryType::SNAPSHOT_RESTORATION: FALLTHROUGH_INTENDED; case SysRowEntryType::XCLUSTER_SAFE_TIME: FALLTHROUGH_INTENDED; + case SysRowEntryType::XCLUSTER_CONFIG: FALLTHROUGH_INTENDED; case SysRowEntryType::UNKNOWN: LOG(DFATAL) << "Invalid id type: " << *entity_type; return id; @@ -12041,6 +12077,13 @@ Status CatalogManager::SetClusterConfig( return Status::OK(); } +Result CatalogManager::GetXClusterConfigVersion() const { + auto xcluster_config = XClusterConfig(); + SCHECK(xcluster_config, IllegalState, "XCluster config is not initialized"); + auto l = xcluster_config->LockForRead(); + return l->pb.version(); +} + Status CatalogManager::ValidateReplicationInfo( const ValidateReplicationInfoRequestPB* req, ValidateReplicationInfoResponsePB* resp) { TSDescriptorVector all_ts_descs; @@ -12737,6 +12780,10 @@ std::shared_ptr CatalogManager::ClusterConfig() const { return cluster_config_; } +std::shared_ptr CatalogManager::XClusterConfig() const { + return xcluster_config_; +} + Status CatalogManager::TryRemoveFromTablegroup(const TableId& table_id) { LockGuard lock(mutex_); auto tablegroup = tablegroup_manager_->FindByTable(table_id); diff --git a/src/yb/master/catalog_manager.h b/src/yb/master/catalog_manager.h index 86bc6837f0b3..b6a643b4d750 100644 --- a/src/yb/master/catalog_manager.h +++ b/src/yb/master/catalog_manager.h @@ -795,6 +795,7 @@ class CatalogManager : public tserver::TabletPeerLookupIf, const ChangeMasterClusterConfigRequestPB* req, ChangeMasterClusterConfigResponsePB* resp) override; + Result GetXClusterConfigVersion() const; // Validator for placement information with respect to cluster configuration Status ValidateReplicationInfo( @@ -1070,6 +1071,7 @@ class CatalogManager : public tserver::TabletPeerLookupIf, friend class MultiStageAlterTable; friend class BackfillTable; friend class BackfillTablet; + friend class XClusterConfigLoader; FRIEND_TEST(SysCatalogTest, TestCatalogManagerTasksTracker); FRIEND_TEST(SysCatalogTest, TestPrepareDefaultClusterConfig); @@ -1121,6 +1123,8 @@ class CatalogManager : public tserver::TabletPeerLookupIf, // Sets the version field of the SysClusterConfigEntryPB to 0. Status PrepareDefaultClusterConfig(int64_t term) REQUIRES(mutex_); + Status PrepareDefaultXClusterConfig(int64_t term) REQUIRES(mutex_); + // Sets up various system configs. Status PrepareDefaultSysConfig(int64_t term) REQUIRES(mutex_); @@ -1651,6 +1655,8 @@ class CatalogManager : public tserver::TabletPeerLookupIf, std::shared_ptr ClusterConfig() const; + std::shared_ptr XClusterConfig() const; + Result GetGlobalTransactionStatusTable(); Result IsCreateTableDone(const TableInfoPtr& table); @@ -1732,6 +1738,8 @@ class CatalogManager : public tserver::TabletPeerLookupIf, // depends on this leader lock. std::shared_ptr cluster_config_ = nullptr; // No GUARD, only write on load. + std::shared_ptr xcluster_config_; // No GUARD, only write on load. + // YSQL Catalog information. scoped_refptr ysql_catalog_config_ = nullptr; // No GUARD, only write on Load. diff --git a/src/yb/master/master_heartbeat.proto b/src/yb/master/master_heartbeat.proto index c7de0fb5a567..e3e3c5c128f2 100644 --- a/src/yb/master/master_heartbeat.proto +++ b/src/yb/master/master_heartbeat.proto @@ -17,6 +17,7 @@ package yb.master; option java_package = "org.yb.master"; import "yb/cdc/cdc_consumer.proto"; +import "yb/cdc/xcluster_producer.proto"; import "yb/common/wire_protocol.proto"; import "yb/consensus/metadata.proto"; import "yb/encryption/encryption.proto"; @@ -206,6 +207,8 @@ message TSHeartbeatRequestPB { repeated TabletReplicationStatusPB replication_state = 16; optional uint32 auto_flags_config_version = 17; + + optional uint32 xcluster_config_version = 18; } message TSHeartbeatResponsePB { @@ -271,6 +274,10 @@ message TSHeartbeatResponsePB { map xcluster_namespace_to_safe_time = 21; optional AutoFlagsConfigPB auto_flags_config = 22; + + optional xcluster.ProducerRegistryPB xcluster_producer_registry = 23; + + optional uint32 xcluster_config_version = 24; } service MasterHeartbeat { diff --git a/src/yb/master/master_replication.proto b/src/yb/master/master_replication.proto index afd3a468f8ca..e3ce67dde634 100644 --- a/src/yb/master/master_replication.proto +++ b/src/yb/master/master_replication.proto @@ -255,6 +255,15 @@ message SetUniverseReplicationEnabledResponsePB { optional MasterErrorPB error = 1; } +message PauseResumeXClusterProducerStreamsRequestPB { + repeated string stream_ids = 1; + optional bool is_paused = 2; +} + +message PauseResumeXClusterProducerStreamsResponsePB { + optional MasterErrorPB error = 1; +} + message GetUniverseReplicationRequestPB { optional string producer_id = 1; } @@ -434,6 +443,8 @@ service MasterReplication { returns (AlterUniverseReplicationResponsePB); rpc SetUniverseReplicationEnabled(SetUniverseReplicationEnabledRequestPB) returns (SetUniverseReplicationEnabledResponsePB); + rpc PauseResumeXClusterProducerStreams(PauseResumeXClusterProducerStreamsRequestPB) + returns (PauseResumeXClusterProducerStreamsResponsePB); rpc GetUniverseReplication(GetUniverseReplicationRequestPB) returns (GetUniverseReplicationResponsePB); rpc IsSetupUniverseReplicationDone(IsSetupUniverseReplicationDoneRequestPB) diff --git a/src/yb/master/master_replication_service.cc b/src/yb/master/master_replication_service.cc index 6984257bb8b8..a7264d9d2700 100644 --- a/src/yb/master/master_replication_service.cc +++ b/src/yb/master/master_replication_service.cc @@ -45,6 +45,7 @@ class MasterReplicationServiceImpl : public MasterServiceBase, public MasterRepl (UpdateConsumerOnProducerMetadata) (ListCDCStreams) (SetUniverseReplicationEnabled) + (PauseResumeXClusterProducerStreams) (SetupUniverseReplication) (UpdateCDCStream) (GetCDCDBStreamInfo) diff --git a/src/yb/master/master_types.proto b/src/yb/master/master_types.proto index 56526df96623..af860405d436 100644 --- a/src/yb/master/master_types.proto +++ b/src/yb/master/master_types.proto @@ -44,6 +44,7 @@ enum SysRowEntryType { DDL_LOG_ENTRY = 13; SNAPSHOT_RESTORATION = 14; XCLUSTER_SAFE_TIME = 15; + XCLUSTER_CONFIG = 16; } // Master specific errors use this protobuf. diff --git a/src/yb/tools/yb-admin_cli.cc b/src/yb/tools/yb-admin_cli.cc index 5f1616397cf0..f91ad94f8018 100644 --- a/src/yb/tools/yb-admin_cli.cc +++ b/src/yb/tools/yb-admin_cli.cc @@ -1861,6 +1861,29 @@ void ClusterAdminCli::RegisterCommandHandlers(ClusterAdminClient* client) { return Status::OK(); }); + Register( + "pause_producer_xcluster_streams", " (|all) [resume]", + [client](const CLIArguments& args) -> Status { + if (args.size() > 2 || args.size() < 1) { + return ClusterAdminCli::kInvalidArguments; + } + bool is_paused = true; + vector stream_ids; + if (!boost::iequals(args[0], "all")) { + boost::split(stream_ids, args[0], boost::is_any_of(",")); + } + if (args.size() == 2) { + if (!boost::iequals(args[args.size() - 1], "resume")) { + return ClusterAdminCli::kInvalidArguments; + } + is_paused = false; + } + RETURN_NOT_OK_PREPEND( + client->PauseResumeXClusterProducerStreams(stream_ids, is_paused), + Substitute("Unable to $0 replication", is_paused ? "pause" : "resume")); + return Status::OK(); + }); + Register( "bootstrap_cdc_producer", " ", [client](const CLIArguments& args) -> Status { diff --git a/src/yb/tools/yb-admin_client.cc b/src/yb/tools/yb-admin_client.cc index a764a0510039..265d3a296a54 100644 --- a/src/yb/tools/yb-admin_client.cc +++ b/src/yb/tools/yb-admin_client.cc @@ -3858,6 +3858,30 @@ Status ClusterAdminClient::SetUniverseReplicationEnabled( return Status::OK(); } +Status ClusterAdminClient::PauseResumeXClusterProducerStreams( + const std::vector& stream_ids, bool is_paused) { + master::PauseResumeXClusterProducerStreamsRequestPB req; + master::PauseResumeXClusterProducerStreamsResponsePB resp; + for (const auto& stream_id : stream_ids) { + req.add_stream_ids(stream_id); + } + req.set_is_paused(is_paused); + const auto toggle = (is_paused ? "paus" : "resum"); + + RpcController rpc; + rpc.set_timeout(timeout_); + RETURN_NOT_OK(master_replication_proxy_->PauseResumeXClusterProducerStreams(req, &resp, &rpc)); + + if (resp.has_error()) { + cout << "Error " << toggle << "ing " + << "replication: " << resp.error().status().message() << endl; + return StatusFromPB(resp.error().status()); + } + + cout << "Replication " << toggle << "ed successfully" << endl; + return Status::OK(); +} + Result ClusterAdminClient::GetFirstRpcAddressForTS() { RepeatedPtrField servers; RETURN_NOT_OK(ListTabletServers(&servers)); diff --git a/src/yb/tools/yb-admin_client.h b/src/yb/tools/yb-admin_client.h index ee98fc954494..18259d422d19 100644 --- a/src/yb/tools/yb-admin_client.h +++ b/src/yb/tools/yb-admin_client.h @@ -411,6 +411,9 @@ class ClusterAdminClient { Status SetUniverseReplicationEnabled(const std::string& producer_id, bool is_enabled); + Status PauseResumeXClusterProducerStreams( + const std::vector& stream_ids, bool is_paused); + Status BootstrapProducer(const std::vector& table_id); Status WaitForReplicationDrain(const std::vector& stream_ids, diff --git a/src/yb/tserver/CMakeLists.txt b/src/yb/tserver/CMakeLists.txt index e144d8d7affb..3a63897a71dd 100644 --- a/src/yb/tserver/CMakeLists.txt +++ b/src/yb/tserver/CMakeLists.txt @@ -245,7 +245,8 @@ set(TSERVER_DEPS yb_pggate_flags ysql_upgrade cdc - cdc_consumer_proto) + cdc_consumer_proto + xcluster_producer_proto) ADD_YB_LIBRARY(tserver SRCS ${TSERVER_SRCS} diff --git a/src/yb/tserver/heartbeater.cc b/src/yb/tserver/heartbeater.cc index 6dbfabdbd8bc..678aea60d192 100644 --- a/src/yb/tserver/heartbeater.cc +++ b/src/yb/tserver/heartbeater.cc @@ -431,6 +431,12 @@ Status Heartbeater::Thread::TryHeartbeat() { req.set_config_index(server_->GetCurrentMasterIndex()); req.set_cluster_config_version(server_->cluster_config_version()); + auto result = server_->XClusterConfigVersion(); + if (result.ok()) { + req.set_xcluster_config_version(*result); + } else if (!result.status().IsNotFound()) { + return result.status(); + } req.set_rtt_us(heartbeat_rtt_.ToMicroseconds()); if (server_->has_faulty_drive()) { req.set_faulty_drive(true); @@ -519,6 +525,12 @@ Status Heartbeater::Thread::TryHeartbeat() { RETURN_NOT_OK(server_->SetCDCServiceEnabled()); } + if (resp.has_xcluster_producer_registry() && resp.has_xcluster_config_version()) { + RETURN_NOT_OK(server_->SetPausedXClusterProducerStreams( + resp.xcluster_producer_registry().paused_producer_stream_ids(), + resp.xcluster_config_version())); + } + // At this point we know resp is a successful heartbeat response from the master so set it as // the last heartbeat response. This invalidates resp so we should use last_hb_response_ instead // below (hence using the nested scope for resp until here). diff --git a/src/yb/tserver/tablet_server.cc b/src/yb/tserver/tablet_server.cc index ece7846d78c7..7a0d379ad2d7 100644 --- a/src/yb/tserver/tablet_server.cc +++ b/src/yb/tserver/tablet_server.cc @@ -1025,6 +1025,22 @@ int32_t TabletServer::cluster_config_version() const { return xcluster_consumer_->cluster_config_version(); } +Result TabletServer::XClusterConfigVersion() const { + SCHECK(cdc_service_, NotFound, "CDC Service not found"); + return cdc_service_->GetXClusterConfigVersion(); +} + +Status TabletServer::SetPausedXClusterProducerStreams( + const ::google::protobuf::Map<::std::string, bool>& paused_producer_stream_ids, + uint32_t xcluster_config_version) { + SCHECK(cdc_service_, NotFound, "CDC Service not found"); + if (VERIFY_RESULT(XClusterConfigVersion()) < xcluster_config_version) { + cdc_service_->SetPausedXClusterProducerStreams( + paused_producer_stream_ids, xcluster_config_version); + } + return Status::OK(); +} + Status TabletServer::ReloadKeysAndCertificates() { if (!secure_context_) { return Status::OK(); diff --git a/src/yb/tserver/tablet_server.h b/src/yb/tserver/tablet_server.h index 4a46cb783edb..15f1969e44d7 100644 --- a/src/yb/tserver/tablet_server.h +++ b/src/yb/tserver/tablet_server.h @@ -240,6 +240,12 @@ class TabletServer : public DbServerBase, public TabletServerIf { // Currently only used by cdc. virtual int32_t cluster_config_version() const; + Result XClusterConfigVersion() const; + + Status SetPausedXClusterProducerStreams( + const ::google::protobuf::Map<::std::string, bool>& paused_producer_stream_ids, + uint32_t xcluster_config_version); + client::TransactionPool& TransactionPool() override; const std::shared_future& client_future() const override; diff --git a/src/yb/yql/pgwrapper/CMakeLists.txt b/src/yb/yql/pgwrapper/CMakeLists.txt index 305b123289c7..c1a8760f9587 100644 --- a/src/yb/yql/pgwrapper/CMakeLists.txt +++ b/src/yb/yql/pgwrapper/CMakeLists.txt @@ -38,6 +38,7 @@ add_dependencies( gen_src_yb_tserver_tserver_proto gen_src_yb_tserver_tserver_types_proto gen_src_yb_cdc_cdc_consumer_proto + gen_src_yb_cdc_xcluster_producer_proto gen_src_yb_encryption_encryption_proto gen_src_yb_master_master_heartbeat_proto gen_src_yb_master_master_types_proto From 33459fa8c8543def104c5af575440564d1044f24 Mon Sep 17 00:00:00 2001 From: kkannan Date: Mon, 6 Mar 2023 11:56:42 +0530 Subject: [PATCH 80/81] [PLAT-5370][UI] Add Custom Variable Support for Email Notification Template Summary: Added: Custom variable creation modal. All features are behind feature flag [[ https://www.figma.com/file/Jahyk9bTcrjTF9kUDhMqgf/Custom-Notification-Template?node-id=382%3A32101&t=HgVWhKzUGqOnC2iR-0 | Figma ]] [[ https://docs.google.com/spreadsheets/d/1pRnOEqqQrY-KejW8q48B4WlTEm-gzF1C684w5XkyjqQ/edit?usp=sharing | Code Review Checklist ]] Test Plan: tested manually {F34930} {F34931} {F34932} {F34933} {F34934} {F34935} {F34936} Reviewers: jmak, rmadhavan, lsangappa, asathyan Reviewed By: lsangappa, asathyan Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D23120 --- .../AlertConfiguration/AlertConfiguration.js | 24 +- managed/ui/src/redesign/assets/trashbin.svg | 3 + .../alerts/TemplateComposer/CommonStyles.ts | 47 +++ .../alerts/TemplateComposer/Composer.tsx | 43 +++ .../CustomVariableEditorModal.tsx | 281 +++++++++++++++ .../TemplateComposer/CustomVariables.tsx | 328 ++++++++++++++++++ .../TemplateComposer/CustomVariablesAPI.ts | 38 ++ .../TemplateComposer/ICustomVariables.ts | 24 ++ .../alerts/TemplateComposer/index.tsx | 10 + managed/ui/src/reducers/feature.js | 6 +- managed/ui/src/translations/en.json | 26 ++ 11 files changed, 827 insertions(+), 3 deletions(-) create mode 100644 managed/ui/src/redesign/assets/trashbin.svg create mode 100644 managed/ui/src/redesign/features/alerts/TemplateComposer/CommonStyles.ts create mode 100644 managed/ui/src/redesign/features/alerts/TemplateComposer/Composer.tsx create mode 100644 managed/ui/src/redesign/features/alerts/TemplateComposer/CustomVariableEditorModal.tsx create mode 100644 managed/ui/src/redesign/features/alerts/TemplateComposer/CustomVariables.tsx create mode 100644 managed/ui/src/redesign/features/alerts/TemplateComposer/CustomVariablesAPI.ts create mode 100644 managed/ui/src/redesign/features/alerts/TemplateComposer/ICustomVariables.ts create mode 100644 managed/ui/src/redesign/features/alerts/TemplateComposer/index.tsx diff --git a/managed/ui/src/components/alerts/AlertConfiguration/AlertConfiguration.js b/managed/ui/src/components/alerts/AlertConfiguration/AlertConfiguration.js index 1e41c08fee43..527d6aaeb153 100644 --- a/managed/ui/src/components/alerts/AlertConfiguration/AlertConfiguration.js +++ b/managed/ui/src/components/alerts/AlertConfiguration/AlertConfiguration.js @@ -5,7 +5,8 @@ // This file will hold all the alert configuration tabs along // with their respective components. -import React, { useEffect, useState } from 'react'; +import React, { Suspense, useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; import { Tab } from 'react-bootstrap'; import { isDisabled } from '../../../utils/LayoutUtils'; import { YBTabsPanel } from '../../panels'; @@ -17,6 +18,7 @@ import CreateAlert from './CreateAlert'; import { getPromiseState } from '../../../utils/PromiseUtils'; import { AlertDestinationChannels } from './AlertDestinationChannels'; import { MaintenanceWindow } from '../MaintenanceWindow'; +const Composer = React.lazy(() => import('../../../redesign/features/alerts/TemplateComposer/Composer')); export const AlertConfiguration = (props) => { const [listView, setListView] = useState(false); @@ -24,6 +26,8 @@ export const AlertConfiguration = (props) => { const [alertUniverseList, setAlertUniverseList] = useState([]); const [enablePlatformAlert, setPlatformAlert] = useState(false); const [alertDestinationListView, setAlertDestinationListView] = useState(false); + const featureFlags = useSelector((state) => state.featureFlags); + const { activeTab, apiToken, @@ -168,6 +172,24 @@ export const AlertConfiguration = (props) => { > + { + (featureFlags.test.enableCustomEmailTemplates || featureFlags.released.enableCustomEmailTemplates) && ( + + New Notification channel +     + } + unmountOnExit + > + Loading...}> + + + + ) + } + ); diff --git a/managed/ui/src/redesign/assets/trashbin.svg b/managed/ui/src/redesign/assets/trashbin.svg new file mode 100644 index 000000000000..26d9b451454c --- /dev/null +++ b/managed/ui/src/redesign/assets/trashbin.svg @@ -0,0 +1,3 @@ + + + diff --git a/managed/ui/src/redesign/features/alerts/TemplateComposer/CommonStyles.ts b/managed/ui/src/redesign/features/alerts/TemplateComposer/CommonStyles.ts new file mode 100644 index 000000000000..6c9abb9326fd --- /dev/null +++ b/managed/ui/src/redesign/features/alerts/TemplateComposer/CommonStyles.ts @@ -0,0 +1,47 @@ +/* + * Created on Thu Feb 16 2023 + * + * Copyright 2021 YugaByte, Inc. and Contributors + * Licensed under the Polyform Free Trial License 1.0.0 (the "License") + * You may not use this file except in compliance with the License. You may obtain a copy of the License at + * http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt + */ + +import { makeStyles } from '@material-ui/core'; + +/** + * default styles for Composer components + */ +export const useCommonStyles = makeStyles((theme) => ({ + defaultBorder: { + border: `1px solid ${theme.palette.ybacolors.ybGrayHover}` + }, + helpText: { + fontFamily: 'Inter', + fontWeight: 400, + lineHeight: `${theme.spacing(2)}px`, + color: '#67666C' + }, + formErrText: { + color: theme.palette.error[500], + margin: 0, + fontSize: '11.5px', + fontWeight: 400, + lineHeight: `${theme.spacing(2)}px` + }, + menuStyles: { + '& .MuiMenuItem-root': { + height: '50px', + minWidth: '190px' + }, + '& .MuiMenuItem-root svg': { + marginRight: theme.spacing(1.4), + color: theme.palette.orange[500], + height: '20px', + width: '20px' + } + }, + clickable: { + cursor: 'pointer' + } +})); diff --git a/managed/ui/src/redesign/features/alerts/TemplateComposer/Composer.tsx b/managed/ui/src/redesign/features/alerts/TemplateComposer/Composer.tsx new file mode 100644 index 000000000000..63e944a9d02d --- /dev/null +++ b/managed/ui/src/redesign/features/alerts/TemplateComposer/Composer.tsx @@ -0,0 +1,43 @@ +/* + * Created on Tue Feb 14 2023 + * + * Copyright 2021 YugaByte, Inc. and Contributors + * Licensed under the Polyform Free Trial License 1.0.0 (the "License") + * You may not use this file except in compliance with the License. You may obtain a copy of the License at + * http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt + */ + +import React from 'react'; +import { Grid, makeStyles } from '@material-ui/core'; +import CustomVariablesEditor from './CustomVariables'; + +type ComposerProps = {}; + +const useStyles = makeStyles(() => ({ + root: { + background: '#F6F6F6', + height: '750px', + boxShadow: '0 4px 10px rgba(0, 0, 0, 0.25)' + }, + editor: { + width: '100%', + height: '100%', + border: 'none' + } +})); + +const Composer = (props: ComposerProps) => { + const classes = useStyles(props); + return ( + + +