diff --git a/build-support/thirdparty_archives.yml b/build-support/thirdparty_archives.yml index bca1e6594407..2d202b7cc1ef 100644 --- a/build-support/thirdparty_archives.yml +++ b/build-support/thirdparty_archives.yml @@ -1,90 +1,101 @@ -sha: 31776c4936a67fe6f1fc218fb64a6a9909c77311 +sha: b6b07342fdfd4a65ee2608d75dd31e4b0ecc0737 archives: - os_type: almalinux8 architecture: x86_64 compiler_type: clang17 - tag: v20240620163555-31776c4936-almalinux8-x86_64-clang17 + tag: v20240713003527-b6b07342fd-almalinux8-x86_64-clang17 + + - os_type: almalinux8 + architecture: x86_64 + compiler_type: clang18 + tag: v20240713003521-b6b07342fd-almalinux8-x86_64-clang18 - os_type: almalinux8 architecture: x86_64 compiler_type: gcc11 - tag: v20240620163541-31776c4936-almalinux8-x86_64-gcc11 + tag: v20240713003520-b6b07342fd-almalinux8-x86_64-gcc11 - os_type: almalinux9 architecture: x86_64 compiler_type: clang17 - tag: v20240620163555-31776c4936-almalinux9-x86_64-clang17 + tag: v20240713003540-b6b07342fd-almalinux9-x86_64-clang17 - os_type: almalinux9 architecture: x86_64 compiler_type: gcc12 - tag: v20240620163623-31776c4936-almalinux9-x86_64-gcc12 + tag: v20240713003537-b6b07342fd-almalinux9-x86_64-gcc12 - - os_type: centos7 + - os_type: amzn2 architecture: aarch64 - compiler_type: clang16 - tag: v20240620163739-31776c4936-centos7-aarch64-clang16 + compiler_type: clang17 + tag: v20240713003725-b6b07342fd-amzn2-aarch64-clang17 - - os_type: centos7 + - os_type: amzn2 architecture: aarch64 - compiler_type: clang16 + compiler_type: clang17 lto_type: full - tag: v20240620163735-31776c4936-centos7-aarch64-clang16-full-lto + tag: v20240713003827-b6b07342fd-amzn2-aarch64-clang17-full-lto - - os_type: centos7 + - os_type: amzn2 architecture: aarch64 - compiler_type: clang17 - tag: v20240620163738-31776c4936-centos7-aarch64-clang17 + compiler_type: clang18 + tag: v20240713003831-b6b07342fd-amzn2-aarch64-clang18 - - os_type: centos7 + - os_type: amzn2 architecture: aarch64 - compiler_type: clang17 + compiler_type: clang18 lto_type: full - tag: v20240620163737-31776c4936-centos7-aarch64-clang17-full-lto + tag: v20240713003853-b6b07342fd-amzn2-aarch64-clang18-full-lto - - os_type: centos7 + - os_type: amzn2 architecture: x86_64 compiler_type: clang17 - tag: v20240620163551-31776c4936-centos7-x86_64-clang17 + tag: v20240713003538-b6b07342fd-amzn2-x86_64-clang17 - - os_type: centos7 + - os_type: amzn2 architecture: x86_64 compiler_type: clang17 lto_type: full - tag: v20240620163542-31776c4936-centos7-x86_64-clang17-full-lto + tag: v20240713003542-b6b07342fd-amzn2-x86_64-clang17-full-lto - - os_type: centos7 + - os_type: amzn2 architecture: x86_64 - compiler_type: gcc11 - tag: v20240620163544-31776c4936-centos7-x86_64-gcc11 + compiler_type: clang18 + tag: v20240713003544-b6b07342fd-amzn2-x86_64-clang18 + + - os_type: amzn2 + architecture: x86_64 + compiler_type: clang18 + lto_type: full + tag: v20240713003540-b6b07342fd-amzn2-x86_64-clang18-full-lto - os_type: macos architecture: arm64 compiler_type: clang - tag: v20240620173126-31776c4936-macos-arm64 + tag: v20240713011052-b6b07342fd-macos-arm64 - os_type: macos architecture: x86_64 compiler_type: clang - tag: v20240620163640-31776c4936-macos-x86_64 + tag: v20240713003540-b6b07342fd-macos-x86_64 - os_type: ubuntu20.04 architecture: x86_64 compiler_type: clang16 - tag: v20240620163547-31776c4936-ubuntu2004-x86_64-clang16 + tag: v20240713003520-b6b07342fd-ubuntu2004-x86_64-clang16 - os_type: ubuntu22.04 architecture: x86_64 compiler_type: clang17 - tag: v20240620163538-31776c4936-ubuntu2204-x86_64-clang17 + tag: v20240713003517-b6b07342fd-ubuntu2204-x86_64-clang17 - os_type: ubuntu22.04 architecture: x86_64 compiler_type: gcc11 - tag: v20240620163535-31776c4936-ubuntu2204-x86_64-gcc11 + tag: v20240713003527-b6b07342fd-ubuntu2204-x86_64-gcc11 - os_type: ubuntu23.04 architecture: x86_64 compiler_type: gcc13 - tag: v20240620163558-31776c4936-ubuntu2304-x86_64-gcc13 + tag: v20240713003516-b6b07342fd-ubuntu2304-x86_64-gcc13 diff --git a/docs/content/preview/architecture/docdb-replication/cdc-logical-replication.md b/docs/content/preview/architecture/docdb-replication/cdc-logical-replication.md new file mode 100644 index 000000000000..c894824bdd26 --- /dev/null +++ b/docs/content/preview/architecture/docdb-replication/cdc-logical-replication.md @@ -0,0 +1,82 @@ +--- +title: Change data capture using Logical Replication in YugabyteDB +headerTitle: CDC using Logical Replication +linkTitle: CDC using Logical Replication +description: Learn how YugabyteDB supports asynchronous replication of data changes (inserts, updates, and deletes) to external databases or applications. +headContent: Asynchronous replication of data changes (inserts, updates, and deletes) to external databases or applications +badges: ea +menu: + preview: + parent: architecture-docdb-replication + identifier: architecture-docdb-replication-cdc-logical-replication + weight: 500 +type: docs +--- + +Change data capture (CDC) in YugabyteDB provides technology to ensure that any changes in data due to operations such as inserts, updates, and deletions are identified, captured, and made available for consumption by applications and other tools. + +CDC in YugabyteDB is based on the PostgreSQL Logical Replication model. The fundamental concept here is that of the Replication Slot. A Replication Slot represents a stream of changes that can be replayed to the client in the order they were made on the origin server in a manner that preserves transactional consistency. This is the basis for the support for Transactional CDC in YugabyteDB. Where the strict requirements of Transactional CDC are not present, multiple replication slots can be used to stream changes from unrelated tables in parallel. + +## Architecture + +![Logical replication architecture](/images/architecture/cdc-logical-replication-architecture.png) + +The following are the main components of the Yugabyte CDC solution: + +1. Walsender - A special purpose PG backend responsible for streaming changes to the client and handling acknowledgments. + +2. Virtual WAL (VWAL) - Assembles changes from all the shards of user tables (under the publication) to maintain transactional consistency. + +3. CDCService - Retrieves changes from the WAL of a specified shard starting from a given checkpoint. + +### Data Flow + +Logical replication starts by copying a snapshot of the data on the publisher database. After that is done, changes on the publisher are streamed to the server as they occur in near real time. + +To setup Logical Replication, an application will first have to create a replication slot. When a replication slot is created, a boundary is established between the snapshot data and the streaming changes. This boundary or `consistent_point` is a consistent state of the source database. It corresponds to a commit time (HybridTime value). Data from transactions with commit time <= commit time corresponding to the `consistent_point` are consumed as part of the initial snapshot. Changes from transactions with commit time greater than the commit time of the `consistent_point` are consumed in the streaming phase in transaction commit time order. + +#### Initial Snapshot + +The initial snapshot data for each table is consumed by executing a corresponding snapshot query (SELECT statement) on that table. This snapshot query should be executed as of the database state corresponding to the `consistent_point`. This database state is represented by a value of HybridTime. + +First, a `SET LOCAL yb_read_time TO ' ht'` command should be executed on the connection (session). The SELECT statement corresponding to the snapshot query should then be executed as part of the same transaction. + +The HybridTime value to use in the `SET LOCAL yb_read_time` command is the value of the `snapshot_name` field that is returned by the `CREATE_REPLICATION_SLOT` command. Alternatively, it can be obtained by querying the `pg_replication_slots` view. + +During Snapshot consumption, the snapshot data from all tables will be from the same consistent state (`consistent_point`). At the end of Snapshot consumption, the state of the target system is at/based on the `consistent_point`. History of the tables as of the `consistent_point` is retained on the source until the snapshot is consumed. + +#### Streaming Data Flow + +YugabyteDB automatically splits user tables into multiple shards (also called tablets) using either a hash- or range-based strategy. The primary key for each row in the table uniquely identifies the location of the tablet in the row. + +Each tablet has its own WAL. WAL is NOT in-memory, but it is disk persisted. Each WAL preserves the information on the changes involved in the transactions (or changes) for that tablet as well as additional metadata related to the transactions. + +**Step 1 - Data flow from the tablet WAL to the VWAL** + +![CDCService-VWAL](/images/architecture/cdc_service_vwal_interaction.png) + +Each tablet sends changes in transaction commit time order. Further, in a transaction, the changes are in the order in which the operations were performed in the transaction. + +**Step 2 - Sorting in the VWAL and sending transactions to the Walsender** + +![VWAL-Walsender](/images/architecture/vwal_walsender_interaction.png) + +VWAL collects changes across multiple tablets, assembles the transactions, assigns LSN to each change and transaction boundary (BEGIN, COMMIT) record, and sends the changes to the Walsender in transaction commit time order. + +**Step 3 - Walsender to client** + +Walsender sends changes to the output plugin, which filters them according to the slot's publication and converts them into the client's desired format. These changes are then streamed to the client using the appropriate streaming replication protocols determined by the output plugin. Yugabyte follows the same streaming replication protocols as defined in PostgreSQL. + + diff --git a/docs/content/preview/architecture/docdb-replication/change-data-capture.md b/docs/content/preview/architecture/docdb-replication/change-data-capture.md index 10f262030ab3..bf7d65d29285 100644 --- a/docs/content/preview/architecture/docdb-replication/change-data-capture.md +++ b/docs/content/preview/architecture/docdb-replication/change-data-capture.md @@ -1,7 +1,7 @@ --- -title: Change data capture (CDC) in YugabyteDB -headerTitle: Change data capture (CDC) -linkTitle: Change data capture (CDC) +title: Change data capture (CDC) gRPC Replication in YugabyteDB +headerTitle: CDC using gRPC Replication +linkTitle: CDC using gRPC Replication description: Learn how YugabyteDB supports asynchronous replication of data changes (inserts, updates, and deletes) to external databases or applications. badges: ea aliases: @@ -10,38 +10,46 @@ menu: preview: parent: architecture-docdb-replication identifier: architecture-docdb-replication-cdc - weight: 500 + weight: 600 type: docs --- -Change data capture (CDC) in YugabyteDB provides technology to ensure that any changes in data due to operations such as inserts, updates, and deletions are identified, captured, and automatically applied to another data repository instance, or made available for consumption by applications and other tools. CDC provides the following guarantees. - -- [Ordering is maintained per-tablet](#per-tablet-ordered-delivery) -- [At-least once delivery](#at-least-once-delivery) -- [No gaps](#no-gaps-in-change-stream) - ## Architecture ![Stateless CDC Service](/images/architecture/stateless_cdc_service.png) Every YB-TServer has a `CDC service` that is stateless. The main APIs provided by the CDC service are the following: -* `createCDCSDKStream` API for creating the stream on the database. -* `getChangesCDCSDK` API that can be used by the client to get the latest set of changes. +- `createCDCSDKStream` API for creating the stream on the database. +- `getChangesCDCSDK` API that can be used by the client to get the latest set of changes. ## CDC streams Creating a new CDC stream returns a stream UUID. This is facilitated via the [yb-admin](../../../admin/yb-admin/#change-data-capture-cdc-commands) tool. -## Debezium +YugabyteDB automatically splits user tables into multiple shards (also called tablets) using either a hash- or range-based strategy. The primary key for each row in the table uniquely identifies the location of the tablet in the row. + +Each tablet has its own WAL file. WAL is NOT in-memory, but it is disk persisted. Each WAL preserves the order in which transactions (or changes) happened. Hybrid TS, Operation ID, and additional metadata about the transaction is also preserved. + +![How does CDC work](/images/explore/cdc-overview-work2.png) + +YugabyteDB normally purges WAL segments after some period of time. This means that the connector does not have the complete history of all changes that have been made to the database. Therefore, when the connector first connects to a particular YugabyteDB database, it starts by performing a consistent snapshot of each of the database schemas. + +The Debezium YugabyteDB connector captures row-level changes in the schemas of a YugabyteDB database. The first time it connects to a YugabyteDB cluster, the connector takes a consistent snapshot of all schemas. After that snapshot is complete, the connector continuously captures row-level changes that insert, update, and delete database content, and that were committed to a YugabyteDB database. + +![How does CDC work](/images/explore/cdc-overview-work.png) + +The connector produces a change event for every row-level insert, update, and delete operation that was captured, and sends change event records for each table in a separate Kafka topic. Client applications read the Kafka topics that correspond to the database tables of interest, and can react to every row-level event they receive from those topics. For each table, the default behavior is that the connector streams all generated events to a separate Kafka topic for that table. Applications and services consume data change event records from that topic. + +The core primitive of CDC is the _stream_. Streams can be enabled and disabled on databases. Every change to a watched database table is emitted as a record in a configurable format to a configurable sink. Streams scale to any YugabyteDB cluster independent of its size and are designed to impact production traffic as little as possible. -To consume the events generated by CDC, Debezium is used as the connector. Debezium is an open-source distributed platform that needs to be pointed at the database using the stream ID. For information on how to set up Debezium for YugabyteDB CDC, see [Debezium integration](../../../integrations/cdc/debezium/). +![How does CDC work](/images/explore/cdc-overview-work3.png) -## Pushing changes to external systems +## CDC guarantees -Using the Debezium connector for YugabyteDB, changes are pushed from YugabyteDB to a Kafka topic, which can then be used by any end-user application for the processing and analysis of the records. +CDC in YugabyteDB provides technology to ensure that any changes in data due to operations (such as inserts, updates, and deletions) are identified, captured, and automatically applied to another data repository instance, or made available for consumption by applications and other tools. CDC provides the following guarantees. -## Per-tablet ordered delivery +### Per-tablet ordered delivery All data changes for one row or multiple rows in the same tablet are received in the order in which they occur. Due to the distributed nature of the problem, however, there is no guarantee for the order across tablets. @@ -53,13 +61,13 @@ Consider the following scenario: In this case, it is possible for CDC to push the later update corresponding to `row #2` change to Kafka before pushing the earlier update, corresponding to `row #1`. -## At-least-once delivery +### At-least-once delivery Updates for rows are pushed at least once. With the at-least-once delivery, you never lose a message, however the message might be delivered to a CDC consumer more than once. This can happen in case of a tablet leader change, where the old leader already pushed changes to Kafka, but the latest pushed `op id` was not updated in the CDC metadata. For example, a CDC client has received changes for a row at times `t1` and `t3`. It is possible for the client to receive those updates again. -## No gaps in change stream +### No gaps in change stream When you have received a change for a row for timestamp `t`, you do not receive a previously unseen change for that row from an earlier timestamp. This guarantees that receiving any change implies that all earlier changes have been received for a row. diff --git a/docs/content/preview/explore/change-data-capture/_index.md b/docs/content/preview/explore/change-data-capture/_index.md index 53788190cbb2..cdc447229f34 100644 --- a/docs/content/preview/explore/change-data-capture/_index.md +++ b/docs/content/preview/explore/change-data-capture/_index.md @@ -14,69 +14,46 @@ menu: weight: 280 type: indexpage --- -In databases, change data capture (CDC) is a set of software design patterns used to determine and track the data that has changed so that action can be taken using the changed data. CDC is beneficial in a number of scenarios. Let us look at few of them. +In databases, change data capture (CDC) is a set of software design patterns used to determine and track the data that has changed so that action can be taken using the changed data. CDC is beneficial in a number of scenarios: -- **Microservice-oriented architectures** : Some microservices require a stream of changes to the data, and using CDC in YugabyteDB can provide consumable data changes to CDC subscribers. +- **Microservice-oriented architectures**: Some microservices require a stream of changes to the data, and using CDC in YugabyteDB can provide consumable data changes to CDC subscribers. -- **Asynchronous replication to remote systems** : Remote systems may subscribe to a stream of data changes and then transform and consume the changes. Maintaining separate database instances for transactional and reporting purposes can be used to manage workload performance. +- **Asynchronous replication to remote systems**: Remote systems may subscribe to a stream of data changes and then transform and consume the changes. Maintaining separate database instances for transactional and reporting purposes can be used to manage workload performance. -- **Multiple data center strategies** : Maintaining multiple data centers enables enterprises to provide high availability (HA). +- **Multiple data center strategies**: Maintaining multiple data centers enables enterprises to provide high availability (HA). -- **Compliance and auditing** : Auditing and compliance requirements can require you to use CDC to maintain records of data changes. +- **Compliance and auditing**: Auditing and compliance requirements can require you to use CDC to maintain records of data changes. -{{}} +YugabyteDB supports the following methods for reading change events. - {{}} +## PostgreSQL Logical Replication Protocol (Recommended) - {{}} +This method uses the PostgreSQL replication protocol, ensuring compatibility with PostgreSQL CDC systems. Logical replication operates through a publish-subscribe model. It replicates data objects and their changes based on the replication identity. -{{}} +It works as follows: -## How does CDC work +1. Create Publications in the YugabyteDB cluster similar to PostgreSQL. +1. Deploy the YugabyteDB Connector in your preferred Kafka Connect environment. +1. The connector uses replication slots to capture change events and publishes them directly to a Kafka topic. -YugabyteDB CDC captures changes made to data in the database and streams those changes to external processes, applications, or other databases. CDC allows you to track and propagate changes in a YugabyteDB database to downstream consumers based on its Write-Ahead Log (WAL). YugabyteDB CDC uses Debezium to capture row-level changes resulting from INSERT, UPDATE, and DELETE operations in the upstream database, and publishes them as events to Kafka using Kafka Connect-compatible connectors. +This is the recommended approach for most CDC applications due to its compatibility with PostgreSQL. -![What is CDC](/images/explore/cdc-overview-what.png) - -{{}} -To know more about the internals of CDC, see [Overview](./cdc-overview). + -## Debezium connector - -To capture and stream your changes in YugabyteDB to an external system, you need a connector that can read the changes in YugabyteDB and stream it out. For this, you can use the Debezium connector. Debezium is deployed as a set of Kafka Connect-compatible connectors, so you first need to define a YugabyteDB connector configuration and then start the connector by adding it to Kafka Connect. +## YugabyteDB gRPC Replication Protocol -{{}} -To understand how the various features and configuration of the connector, see [Debezium connector](./debezium-connector-yugabytedb). -{{}} +This method involves setting up a change stream in YugabyteDB that uses the native gRPC replication protocol to publish change events. -## Monitoring +It works as follows: -You can monitor the activities and status of the deployed connectors using the http end points provided by YugabyteDB. +1. Establish a change stream in the YugabyteDB cluster using the yb_admin CLI commands. +1. Deploy the YugabyteDB gRPC Connector in your preferred Kafka Connect environment. +1. The connector captures change events using YugabyteDB's native gRPC replication and directly publishes them to a Kafka topic. -{{}} -To know more about how to monitor your CDC setup, see [Monitor](./cdc-monitor). +{{}} +To learn about gRPC Replication, see [Using YugabyteDB gRPC Replication](./using-yugabytedb-grpc-replication/). {{}} - -For tutorials on streaming data to Kafka environments, including Amazon MSK, Azure Event Hubs, and Confluent Cloud, see [Kafka environments](/preview/tutorials/cdc-tutorials/). - -## Learn more - -- [Examples of CDC usage and patterns](https://github.com/yugabyte/cdc-examples/tree/main) {{}} -- [Tutorials to deploy in different Kafka environments](../../tutorials/cdc-tutorials/) {{}} -- [Data Streaming Using YugabyteDB CDC, Kafka, and SnowflakeSinkConnector](https://www.yugabyte.com/blog/data-streaming-using-yugabytedb-cdc-kafka-and-snowflakesinkconnector/) {{}} -- [Unlock Azure Storage Options With YugabyteDB CDC](https://www.yugabyte.com/blog/unlocking-azure-storage-options-with-yugabytedb-cdc/) {{}} -- [Change Data Capture From YugabyteDB to Elasticsearch](https://www.yugabyte.com/blog/change-data-capture-cdc-yugabytedb-elasticsearch/) {{}} -- [Snowflake CDC: Publishing Data Using Amazon S3 and YugabyteDB](https://www.yugabyte.com/blog/snowflake-cdc-publish-data-using-amazon-s3-yugabytedb/) {{}} -- [Streaming Changes From YugabyteDB to Downstream Databases](https://www.yugabyte.com/blog/streaming-changes-yugabytedb-cdc-downstream-databases/) {{}} -- [Change Data Capture from YugabyteDB CDC to ClickHouse](https://www.yugabyte.com/blog/change-data-capture-cdc-yugabytedb-clickhouse/) {{}} -- [How to Run Debezium Server with Kafka as a Sink](https://www.yugabyte.com/blog/change-data-capture-cdc-run-debezium-server-kafka-sink/) {{}} -- [Change Data Capture Using a Spring Data Processing Pipeline](https://www.yugabyte.com/blog/change-data-capture-cdc-spring-data-processing-pipeline/) {{}} diff --git a/docs/content/preview/explore/change-data-capture/cdc-overview.md b/docs/content/preview/explore/change-data-capture/cdc-overview.md deleted file mode 100644 index b81c86623a23..000000000000 --- a/docs/content/preview/explore/change-data-capture/cdc-overview.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: Overview of CDC internals -linkTitle: Overview -description: Change Data Capture in YugabyteDB. -headcontent: Change Data Capture in YugabyteDB -menu: - preview: - parent: explore-change-data-capture - identifier: cdc-overview - weight: 10 -type: docs ---- - -YugabyteDB automatically splits user tables into multiple shards (also called tablets) using either a hash- or range-based strategy. The primary key for each row in the table uniquely identifies the location of the tablet in the row. - -Each tablet has its own WAL file. WAL is NOT in-memory, but it is disk persisted. Each WAL preserves the order in which transactions (or changes) happened. Hybrid TS, Operation ID, and additional metadata about the transaction is also preserved. - -![How does CDC work](/images/explore/cdc-overview-work2.png) - -YugabyteDB normally purges WAL segments after some period of time. This means that the connector does not have the complete history of all changes that have been made to the database. Therefore, when the connector first connects to a particular YugabyteDB database, it starts by performing a consistent snapshot of each of the database schemas. - -The Debezium YugabyteDB connector captures row-level changes in the schemas of a YugabyteDB database. The first time it connects to a YugabyteDB cluster, the connector takes a consistent snapshot of all schemas. After that snapshot is complete, the connector continuously captures row-level changes that insert, update, and delete database content, and that were committed to a YugabyteDB database. - -![How does CDC work](/images/explore/cdc-overview-work.png) - -The connector produces a change event for every row-level insert, update, and delete operation that was captured, and sends change event records for each table in a separate Kafka topic. Client applications read the Kafka topics that correspond to the database tables of interest, and can react to every row-level event they receive from those topics. For each table, the default behavior is that the connector streams all generated events to a separate Kafka topic for that table. Applications and services consume data change event records from that topic. - -The core primitive of CDC is the _stream_. Streams can be enabled and disabled on databases. Every change to a watched database table is emitted as a record in a configurable format to a configurable sink. Streams scale to any YugabyteDB cluster independent of its size and are designed to impact production traffic as little as possible. - -![How does CDC work](/images/explore/cdc-overview-work3.png) - -## Known limitations - -* A single stream can only be used to stream data from one namespace only. -* There should be a primary key on the table you want to stream the changes from. -* CDC is not supported on a target table for xCluster replication [11829](https://github.com/yugabyte/yugabyte-db/issues/11829). -* Currently we don't support schema evolution for changes that require table rewrites (ex: ALTER TYPE). -* YCQL tables aren't currently supported. Issue [11320](https://github.com/yugabyte/yugabyte-db/issues/11320). - -In addition, CDC support for the following features will be added in upcoming releases: - -* Support for point-in-time recovery (PITR) is tracked in issue [10938](https://github.com/yugabyte/yugabyte-db/issues/10938). -* Support for transaction savepoints is tracked in issue [10936](https://github.com/yugabyte/yugabyte-db/issues/10936). -* Support for enabling CDC on Read Replicas is tracked in issue [11116](https://github.com/yugabyte/yugabyte-db/issues/11116). -* Support for schema evolution with before image is tracked in issue [15197](https://github.com/yugabyte/yugabyte-db/issues/15197). - -## Learn more - -* Refer to [CDC Examples](https://github.com/yugabyte/cdc-examples/tree/main) for CDC usage and pattern examples. -* Refer to [Tutorials](../../../tutorials/cdc-tutorials/) to deploy in different Kafka environments. -* Refer to blogs about CDC: - * [Data Streaming Using YugabyteDB CDC, Kafka, and SnowflakeSinkConnector](https://www.yugabyte.com/blog/data-streaming-using-yugabytedb-cdc-kafka-and-snowflakesinkconnector/) - * [Unlock Azure Storage Options With YugabyteDB CDC](https://www.yugabyte.com/blog/unlocking-azure-storage-options-with-yugabytedb-cdc/) - * [Change Data Capture From YugabyteDB to Elasticsearch](https://www.yugabyte.com/blog/change-data-capture-cdc-yugabytedb-elasticsearch/) - * [Snowflake CDC: Publishing Data Using Amazon S3 and YugabyteDB](https://www.yugabyte.com/blog/snowflake-cdc-publish-data-using-amazon-s3-yugabytedb/) - * [Streaming Changes From YugabyteDB to Downstream Databases](https://www.yugabyte.com/blog/streaming-changes-yugabytedb-cdc-downstream-databases/) - * [Change Data Capture from YugabyteDB CDC to ClickHouse](https://www.yugabyte.com/blog/change-data-capture-cdc-yugabytedb-clickhouse/) - * [How to Run Debezium Server with Kafka as a Sink](https://www.yugabyte.com/blog/change-data-capture-cdc-run-debezium-server-kafka-sink/) - * [Change Data Capture Using a Spring Data Processing Pipeline](https://www.yugabyte.com/blog/change-data-capture-cdc-spring-data-processing-pipeline/) diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/_index.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/_index.md new file mode 100644 index 000000000000..5ff2d5b2dbf8 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/_index.md @@ -0,0 +1,17 @@ + \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/advanced-configuration.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/advanced-configuration.md new file mode 100644 index 000000000000..0f676e580ab7 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/advanced-configuration.md @@ -0,0 +1,12 @@ +--- +title: Advanced Configurations +headerTitle: Advanced Configurations +linkTitle: Advanced Configurations +description: Advanced Configurations for Change Data Capture in YugabyteDB. +menu: + preview: + parent: explore-change-data-capture-logical-replication + identifier: advanced-configurations + weight: 40 +type: docs +--- \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/advanced-topic.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/advanced-topic.md new file mode 100644 index 000000000000..73c365ffd09a --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/advanced-topic.md @@ -0,0 +1,12 @@ +--- +title: Advanced topics +headerTitle: Advanced topics +linkTitle: Advanced topics +description: Advanced topics for Change Data Capture in YugabyteDB. +menu: + preview: + parent: explore-change-data-capture-logical-replication + identifier: advanced-topics + weight: 50 +type: docs +--- \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/cdc-best-practices.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/cdc-best-practices.md new file mode 100644 index 000000000000..a2e5ab485373 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/cdc-best-practices.md @@ -0,0 +1,12 @@ +--- +title: Best Practices +headerTitle: Best Practices +linkTitle: Best Practices +description: Best Practices for Change Data Capture in YugabyteDB. +menu: + preview: + parent: explore-change-data-capture-logical-replication + identifier: best-practices-cdc + weight: 60 +type: docs +--- \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/get-started.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/get-started.md new file mode 100644 index 000000000000..c2f1375ace02 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/get-started.md @@ -0,0 +1,13 @@ +--- +title: Get started with CDC in YugabyteDB +headerTitle: Get started +linkTitle: Get started +description: Get started with Change Data Capture in YugabyteDB. +headcontent: Get set up for using CDC in YugabyteDB +menu: + preview: + parent: explore-change-data-capture-logical-replication + identifier: get-started + weight: 20 +type: docs +--- \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/monitor.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/monitor.md new file mode 100644 index 000000000000..8aaf773a9913 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/monitor.md @@ -0,0 +1,12 @@ +--- +title: CDC monitoring in YugabyteDB +headerTitle: Monitor +linkTitle: Monitor +description: Monitor Change Data Capture in YugabyteDB. +menu: + preview: + parent: explore-change-data-capture-logical-replication + identifier: monitor + weight: 30 +type: docs +--- \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/overview.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/overview.md new file mode 100644 index 000000000000..dbf2a9522fa9 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/overview.md @@ -0,0 +1,12 @@ +--- +title: Overview of CDC - logical replication +linkTitle: Overview +description: Change Data Capture in YugabyteDB. +headcontent: Change Data Capture in YugabyteDB +menu: + preview: + parent: explore-change-data-capture-logical-replication + identifier: overview + weight: 10 +type: docs +--- \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-logical-replication/yugabtyedb-connector.md b/docs/content/preview/explore/change-data-capture/using-logical-replication/yugabtyedb-connector.md new file mode 100644 index 000000000000..bb203f0a9b04 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-logical-replication/yugabtyedb-connector.md @@ -0,0 +1,12 @@ +--- +title: YugabyteDB connector +headerTitle: YugabyteDB connector +linkTitle: YugabyteDB connector +description: YugabyteDB connector for Change Data Capture in YugabyteDB. +menu: + preview: + parent: explore-change-data-capture-logical-replication + identifier: yugabytedb-connector + weight: 70 +type: docs +--- \ No newline at end of file diff --git a/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/_index.md b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/_index.md new file mode 100644 index 000000000000..1cf103f6a468 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/_index.md @@ -0,0 +1,85 @@ +--- +title: Using YugabyteDB gRPC replication +headerTitle: Using YugabyteDB gRPC replication +linkTitle: Using YugabyteDB gRPC replication +description: CDC or Change data capture is a process to capture changes made to data in the database. +headcontent: Capture changes made to data in the database +image: /images/section_icons/index/develop.png +cascade: + earlyAccess: /preview/releases/versioning/#feature-maturity +aliases: + - /preview/explore/change-data-capture/cdc-overview/ +menu: + preview: + identifier: explore-change-data-capture-grpc-replication + parent: explore-change-data-capture + weight: 280 +type: indexpage +showRightNav: true +--- + +## Overview + +YugabyteDB CDC captures changes made to data in the database and streams those changes to external processes, applications, or other databases. CDC allows you to track and propagate changes in a YugabyteDB database to downstream consumers based on its Write-Ahead Log (WAL). YugabyteDB CDC uses Debezium to capture row-level changes resulting from INSERT, UPDATE, and DELETE operations in the upstream database, and publishes them as events to Kafka using Kafka Connect-compatible connectors. + +![What is CDC](/images/explore/cdc-overview-what.png) + + + +## YugabyteDB gRPC Connector + +To capture and stream your changes in YugabyteDB to an external system, you need a connector that can read the changes in YugabyteDB and stream it out. For this, you can use the YugabyteDB gRPC (Debezium) connector. The connector is deployed as a set of Kafka Connect-compatible connectors, so you first need to define a YugabyteDB connector configuration and then start the connector by adding it to Kafka Connect. + +{{}} +To understand how the various features and configuration of the connector, see [YugabyteDB gRPC Connector](./debezium-connector-yugabytedb). +{{}} + +## Get started + +Get started with Yugabyte gRPC replication. + +For tutorials on streaming data to Kafka environments, including Amazon MSK, Azure Event Hubs, and Confluent Cloud, see [Kafka environments](/preview/tutorials/cdc-tutorials/). + +{{}} +To learn how get started with the connector, see [Get started](./cdc-get-started). +{{}} + +## Monitoring + +You can monitor the activities and status of the deployed connectors using the http end points provided by YugabyteDB. + +{{}} +To know more about how to monitor your CDC setup, see [Monitor](./cdc-monitor). +{{}} + +## Known limitations + +* A single stream can only be used to stream data from one namespace only. +* There should be a primary key on the table you want to stream the changes from. +* CDC is not supported on a target table for xCluster replication [11829](https://github.com/yugabyte/yugabyte-db/issues/11829). +* Currently we don't support schema evolution for changes that require table rewrites (ex: ALTER TYPE). +* YCQL tables aren't currently supported. Issue [11320](https://github.com/yugabyte/yugabyte-db/issues/11320). + +In addition, CDC support for the following features will be added in upcoming releases: + +* Support for point-in-time recovery (PITR) is tracked in issue [10938](https://github.com/yugabyte/yugabyte-db/issues/10938). +* Support for transaction savepoints is tracked in issue [10936](https://github.com/yugabyte/yugabyte-db/issues/10936). +* Support for enabling CDC on Read Replicas is tracked in issue [11116](https://github.com/yugabyte/yugabyte-db/issues/11116). +* Support for schema evolution with before image is tracked in issue [15197](https://github.com/yugabyte/yugabyte-db/issues/15197). + +## Learn more + +- [Examples of CDC usage and patterns](https://github.com/yugabyte/cdc-examples/tree/main) {{}} +- [Tutorials to deploy in different Kafka environments](../../../tutorials/cdc-tutorials/) {{}} +- [Data Streaming Using YugabyteDB CDC, Kafka, and SnowflakeSinkConnector](https://www.yugabyte.com/blog/data-streaming-using-yugabytedb-cdc-kafka-and-snowflakesinkconnector/) {{}} +- [Unlock Azure Storage Options With YugabyteDB CDC](https://www.yugabyte.com/blog/unlocking-azure-storage-options-with-yugabytedb-cdc/) {{}} +- [Change Data Capture From YugabyteDB to Elasticsearch](https://www.yugabyte.com/blog/change-data-capture-cdc-yugabytedb-elasticsearch/) {{}} +- [Snowflake CDC: Publishing Data Using Amazon S3 and YugabyteDB](https://www.yugabyte.com/blog/snowflake-cdc-publish-data-using-amazon-s3-yugabytedb/) {{}} +- [Streaming Changes From YugabyteDB to Downstream Databases](https://www.yugabyte.com/blog/streaming-changes-yugabytedb-cdc-downstream-databases/) {{}} +- [Change Data Capture from YugabyteDB CDC to ClickHouse](https://www.yugabyte.com/blog/change-data-capture-cdc-yugabytedb-clickhouse/) {{}} +- [How to Run Debezium Server with Kafka as a Sink](https://www.yugabyte.com/blog/change-data-capture-cdc-run-debezium-server-kafka-sink/) {{}} +- [Change Data Capture Using a Spring Data Processing Pipeline](https://www.yugabyte.com/blog/change-data-capture-cdc-spring-data-processing-pipeline/) {{}} diff --git a/docs/content/preview/explore/change-data-capture/cdc-get-started.md b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-get-started.md similarity index 75% rename from docs/content/preview/explore/change-data-capture/cdc-get-started.md rename to docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-get-started.md index ac0c6fa410ea..37e92fd28dde 100644 --- a/docs/content/preview/explore/change-data-capture/cdc-get-started.md +++ b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-get-started.md @@ -4,41 +4,43 @@ headerTitle: Get started linkTitle: Get started description: Get started with Change Data Capture in YugabyteDB. headcontent: Get set up for using CDC in YugabyteDB +aliases: + - /preview/explore/change-data-capture/cdc-get-started/ menu: preview: - parent: explore-change-data-capture + parent: explore-change-data-capture-grpc-replication identifier: cdc-get-started weight: 30 type: docs --- -To stream data change events from YugabyteDB databases, you need to use Debezium YugabyteDB connector. To deploy a Debezium YugabyteDB connector, you install the Debezium YugabyteDB connector archive, configure the connector, and start the connector by adding its configuration to Kafka Connect. You can download the connector from [GitHub releases](https://github.com/yugabyte/debezium-connector-yugabytedb/releases). The connector supports Kafka Connect version 2.x and later, and for YugabyteDB, it supports version 2.14 and later. For more connector configuration details and complete steps, refer to [Debezium connector](../debezium-connector-yugabytedb/). - -## Ordering guarantees - -|Ordering guarantee| Description| -|----------| ----------------------------| -|Per-tablet ordered delivery guarantee|All changes for a row (or rows in the same tablet) are received in the order in which they happened. However, due to the distributed nature of the problem, there is no guarantee of the order across tablets.| -|At least once delivery|Updates for rows are streamed at least once. This can happen in the case of Kafka Connect Node failure. If the Kafka Connect Node pushes the records to Kafka and crashes before committing the offset, on restart, it will again get the same set of records.| -|No gaps in change stream|Note that after you have received a change for a row for some timestamp `t`, you won't receive a previously unseen change for that row at a lower timestamp. Receiving any change implies that you have received _all older changes_ for that row.| - ## Set up YugabyteDB for CDC -The following steps are necessary to set up YugabyteDB for use with the Debezium YugabyteDB connector: +The following steps are necessary to set up YugabyteDB for use with the YugabyteDB gRPC connector: - Create a DB stream ID. - Before you use the YugabyteDB connector to retrieve data change events from YugabyteDB, create a stream ID using the yb-admin CLI command. Refer to the [yb-admin](../../../admin/yb-admin/#change-data-capture-cdc-commands) CDC command reference documentation for more details. + Before you use the YugabyteDB connector to retrieve data change events from YugabyteDB, create a stream ID using the yb-admin CLI command. Refer to the [yb-admin](../../../../admin/yb-admin/#change-data-capture-cdc-commands) CDC command reference documentation for more details. - Make sure the YB-Master and YB-TServer ports are open. - The connector connects to the YB-Master and YB-TServer processes running on the YugabyteDB server. Make sure the ports on which these processes are running are open. The [default ports](../../../reference/configuration/default-ports/) on which the processes run are `7100` and `9100` respectively. + The connector connects to the YB-Master and YB-TServer processes running on the YugabyteDB server. Make sure the ports on which these processes are running are open. The [default ports](../../../../reference/configuration/default-ports/) on which the processes run are `7100` and `9100` respectively. - Monitor available disk space. The change records for CDC are read from the WAL. YugabyteDB CDC maintains checkpoints internally for each DB stream ID and garbage collects the WAL entries if those have been streamed to the CDC clients. - In case CDC is lagging or away for some time, the disk usage may grow and cause YugabyteDB cluster instability. To avoid this scenario, if a stream is inactive for a configured amount of time, the WAL is garbage collected. This is configurable using a [YB-TServer flag](../../../reference/configuration/yb-tserver/#change-data-capture-cdc-flags). + In case CDC is lagging or away for some time, the disk usage may grow and cause YugabyteDB cluster instability. To avoid this scenario, if a stream is inactive for a configured amount of time, the WAL is garbage collected. This is configurable using a [YB-TServer flag](../../../../reference/configuration/yb-tserver/#change-data-capture-cdc-flags). + +## Deploying the YugabyteDB gRPC Connector +To stream data change events from YugabyteDB databases, follow these steps to deploy the YugabyteDB gRPC Connector: + +* Download the Connector: You can download the connector from the [GitHub releases](https://github.com/yugabyte/debezium-connector-yugabytedb/releases) +* Install the Connector: Extract and install the connector archive in your Kafka Connect environment. +* Configure the Connector: Modify the connector configuration to suit your specific requirements. +* Start the Connector: Add the connector's configuration to Kafka Connect and start the connector. + +For more details on connector configuration and deployment steps, refer to the [YugabyteDB gRPC Connector documentation]((../debezium-connector-yugabytedb/)). ## Serialization @@ -110,9 +112,9 @@ To use the [protobuf](http://protobuf.dev) format for the serialization/de-seria Before image refers to the state of the row _before_ the change event occurred. The YugabyteDB connector sends the before image of the row when it will be configured using a stream ID enabled with before image. It is populated for UPDATE and DELETE events. For INSERT events, before image doesn't make sense as the change record itself is in the context of new row insertion. -Yugabyte uses multi-version concurrency control (MVCC) mechanism, and compacts data at regular intervals. The compaction or the history retention is controlled by the [history retention interval flag](../../../reference/configuration/yb-tserver/#timestamp-history-retention-interval-sec). However, when before image is enabled for a database, YugabyteDB adjusts the history retention for that database based on the most lagging active CDC stream so that the previous row state is retained, and available. Consequently, in the case of a lagging CDC stream, the amount of space required for the database grows as more data is retained. On the other hand, older rows that are not needed for any of the active CDC streams are identified and garbage collected. +Yugabyte uses multi-version concurrency control (MVCC) mechanism, and compacts data at regular intervals. The compaction or the history retention is controlled by the [history retention interval flag](../../../../reference/configuration/yb-tserver/#timestamp-history-retention-interval-sec). However, when before image is enabled for a database, YugabyteDB adjusts the history retention for that database based on the most lagging active CDC stream so that the previous row state is retained, and available. Consequently, in the case of a lagging CDC stream, the amount of space required for the database grows as more data is retained. On the other hand, older rows that are not needed for any of the active CDC streams are identified and garbage collected. -Schema version that is currently being used by a CDC stream will be used to frame before and current row images. The before image functionality is disabled by default unless it is specifically turned on during the CDC stream creation. The [yb-admin](../../../admin/yb-admin/#enabling-before-image) `create_change_data_stream` command can be used to create a CDC stream with before image enabled. +Schema version that is currently being used by a CDC stream will be used to frame before and current row images. The before image functionality is disabled by default unless it is specifically turned on during the CDC stream creation. The [yb-admin](../../../../admin/yb-admin/#enabling-before-image) `create_change_data_stream` command can be used to create a CDC stream with before image enabled. {{< tip title="Use transformers" >}} @@ -501,7 +503,7 @@ CDC record for UPDATE (using schema version 1): ## Colocated tables -YugabyteDB supports streaming of changes from [colocated tables](../../../architecture/docdb-sharding/colocated-tables). The connector can be configured with regular configuration properties and deployed for streaming. +YugabyteDB supports streaming of changes from [colocated tables](../../../../architecture/docdb-sharding/colocated-tables). The connector can be configured with regular configuration properties and deployed for streaming. {{< note title="Note" >}} @@ -513,15 +515,15 @@ To stream the changes for the new table, delete the existing connector and deplo ## Important configuration settings -You can use several flags to fine-tune YugabyteDB's CDC behavior. These flags are documented in the [Change data capture flags](../../../reference/configuration/yb-tserver/#change-data-capture-cdc-flags) section of the YB-TServer reference and [Change data capture flags](../../../reference/configuration/yb-master/#change-data-capture-cdc-flags) section of the YB-Master reference. The following flags are particularly important for configuring CDC: +You can use several flags to fine-tune YugabyteDB's CDC behavior. These flags are documented in the [Change data capture flags](../../../../reference/configuration/yb-tserver/#change-data-capture-cdc-flags) section of the YB-TServer reference and [Change data capture flags](../../../../reference/configuration/yb-master/#change-data-capture-cdc-flags) section of the YB-Master reference. The following flags are particularly important for configuring CDC: -- [cdc_intent_retention_ms](../../../reference/configuration/yb-tserver/#cdc-intent-retention-ms) - Controls retention of intents, in ms. If a request for change records is not received for this interval, un-streamed intents are garbage collected and the CDC stream is considered expired. This expiry is not reversible, and the only course of action would be to create a new CDC stream. The default value of this flag is 4 hours (4 x 3600 x 1000 ms). +- [cdc_intent_retention_ms](../../../../reference/configuration/yb-tserver/#cdc-intent-retention-ms) - Controls retention of intents, in ms. If a request for change records is not received for this interval, un-streamed intents are garbage collected and the CDC stream is considered expired. This expiry is not reversible, and the only course of action would be to create a new CDC stream. The default value of this flag is 4 hours (4 x 3600 x 1000 ms). -- [cdc_wal_retention_time_secs](../../../reference/configuration/yb-master/#cdc-wal-retention-time-secs) - Controls how long WAL is retained, in seconds. This is irrespective of whether a request for change records is received or not. The default value of this flag is 4 hours (14400 seconds). +- [cdc_wal_retention_time_secs](../../../../reference/configuration/yb-master/#cdc-wal-retention-time-secs) - Controls how long WAL is retained, in seconds. This is irrespective of whether a request for change records is received or not. The default value of this flag is 4 hours (14400 seconds). -- [cdc_snapshot_batch_size](../../../reference/configuration/yb-tserver/#cdc-snapshot-batch-size) - This flag's default value is 250 records included per batch in response to an internal call to get the snapshot. If the table contains a very large amount of data, you may need to increase this value to reduce the amount of time it takes to stream the complete snapshot. You can also choose not to take a snapshot by modifying the [Debezium](../debezium-connector-yugabytedb/) configuration. +- [cdc_snapshot_batch_size](../../../../reference/configuration/yb-tserver/#cdc-snapshot-batch-size) - This flag's default value is 250 records included per batch in response to an internal call to get the snapshot. If the table contains a very large amount of data, you may need to increase this value to reduce the amount of time it takes to stream the complete snapshot. You can also choose not to take a snapshot by modifying the [Debezium](../debezium-connector-yugabytedb/) configuration. -- [cdc_max_stream_intent_records](../../../reference/configuration/yb-tserver/#cdc-max-stream-intent-records) - Controls how many intent records can be streamed in a single `GetChanges` call. Essentially, intents of large transactions are broken down into batches of size equal to this flag, hence this controls how many batches of `GetChanges` calls are needed to stream the entire large transaction. The default value of this flag is 1680, and transactions with intents less than this value are streamed in a single batch. The value of this flag can be increased, if the workload has larger transactions and CDC throughput needs to be increased. Note that high values of this flag can increase the latency of each `GetChanges` call. +- [cdc_max_stream_intent_records](../../../../reference/configuration/yb-tserver/#cdc-max-stream-intent-records) - Controls how many intent records can be streamed in a single `GetChanges` call. Essentially, intents of large transactions are broken down into batches of size equal to this flag, hence this controls how many batches of `GetChanges` calls are needed to stream the entire large transaction. The default value of this flag is 1680, and transactions with intents less than this value are streamed in a single batch. The value of this flag can be increased, if the workload has larger transactions and CDC throughput needs to be increased. Note that high values of this flag can increase the latency of each `GetChanges` call. ## Retaining data for longer durations @@ -535,7 +537,7 @@ Longer values of `cdc_intent_retention_ms`, coupled with longer CDC lags (period ## Content-based routing -By default, the Yugabyte Debezium connector streams all of the change events that it reads from a table to a single static topic. However, you may want to re-route the events into different Kafka topics based on the event's content. You can do this using the Debezium `ContentBasedRouter`. But first, two additional dependencies need to be placed in the Kafka-Connect environment. These are not included in the official *yugabyte-debezium-connector* for security reasons. These dependencies are: +By default, the connector streams all of the change events that it reads from a table to a single static topic. However, you may want to re-route the events into different Kafka topics based on the event's content. You can do this using the Debezium `ContentBasedRouter`. But first, two additional dependencies need to be placed in the Kafka-Connect environment. These are not included in the official *yugabyte-debezium-connector* for security reasons. These dependencies are: - Debezium routing SMT (Single Message Transform) - Groovy JSR223 implementation (or other scripting languages that integrate with [JSR 223](https://jcp.org/en/jsr/detail?id=223)) diff --git a/docs/content/preview/explore/change-data-capture/cdc-monitor.md b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-monitor.md similarity index 98% rename from docs/content/preview/explore/change-data-capture/cdc-monitor.md rename to docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-monitor.md index dbce650c904f..4dad0867f92a 100644 --- a/docs/content/preview/explore/change-data-capture/cdc-monitor.md +++ b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-monitor.md @@ -4,9 +4,11 @@ headerTitle: Monitor linkTitle: Monitor description: Monitor Change Data Capture in YugabyteDB. headcontent: Monitor deployed CDC connectors +aliases: + - /preview/explore/change-data-capture/cdc-monitor/ menu: preview: - parent: explore-change-data-capture + parent: explore-change-data-capture-grpc-replication identifier: cdc-monitor weight: 60 type: docs diff --git a/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-overview.md b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-overview.md new file mode 100644 index 000000000000..ee018de84821 --- /dev/null +++ b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-overview.md @@ -0,0 +1,13 @@ + diff --git a/docs/content/preview/explore/change-data-capture/debezium-connector-yugabytedb.md b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md similarity index 95% rename from docs/content/preview/explore/change-data-capture/debezium-connector-yugabytedb.md rename to docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md index 31d0538b2c74..6e8212cc4627 100644 --- a/docs/content/preview/explore/change-data-capture/debezium-connector-yugabytedb.md +++ b/docs/content/preview/explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb.md @@ -1,16 +1,17 @@ --- -title: Debezium connector for YugabyteDB -headerTitle: Debezium connector for YugabyteDB -linkTitle: Debezium connector -description: Debezium is an open source distributed platform used to capture the changes in a database. +title: YugabyteDB gRPC Connector (Debezium) +headerTitle: YugabyteDB gRPC Connector +linkTitle: YugabyteDB gRPC Connector +description: YugabyteDB gRPC Connector is an open source distributed platform used to capture the changes in a database. aliases: - /preview/explore/change-data-capture/debezium-connector-yugabytedb-ysql - /preview/explore/change-data-capture/debezium-connector - /preview/explore/change-data-capture/debezium - /preview/explore/change-data-capture/debezium-connector-postgresql + - /preview/explore/change-data-capture/debezium-connector-yugabytedb menu: preview: - parent: explore-change-data-capture + parent: explore-change-data-capture-grpc-replication identifier: debezium-connector-yugabytedb weight: 20 type: docs @@ -18,39 +19,22 @@ rightNav: hideH4: true --- -The Debezium connector for YugabyteDB captures row-level changes in the schemas of a YugabyteDB database. - -The first time it connects to a YugabyteDB cluster or universe, the connector takes a consistent snapshot of the tables it is configured for. After that snapshot is complete, the connector continuously captures row-level changes that insert, update, and delete database content that are committed to a YugabyteDB database. The connector generates data change event records and streams them to Kafka topics. For each table, the default behavior is that the connector streams all generated events to a separate Kafka topic for that table. Applications and services consume data change event records from that topic. - -## Overview - -The Debezium connector for YugabyteDB reads the changes produced by YugabyteDB. It uses the CDC service APIs implemented on the server side to get the changes. - -The connector produces a change event for every row-level insert, update, and delete operation that was captured, and sends change event records for each table in separate Kafka topics. Client applications read the Kafka topics corresponding to database tables of interest, and can react to every row-level event they receive from those topics. - -![What is CDC](/images/explore/cdc-overview-what.png) - -YugabyteDB normally purges write-ahead log (WAL) segments after some period of time. This means that the connector does not have the complete history of all changes that have been made to the database. Therefore, when the YugabyteDB connector first connects to a particular YugabyteDB database, it starts by taking a snapshot of each of the database schemas. After the connector completes the snapshot, it continues streaming changes from the exact point at which the snapshot was made. This way, the connector starts with a consistent view of all of the data, and does not omit any changes that were made while the snapshot was being taken. - -The connector is tolerant of failures. As the connector reads changes and produces events, it records the WAL position for each event. If the connector stops for any reason (including communication failures, network problems, or crashes), upon restart the connector continues reading the WAL where it last left off using the WAL position called checkpoints managed on the Kafka side as well as on the YugabyteDB cluster. - -{{< tip title="Use UTF-8 encoding" >}} - -Debezium supports databases with UTF-8 character encoding only. With a single-byte character encoding, it's not possible to correctly process strings that contain extended ASCII code characters. - -{{< /tip >}} - -## Connector compatibility +The YugabyteDB gRPC Connector captures row-level changes in a YugabyteDB database's schemas. +## YugabyteDB gRPC Connector compatibility The connector is compatible with the following versions of YugabyteDB. | YugabyteDB | Connector | | :--- | :--- | -| 2.14 (EA) | 1.9.5.y.3 | +| 2.14 | 1.9.5.y.3 | | 2.16 | 1.9.5.y.24 | | 2.18.2 | 1.9.5.y.33.2 | | 2.20 | 1.9.5.y.220.2 | +Compatibility +* Kafka Connect: The connector supports version 2.x and later. +* YugabyteDB: The connector supports version 2.14 and later. + {{< note title="Note" >}} Starting with YugabyteDB v2.20, the naming convention for releases of the connector uses the scheme *major.y.minor*, as follows: @@ -60,6 +44,33 @@ The connector is backward compatible with previous releases of YugabyteDB unless {{< /note >}} +## Initial Snapshot and Continuous Streaming: + +* Initial Snapshot: Upon its first connection to a YugabyteDB cluster, the connector takes a consistent snapshot of the configured tables. +* Continuous Streaming: After the snapshot, it continuously captures row-level changes (insertions, updates, and deletions) from the database. It then generates data change event records and streams them to Kafka topics. + +![What is CDC](/images/explore/cdc-overview-what.png) + + + +## Kafka Integration: + +For each table, the connector streams all generated events to a separate Kafka topic. Client applications and services can consume these data change event records from their respective topics. + +* CDC (Change Data Capture) Service: The Debezium connector for YugabyteDB leverages the CDC service APIs to read the changes from YugabyteDB. +* Event Production: For every row-level insert, update, and delete operation captured, the connector produces a corresponding change event and sends it to separate Kafka topics dedicated to each table. +* Client Consumption: Applications read the Kafka topics corresponding to the database tables they are interested in and react to the row-level events received. + +## Failure Tolerance +The connector records the WAL position for each event as it reads changes and produces events. If the connector stops (due to communication failures, network problems, or crashes), it resumes reading the WAL from the last recorded position upon restart. This uses checkpoints managed on both the Kafka side and the YugabyteDB cluster. +{{< tip title="Use UTF-8 encoding" >}} + +Debezium supports databases with UTF-8 character encoding only. With a single-byte character encoding, it's not possible to correctly process strings that contain extended ASCII code characters. + +{{< /tip >}} + + + ## How the connector works To optimally configure and run a Debezium YugabyteDB connector, it is helpful to understand how the connector performs snapshots, streams change events, determines Kafka topic names, and uses metadata. @@ -697,7 +708,7 @@ When a row is deleted, the _delete_ event value still works with log compaction, {{< tip title="TRUNCATE tables when CDC is enabled" >}} -By default, the YugabyteDB CDC implementation does not allow you to TRUNCATE a table while an active CDC stream is present on the namespace. To allow truncating tables while CDC is active, set the [enable_truncate_cdcsdk_table](../../../reference/configuration/yb-tserver/#enable-truncate-cdcsdk-table) flag to true. +By default, the YugabyteDB CDC implementation does not allow you to TRUNCATE a table while an active CDC stream is present on the namespace. To allow truncating tables while CDC is active, set the [enable_truncate_cdcsdk_table](../../../../reference/configuration/yb-tserver/#enable-truncate-cdcsdk-table) flag to true. {{< /tip >}} @@ -917,8 +928,8 @@ Support for the following YugabyteDB data types will be enabled in future releas Before using the YugabyteDB connector to monitor the changes on a YugabyteDB server, you need to ensure the following: -* You have a stream ID created on the database you want to monitor the changes for. The stream can be created using the [yb-admin create_change_data_stream](../../../admin/yb-admin#create_change_data_stream) command. -* The table which is supposed to be monitored should have a primary key. Only tables which have a primary key can be streamed. See [limitations](../../change-data-capture/cdc-overview/#known-limitations). +* You have a stream ID created on the database you want to monitor the changes for. The stream can be created using the [yb-admin create_change_data_stream](../../../../admin/yb-admin#create_change_data_stream) command. +* The table which is supposed to be monitored should have a primary key. Only tables which have a primary key can be streamed. See [limitations](../cdc-overview/#known-limitations). ### WAL disk space consumption @@ -928,7 +939,7 @@ For example, the connector is lagging behind in streaming the changes. In this c ## Deployment -To deploy a Debezium YugabyteDB connector, you install the Debezium YugabyteDB connector archive, configure the connector, and start the connector by adding its configuration to Kafka Connect. For complete steps, follow the guide to [running the Debezium connector for YugabyteDB](../../../integrations/cdc/debezium/). +To deploy a Debezium YugabyteDB connector, you install the Debezium YugabyteDB connector archive, configure the connector, and start the connector by adding its configuration to Kafka Connect. For complete steps, follow the guide to [running the Debezium connector for YugabyteDB](../../../../integrations/cdc/debezium/). ### Connector configuration example @@ -959,7 +970,7 @@ You can choose to produce events for a subset of the schemas and tables in a dat 1. The address of this YugabyteDB server. 1. The port number of the YugabyteDB YSQL process. 1. List of comma separated values of master nodes of the YugabyteDB server. Usually in the form `host`:`port`. -1. The DB stream ID created using [yb-admin](../../../admin/yb-admin/#change-data-capture-cdc-commands). +1. The DB stream ID created using [yb-admin](../../../../admin/yb-admin/#change-data-capture-cdc-commands). 1. The name of the YugabyteDB user having the privileges to connect to the database. 1. The password for the above specified YugabyteDB user. 1. The name of the YugabyteDB database to connect to. @@ -1017,7 +1028,7 @@ The following properties are _required_ unless a default value is available: | database.password | N/A | Password for the given user. | | database.dbname | N/A | The database from which to stream. | | database.server.name | N/A | Logical name that identifies and provides a namespace for the particular YugabyteDB database server or cluster for which Debezium is capturing changes. This name must be unique, as it's also used to form the Kafka topic. | -| database.streamid | N/A | Stream ID created using [yb-admin](../../../admin/yb-admin/#change-data-capture-cdc-commands) for Change data capture. | +| database.streamid | N/A | Stream ID created using [yb-admin](../../../../admin/yb-admin/#change-data-capture-cdc-commands) for Change data capture. | | table.include.list | N/A | Comma-separated list of table names and schema names, such as `public.test` or `test_schema.test_table_name`. | | table.max.num.tablets | 300 | Maximum number of tablets the connector can poll for. This should be greater than or equal to the number of tablets the table is split into. | | database.sslmode | disable | Whether to use an encrypted connection to the YugabyteDB cluster. Supported options are:
  • `disable` uses an unencrypted connection
  • `require` uses an encrypted connection and fails if it can't be established
  • `verify-ca` uses an encrypted connection, verifies the server TLS certificate against the configured Certificate Authority (CA) certificates, and fails if no valid matching CA certificates are found.
| @@ -1044,9 +1055,9 @@ The APIs used to fetch the changes are set up to work with TLSv1.2 only. Make su If you have a YugabyteDB cluster with SSL enabled, you need to obtain the root certificate and provide the path of the file in the `database.sslrootcert` configuration property. You can follow these links to get the certificates for your universe: -* [Local deployments](../../../secure/tls-encryption/) -* [YugabyteDB Anywhere](../../../yugabyte-platform/security/enable-encryption-in-transit/#connect-to-a-ysql-endpoint-with-tls) -* [YugabyteDB Aeon](../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate) +* [Local deployments](../../../../secure/tls-encryption/) +* [YugabyteDB Anywhere](../../../../yugabyte-platform/security/enable-encryption-in-transit/#connect-to-a-ysql-endpoint-with-tls) +* [YugabyteDB Aeon](../../../../yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#download-your-cluster-certificate) {{< /note >}} @@ -1063,7 +1074,7 @@ Advanced connector configuration properties: | time.precision.mode | adaptive | Time, date, and timestamps can be represented with different kinds of precision:

`adaptive` captures the time and timestamp values exactly as in the database using millisecond precision values based on the database column's type.

`adaptive_time_microseconds` captures the date, datetime and timestamp values exactly as in the database using millisecond precision values based on the database column's type. An exception is `TIME` type fields, which are always captured as microseconds.

`connect` always represents time and timestamp values by using Kafka Connect's built-in representations for Time, Date, and Timestamp, which use millisecond precision regardless of the database columns' precision. See temporal values. | | decimal.handling.mode | double | The `precise` mode is not currently supported.

`double` maps all the numeric, double, and money types as Java double values (FLOAT64).

`string` represents the numeric, double, and money types as their string-formatted form.

| | binary.handling.mode | hex | `hex` is the only supported mode. All binary strings are converted to their respective hex format and emitted as their string representation . | -| interval.handling.mode | numeric | Specifies how the connector should handle values for interval columns:

`numeric` represents intervals using approximate number of microseconds.

`string` represents intervals exactly by using the string pattern representation
`PYMDTHMS`.
For example: P1Y2M3DT4H5M6.78S. See [YugabyteDB data types](../../../api/ysql/datatypes/). | +| interval.handling.mode | numeric | Specifies how the connector should handle values for interval columns:

`numeric` represents intervals using approximate number of microseconds.

`string` represents intervals exactly by using the string pattern representation
`PYMDTHMS`.
For example: P1Y2M3DT4H5M6.78S. See [YugabyteDB data types](../../../../api/ysql/datatypes/). | | transaction.topic | `${database.server.name}`
`.transaction` | Controls the name of the topic to which the connector sends transaction metadata messages. The placeholder `${database.server.name}` can be used for referring to the connector's logical name; defaults to `${database.server.name}.transaction`, for example `dbserver1.transaction`. | | provide.transaction.metadata | `false` | Determines whether the connector generates events with transaction boundaries and enriches change event envelopes with transaction metadata. Specify `true` if you want the connector to do this. See [Transaction metadata](#transaction-metadata) for details. | | skipped.operations | N/A | A comma-separated list of operation types to be skipped during streaming. The types are `c` for insert/create operations, `u` for update operations, and `d` for delete operations. By default, no operations are skipped. | @@ -1161,7 +1172,7 @@ The connector publishes metadata that can be used to distinguish transaction bou ### Prerequisites -* Create the Stream ID should in the `EXPLICIT` checkpointing mode. For more information, see [yb-admin create\_change\_data_stream](../../../admin/yb-admin#create-change-data-stream). +* Create the Stream ID should in the `EXPLICIT` checkpointing mode. For more information, see [yb-admin create\_change\_data_stream](../../../../admin/yb-admin#create-change-data-stream). * You should always run the connector with a single task, that is, `tasks.max` should always be set to 1. ### Known limitations @@ -1196,7 +1207,7 @@ In case one of the tablet servers crashes, the replicas on other YB-TServer node ### YugabyteDB server failures -In case of YugabyteDB server failures, the Debezium YugabyteDB connector will try for a configurable (using a [flag](../../../reference/configuration/yb-tserver/#change-data-capture-cdc-flags)) amount of time for the availability of the YB-TServer and will stop if the cluster cannot start. When the cluster is restarted, the connector can be run again and it will start processing the changes with the committed checkpoint. +In case of YugabyteDB server failures, the Debezium YugabyteDB connector will try for a configurable (using a [flag](../../../../reference/configuration/yb-tserver/#change-data-capture-cdc-flags)) amount of time for the availability of the YB-TServer and will stop if the cluster cannot start. When the cluster is restarted, the connector can be run again and it will start processing the changes with the committed checkpoint. ### Connector unable to find table association with stream ID diff --git a/docs/content/preview/explore/fault-tolerance/_index.md b/docs/content/preview/explore/fault-tolerance/_index.md index 6e56a5008f6b..f5503788a4b4 100644 --- a/docs/content/preview/explore/fault-tolerance/_index.md +++ b/docs/content/preview/explore/fault-tolerance/_index.md @@ -18,7 +18,7 @@ type: indexpage showRightNav: true --- -Resiliency, in the context of cloud databases, refers to the ability to withstand and recover from various types of failures, ranging from hardware malfunctions and software bugs to network outages and natural disasters. A resilient database system is designed to maintain data integrity, accessibility, and continuity of operations, even in the face of adverse events. Achieving resilience in cloud databases requires a multi-faceted approach, involving robust architectural design, effective data replication and backup strategies, load balancing, failover mechanisms, and comprehensive monitoring and incident response procedures. +Resiliency, in the context of cloud databases, refers to the ability to withstand and recover from various types of failures. These can range from hardware malfunctions and software bugs to network outages and natural disasters. A resilient database system is designed to maintain data integrity, accessibility, and continuity of operations, even in the face of adverse events. Achieving resilience in cloud databases requires a multi-faceted approach, involving robust architectural design, effective data replication and backup strategies, load balancing, failover mechanisms, and comprehensive monitoring and incident response procedures. YugabyteDB has been designed ground up to be resilient. YugabyteDB can continuously serve requests in the event of planned or unplanned outages, such as system upgrades and outages related to a node, availability zone, or region. YugabyteDB's High availability is achieved through a combination of distributed architecture, data replication, consensus algorithms, automatic rebalancing, and failure detection mechanisms, ensuring that the database remains available, consistent, and resilient to failures of fault domains. diff --git a/docs/content/preview/integrations/cdc/debezium.md b/docs/content/preview/integrations/cdc/debezium.md index 6d62c15f8616..a4b955587759 100644 --- a/docs/content/preview/integrations/cdc/debezium.md +++ b/docs/content/preview/integrations/cdc/debezium.md @@ -154,7 +154,7 @@ Do the following: }' ``` -For a list of all the configuration options provided with the Debezium YugabyteDB connector, see [Connector configuration properties](../../../explore/change-data-capture/debezium-connector-yugabytedb/#connector-configuration-properties). +For a list of all the configuration options provided with the Debezium YugabyteDB connector, see [Connector configuration properties](../../../explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb/#connector-configuration-properties). {{< tip title="TRUNCATE tables when CDC is enabled" >}} diff --git a/docs/content/preview/reference/configuration/yb-tserver.md b/docs/content/preview/reference/configuration/yb-tserver.md index fff0e4835bca..e1b2381d05cd 100644 --- a/docs/content/preview/reference/configuration/yb-tserver.md +++ b/docs/content/preview/reference/configuration/yb-tserver.md @@ -855,6 +855,14 @@ Default: `-1` (disables logging statement durations) Specifies the lowest YSQL message level to log. +##### --ysql_output_buffer_size + +Size of YSQL layer output buffer, in bytes. YSQL buffers query responses in this output buffer until either a buffer flush is requested by the client or the buffer overflows. + +As long as no data has been flushed from the buffer, the database can retry queries on retryable errors. For example, you can increase the size of the buffer so that YSQL can retry [read restart errors](../../../architecture/transactions/read-restart-error). + +Default: `262144` (256kB, type: int32) + ### YCQL The following flags support the use of the [YCQL API](../../../api/ycql/): diff --git a/docs/content/preview/releases/yba-releases/v2.18.md b/docs/content/preview/releases/yba-releases/v2.18.md index baa8f0faadcd..d8476c216a96 100644 --- a/docs/content/preview/releases/yba-releases/v2.18.md +++ b/docs/content/preview/releases/yba-releases/v2.18.md @@ -33,6 +33,31 @@ What follows are the release notes for all releases in the **YugabyteDB Anywhere For an RSS feed of all release series to track the latest product updates, point your feed reader to the [RSS feed for releases](../index.xml). +## v2.18.8.1 - July 18, 2024 {#v2.18.8.1} + +**Build:** `2.18.8.1-b3` + +**Third-party licenses:** [YugabyteDB](https://downloads.yugabyte.com/releases/2.18.8.1/yugabytedb-2.18.8.1-b3-third-party-licenses.html), [YugabyteDB Anywhere](https://downloads.yugabyte.com/releases/2.18.8.1/yugabytedb-anywhere-2.18.8.1-b3-third-party-licenses.html) + +### Download + + + +For instructions on installing YugabyteDB Anywhere, refer to [Install YugabyteDB Anywhere](../../../yugabyte-platform/install-yugabyte-platform/). + +### Bug fixes + +#### Other + +* Repairs build failure in CentOS 7 pex/yugabundle builder Docker image. PLAT-14543 + ## v2.18.8.0 - June 21, 2024 {#v2.18.8.0} **Build:** `2.18.8.0-b42` diff --git a/docs/content/preview/releases/ybdb-releases/v2.18.md b/docs/content/preview/releases/ybdb-releases/v2.18.md index 36f8aee3308c..cb8c97dfcbbc 100644 --- a/docs/content/preview/releases/ybdb-releases/v2.18.md +++ b/docs/content/preview/releases/ybdb-releases/v2.18.md @@ -33,6 +33,47 @@ What follows are the release notes for the YugabyteDB v2.18 release series. Cont For an RSS feed of all release series to track the latest product updates, point your feed reader to the [RSS feed for releases](../index.xml). +## v2.18.8.1 - July 18, 2024 {#v2.18.8.1} + +**Build:** `2.18.8.1-b3` + +**Third-party licenses:** [YugabyteDB](https://downloads.yugabyte.com/releases/2.18.8.1/yugabytedb-2.18.8.1-b3-third-party-licenses.html), [YugabyteDB Anywhere](https://downloads.yugabyte.com/releases/2.18.8.1/yugabytedb-anywhere-2.18.8.1-b3-third-party-licenses.html) + +### Downloads + + + +### Docker + +```sh +docker pull yugabytedb/yugabyte:2.18.8.1-b3 +``` + +### Improvements + +#### DocDB + +* Allows asynchronous DNS cache updating and resolution retry upon failure to reduce RPC call delays and prevent unexpected leadership changes. {{}},{{}} + ## v2.18.8.0 - June 21, 2024 {#v2.18.8.0} **Build:** `2.18.8.0-b42` diff --git a/docs/content/preview/tutorials/azure/azure-event-hubs.md b/docs/content/preview/tutorials/azure/azure-event-hubs.md index c65d08ed71a8..0a59ea932953 100644 --- a/docs/content/preview/tutorials/azure/azure-event-hubs.md +++ b/docs/content/preview/tutorials/azure/azure-event-hubs.md @@ -13,9 +13,9 @@ menu: type: docs --- -The [Azure Event Hubs](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-about) data streaming service is [Apache Kafka](https://kafka.apache.org/intro) compatible, enabling existing workloads to easily be moved to Azure. With the [Debezium Connector for YugabyteDB](../../../explore/change-data-capture/debezium-connector-yugabytedb), we can stream changes from a YugabyteDB cluster to a Kafka topic using [Kafka Connect](https://docs.confluent.io/platform/current/connect/index.html#:~:text=Kafka%20Connect%20is%20a%20tool,in%20and%20out%20of%20Kafka.). +The [Azure Event Hubs](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-about) data streaming service is [Apache Kafka](https://kafka.apache.org/intro) compatible, enabling existing workloads to easily be moved to Azure. With the [Debezium Connector for YugabyteDB](../../../explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb), we can stream changes from a YugabyteDB cluster to a Kafka topic using [Kafka Connect](https://docs.confluent.io/platform/current/connect/index.html#:~:text=Kafka%20Connect%20is%20a%20tool,in%20and%20out%20of%20Kafka.). -In this tutorial, we'll examine how the [YugabyteDB CDC](../../../explore/change-data-capture/cdc-overview/) can be used with Azure Event Hubs to stream real-time data for downstream processing. +In this tutorial, we'll examine how the [YugabyteDB CDC](../../../explore/change-data-capture/using-yugabytedb-grpc-replication/cdc-overview/) can be used with Azure Event Hubs to stream real-time data for downstream processing. In the following sections, you will: diff --git a/docs/content/preview/tutorials/cdc-tutorials/cdc-azure-event-hub.md b/docs/content/preview/tutorials/cdc-tutorials/cdc-azure-event-hub.md index 6dcd3f9768f8..a58b503724e0 100644 --- a/docs/content/preview/tutorials/cdc-tutorials/cdc-azure-event-hub.md +++ b/docs/content/preview/tutorials/cdc-tutorials/cdc-azure-event-hub.md @@ -25,7 +25,7 @@ The following table describes the components. | Component | Description | | :--- | :--- | | YugabyteDB Anywhere | Yugabyte self-managed DBaaS for deploying YugabyteDB at scale that includes YugabyteDB's self-managed, stateless CDC functionality with support for Confluent Kafka Connect. A pull-based model is used to report changes from the database Write-Ahead-Log (WAL). For more information on CDC, refer to [Change Data Capture](../../../explore/change-data-capture/). | -| [Debezium connector](../../../explore/change-data-capture/debezium-connector-yugabytedb/) | Pulls data from YugabyteDB, publishes it to Kafka, and runs in a Microsoft Azure Virtual Machine. | +| [Debezium connector](../../../explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb/) | Pulls data from YugabyteDB, publishes it to Kafka, and runs in a Microsoft Azure Virtual Machine. | | [Azure Event Hubs](https://learn.microsoft.com/en-us/azure/event-hubs/event-hubs-about) | Azure big data streaming platform and event ingestion service. | [Azure Synapse Pipelines](https://learn.microsoft.com/en-us/azure/data-factory/concepts-pipelines-activities?) | Pipelines and activities in Azure Data Factory and Azure Synapse Analytics required to construct end-to-end data-driven workflows to move and process your data. | | [ADLS Gen2](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) | Storage account | diff --git a/docs/content/preview/tutorials/cdc-tutorials/cdc-confluent-cloud.md b/docs/content/preview/tutorials/cdc-tutorials/cdc-confluent-cloud.md index 848e2002665d..a73c70dff36c 100644 --- a/docs/content/preview/tutorials/cdc-tutorials/cdc-confluent-cloud.md +++ b/docs/content/preview/tutorials/cdc-tutorials/cdc-confluent-cloud.md @@ -155,4 +155,4 @@ CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: "SCHEMA_REGISTRY_U docker-compose up ``` -1. Deploy the connector. For more information, refer to [Deployment](../../../explore/change-data-capture/debezium-connector-yugabytedb/#deployment). +1. Deploy the connector. For more information, refer to [Deployment](../../../explore/change-data-capture/using-yugabytedb-grpc-replication/debezium-connector-yugabytedb/#deployment). diff --git a/docs/content/preview/yugabyte-cloud/cloud-connect/connect-client-shell.md b/docs/content/preview/yugabyte-cloud/cloud-connect/connect-client-shell.md index 96f5360dde66..c3387f3e134e 100644 --- a/docs/content/preview/yugabyte-cloud/cloud-connect/connect-client-shell.md +++ b/docs/content/preview/yugabyte-cloud/cloud-connect/connect-client-shell.md @@ -26,7 +26,7 @@ Before you can connect a desktop client to a YugabyteDB Aeon cluster, you need t Before you can connect using a shell or other client, you need to add your computer to the cluster IP allow list. -By default, clusters deployed in a VPC do not expose any publicly-accessible IP addresses. To add public IP addresses, enable [Public Access](../../../yugabyte-cloud/cloud-secure-clusters/add-connections/#enabling-public-access) on the cluster **Settings > Network Access** tab. Alternatively, use the [Cloud shell](../connect-cloud-shell/) instead. +By default, clusters deployed in a VPC do not expose any publicly-accessible IP addresses. To add public IP addresses, enable [Public Access](../../cloud-secure-clusters/add-connections/#enabling-public-access) on the cluster **Settings > Network Access** tab. Alternatively, use the [Cloud shell](../connect-cloud-shell/) instead. For more information, refer to [IP allow list](../../cloud-secure-clusters/add-connections). diff --git a/docs/content/preview/yugabyte-cloud/cloud-connect/connect/ysql.md b/docs/content/preview/yugabyte-cloud/cloud-connect/connect/ysql.md index beee894fbfc3..f88e7143ae6e 100644 --- a/docs/content/preview/yugabyte-cloud/cloud-connect/connect/ysql.md +++ b/docs/content/preview/yugabyte-cloud/cloud-connect/connect/ysql.md @@ -14,7 +14,7 @@ To connect to a cluster using `ysqlsh`: 1. If your cluster is deployed in a VPC, choose **Private Address** if you are connecting from a peered VPC. Otherwise, choose **Public Address** (only available if you have enabled Public Access for the cluster; not recommended for production). 1. Copy the **YSQL** connection string. - The connection string includes flags specifying the host (`host`), username (`user`), database (`dbname`), and TLS settings (`sslmode` and `sslrootcert`). The command specifies that the connection will use the CA certificate you installed on your computer. For information on using other SSL modes, refer to [SSL modes in YSQL](../../../cloud-secure-clusters/cloud-authentication/#ssl-modes-in-ysql). + The connection string includes flags specifying the host (`host`), username (`user`), database (`dbname`), and TLS settings (`sslmode` and `sslrootcert`). The command specifies that the connection will use the CA certificate you installed on your computer. For information on using other SSL modes, refer to [SSL modes in YSQL](/preview/yugabyte-cloud/cloud-secure-clusters/cloud-authentication/#ssl-modes-in-ysql). Here's an example of the generated `ysqlsh` command: diff --git a/docs/content/preview/yugabyte-cloud/cloud-secure-clusters/_index.md b/docs/content/preview/yugabyte-cloud/cloud-secure-clusters/_index.md index 8c978512f531..9b3cff12603c 100644 --- a/docs/content/preview/yugabyte-cloud/cloud-secure-clusters/_index.md +++ b/docs/content/preview/yugabyte-cloud/cloud-secure-clusters/_index.md @@ -19,7 +19,7 @@ YugabyteDB Aeon clusters include the following security features: | :--- | :--- | | [Network authorization](add-connections/) | Access to YugabyteDB Aeon clusters is limited to IP addresses that you explicitly allow using IP allow lists.
You can further enhance security and lower network latencies by deploying clusters in a [virtual private cloud (VPC) network](../cloud-basics/cloud-vpcs/). | | [Database authorization](cloud-users/) | YugabyteDB uses [role-based access control](cloud-users/) for database authorization. Using the default database admin user that is created when a cluster is deployed, you can [add additional roles and users](add-users/) to provide custom access to database resources to other team members and database clients. | -| [Encryption in transit](cloud-authentication/) | YugabyteDB Aeon uses encryption-in-transit for client-server and intra-node connectivity. | +| [Encryption in transit](cloud-authentication/) | YugabyteDB Aeon uses encryption in transit for client-server and intra-node connectivity. | | [Encryption at rest](managed-ear/) | Data at rest, including clusters and backups, is AES-256 encrypted using native cloud provider technologies: S3 and EBS volume encryption for AWS, Azure disk encryption, and server-side and persistent disk encryption for GCP. For additional security, you can encrypt your clusters using keys that you manage yourself. | | [Auditing](cloud-activity/) | YugabyteDB Aeon provides detailed auditing of activity on your account, including cluster creation, changes to clusters, changes to IP allow lists, backup activity, billing, access history, and more. | diff --git a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/high-availability.md b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/high-availability.md index 2efd9cdbb21a..513cf31a8307 100644 --- a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/high-availability.md +++ b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/high-availability.md @@ -144,7 +144,7 @@ For example, if your metrics retention is 14 days on your active instance, and y After HA is operational, it is recommended that you enable certificate validation to improve security of communication between the active and any standby instances. Enable certificate validation as follows: -1. Add certificates for the active and all standbys to the active instance [trust store](../../security/enable-encryption-in-transit/#add-certificates-to-your-trust-store). +1. Add certificates for the active and all standbys to the active instance [trust store](../../security/enable-encryption-in-transit/trust-store/). - If YBA was set up to use a custom server certificate, locate the corresponding Certificate Authority (CA) certificate. - If YBA was set up to use automatically generated self-signed certificates and you installed YBA using YBA Installer, locate the CA certificate at `/opt/yugabyte/data/yba-installer/certs/ca_cert.pem` on both the YBA active and standby instances. (If you configured a custom install root, replace `/opt/yugabyte` with the path you configured.) diff --git a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md index f30d09ba4f3f..e6e34d2705ff 100644 --- a/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md +++ b/docs/content/preview/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md @@ -94,6 +94,7 @@ You configure OIDC as follows: - In the **Scope** field, enter your identity provider OIDC scope that is allowed to be requested. This field accepts a space-separated list of values. If left blank, all scopes will be considered. - In the **Email Attribute** field, enter the OIDC scope containing the user email identifier. This field accepts a case-sensitive custom configuration. Typically, this field is left blank. - If you have configured OIDC to use [refresh tokens](https://openid.net/specs/openid-connect-core-1_0.html#RefreshTokens), in the **Refresh Token URL** field, enter the URL of the refresh token endpoint. + - If you have configured [OIDC enhancements](../../security/authentication/oidc-authentication-aad/#enable-oidc-enhancements), you can select the **Display JWT token on login** option to allow users to access their JWT from the YugabyteDB Anywhere sign in page. See [Set up OIDC with Azure AD on YugabyteDB Anywhere](../../security/authentication/oidc-authentication-aad/#set-up-oidc-with-azure-ad-on-yugabytedb-anywhere). 1. You can assign the default [role](../anywhere-rbac/#built-in-roles) for OIDC users to be ReadOnly or ConnectOnly. diff --git a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/aws.md b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/aws.md index c652473698c7..8b6e9874224f 100644 --- a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/aws.md +++ b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/aws.md @@ -101,7 +101,7 @@ Enter a Provider name. The Provider name is an internal tag used for organizing **Credential Type**. YBA requires the ability to create VMs in AWS. To do this, you can do one of the following: -- **Specify Access ID and Secret Key** - Create an AWS Service Account with the required permissions (refer to [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes/)), and provide your AWS Access Key ID and Secret Access Key. +- **Specify Access ID and Secret Key** - Create an AWS Service Account with the required permissions (refer to [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes-aws/)), and provide your AWS Access Key ID and Secret Access Key. - **Use IAM Role from this YBA host's instance** - Provision the YBA VM instance with an IAM role that has sufficient permissions by attaching an [IAM role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) to the YBA VM in the **EC2** tab. This option is only available if YBA is installed on AWS. **Use AWS Route 53 DNS Server**. Choose whether to use the cloud DNS Server / load balancer for universes deployed using this provider. Generally, SQL clients should prefer to use [smart client drivers](../../../drivers-orms/smart-drivers/) to connect to cluster nodes, rather than load balancers. However, in some cases (for example, if no smart driver is available in the language), you may use a DNS Server or load-balancer. The DNS Server acts as a load-balancer that routes clients to various nodes in the database universe. YBA integrates with [Amazon Route53](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/Welcome.html) to provide managed Canonical Name (CNAME) entries for your YugabyteDB universes, and automatically updates the DNS entry as nodes get created, removed, or undergo maintenance. diff --git a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/azure.md b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/azure.md index 78f8ab20a5d3..edc806bd49a0 100644 --- a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/azure.md +++ b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/azure.md @@ -51,13 +51,13 @@ When deploying a universe, YBA uses the provider configuration settings to do th ## Prerequisites -You need to add the following Azure cloud provider credentials via YBA: +You need to add the following Azure cloud provider credentials: +- Application client ID and (if using credentials) client secret +- Resource group name - Subscription ID - Tenant ID - SSH port and user -- Application client ID and secret -- Resource group YBA uses the credentials to automatically provision and deprovision YugabyteDB instances. @@ -107,11 +107,29 @@ Enter a Provider name. The Provider name is an internal tag used for organizing ### Cloud Info -- **Client ID** represents the [ID of an application](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#option-2-create-a-new-application-secret) registered in your Azure Active Directory. -- **Client Secret** represents the secret of an application registered in your Azure Active Directory. You need to enter the `Value` of the secret (not the `Secret ID`). -- **Resource Group** represents the group in which YugabyteDB nodes compute and network resources are created. Your Azure Active Directory application (client ID and client secret) needs to have `Network Contributor` and `Virtual Machine Contributor` roles assigned for this resource group. -- **Subscription ID** is required for cost management. The virtual machine resources managed by YBA are tagged with this subscription. -- **Tenant ID** represents the Azure Active Directory tenant ID which belongs to an active subscription. To find your tenant ID, follow instructions provided in [How to find your Azure Active Directory tenant ID](https://learn.microsoft.com/en-us/azure/active-directory/fundamentals/how-to-find-tenant). +Enter the following details of your Azure cloud account, as described in [Azure cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes-azure/). + +#### Client ID + +Provide the ID of the application you registered with the Microsoft Identity Platform. + +#### Credential type + +If you [added credentials](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app?tabs=client-secret#add-credentials) in the form of a client secret to your registered application: + +1. Select **Specify Client Secret**. +1. Enter the Client Secret of the application associated with the Client ID you provided. You need to enter the `Value` of the secret (not the `Secret ID`). + +If you are using the [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/qs-configure-portal-windows-vm) of the Azure VM hosting YugabyteDB Anywhere to authenticate: + +- Select **Use Managed Identity from this YBA host's instance**. + +#### Additional fields + +- **Resource Group** is the name of the resource group you created for your application, and in which YugabyteDB node compute and network resources will be created. +- **Subscription ID** is required for cost management. The virtual machine resources managed by YBA are tagged with this subscription. To get the subscription ID, open Subscriptions in Azure portal and find your subscription. Then, copy the Subscription ID. +- Optionally, if you created a different resource group for your network interfaces, provide the **Network Resource Group** name and the associated **Network Subscription ID**. If you do not provide a Network Resource Group or Subscription ID, network resources will be created in the default resource group. +- **Tenant ID** represents the tenant ID which belongs to an active subscription. To find your tenant ID, follow instructions provided in [How to find your Microsoft Entra tenant ID](https://learn.microsoft.com/en-us/entra/fundamentals/how-to-find-tenant). - **Private DNS zone** lets you use a custom domain name for the nodes in your universe. For details and instructions, see [Define a private DNS zone](#define-a-private-dns-zone). ### Regions diff --git a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/gcp.md b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/gcp.md index 9b9fc45bf950..c421e72c24e0 100644 --- a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/gcp.md +++ b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/gcp.md @@ -110,7 +110,7 @@ Enter a Provider name. The Provider name is an internal tag used for organizing ### Cloud Info -If your YBA instance is not running inside GCP, you need to supply YBA with credentials to the desired GCP project by uploading a configuration file. To do this, set **Credential Type** to **Upload Service Account config** and proceed to upload the JSON file that you obtained when you created your service account, as described in [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes/). +If your YBA instance is not running inside GCP, you need to supply YBA with credentials to the desired GCP project by uploading a configuration file. To do this, set **Credential Type** to **Upload Service Account config** and proceed to upload the JSON file that you obtained when you created your service account, as described in [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes-gcp/). If your YBA instance is running inside GCP, the preferred method for authentication to the GCP APIs is to add a service account role to the GCP instance running YBA and then configure YBA to use the instance's service account. To do this, set **Credential Type** to **Use service account from this YBA host's instance**. diff --git a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/kubernetes.md b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/kubernetes.md index 4c38e5e644ae..fcdd0bcaea50 100644 --- a/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/kubernetes.md +++ b/docs/content/preview/yugabyte-platform/configure-yugabyte-platform/kubernetes.md @@ -122,7 +122,7 @@ Continue configuring your Kubernetes provider by clicking **Add region** and com 1. Complete the **Overrides** field using one of the provided [options](#overrides). If you do not specify anything, YBA uses defaults specified inside the Helm chart. For additional information, see [Open source Kubernetes](../../../deploy/kubernetes/single-zone/oss/helm-chart/). -1. If you are using [Kubernetes cert-manager](https://cert-manager.io) to manage TLS certificates, specify the issuer type and enter the issuer name. For more information, refer to [Enable encryption in transit](../../security/enable-encryption-in-transit/#kubernetes-cert-manager). +1. If you are using [Kubernetes cert-manager](https://cert-manager.io) to manage TLS certificates, specify the issuer type and enter the issuer name. For more information, refer to [Enable encryption in transit](../../security/enable-encryption-in-transit/add-certificate-kubernetes/). If required, add a new zone by clicking **Add Zone**, as your configuration may have multiple zones. diff --git a/docs/content/preview/yugabyte-platform/create-deployments/connect-to-universe.md b/docs/content/preview/yugabyte-platform/create-deployments/connect-to-universe.md index 004189ced4a9..676fac33bf7c 100644 --- a/docs/content/preview/yugabyte-platform/create-deployments/connect-to-universe.md +++ b/docs/content/preview/yugabyte-platform/create-deployments/connect-to-universe.md @@ -21,13 +21,21 @@ You can connect to the database on a universe in the following ways: ## Download the universe certificate -If the universe uses encryption in transit, to connect you need to first download the universe TLS root certificate. Do the following: +If the universe uses Client-to-Node encryption in transit, to connect you need to first download the universe TLS certificate. Do the following: 1. Navigate to **Configs > Security > Encryption in Transit**. -1. Find the certificate for your universe in the list and click **Actions** and download the certificate. +1. Find your universe in the list. -For more information on connecting to TLS-enabled universes, refer to [Connect to clusters](../../security/enable-encryption-in-transit/#connect-to-clusters). +1. Click **Actions** and choose **Download Root CA Cert**. + + This downloads the `root.crt` file. + +For information on connecting using a client shell using this certificate, see [Connect from your desktop](#connect-from-your-desktop). + +To use TLS to connect an application, refer to the [driver documentation](../../../reference/drivers/). If you are using a PostgreSQL JDBC driver to connect to YugabyteDB, you can also refer to [Configuring the client](https://jdbc.postgresql.org/documentation/head/ssl-client.html) for more details. + +If you are using PostgreSQL/YugabyteDB JDBC driver with SSL, you need to convert the certificates to DER format. To do this, you need to perform only steps 6 and 7 from [Set up SSL certificates for Java applications](../../../reference/drivers/java/postgres-jdbc-reference/#set-up-ssl-certificates-for-java-applications) section after downloading the certificates. ## Connect to a universe node @@ -118,21 +126,9 @@ curl --location --request PUT 'http:///api/v1/customers//runt ### Prerequisites -- If you are using a Yugabyte client shell, ensure you are running the latest versions of the shells (Yugabyte Client 2.6 or later). - - You can download using the following command on Linux or macOS: - - ```sh - $ curl -sSL https://downloads.yugabyte.com/get_clients.sh | bash - ``` - - Windows client shells require Docker. For example: - - ```sh - docker run -it yugabytedb/yugabyte-client ysqlsh -h -p - ``` +- If you are using [ysqlsh](../../../admin/ysqlsh/) or [ycqlsh](../../../admin/ycqlsh/), ensure you are running the latest versions of the shells. -- If your universe has TLS/SSL (encryption in-transit) enabled, you need to [download the certificate](#download-the-universe-certificate) to your computer. +- If your universe has Client-to-Node encryption in transit enabled, you need to [download the certificate](#download-the-universe-certificate) to your computer. - The host address of an endpoint on your universe. @@ -181,7 +177,7 @@ Replace the following: - `` with the IP address of an endpoint on your universe. - `` with your database username. - `yugabyte` with the database name, if you're connecting to a database other than the default (yugabyte). -- `` with the path to the root certificate on your computer. +- `` with the path to the universe root certificate you downloaded to your computer. To load sample data and explore an example using ysqlsh, follow the instructions in [Install the Retail Analytics sample database](../../../sample-data/retail-analytics/#install-the-retail-analytics-sample-database). @@ -203,7 +199,7 @@ Replace the following: - `` with the IP address of an endpoint on your universe. - `` with your database username. -- `` with the path to the root certificate on your computer. +- `` with the path to the universe root certificate you downloaded to your computer. @@ -224,7 +220,7 @@ Replace the following: - `` with the IP address of an endpoint on your universe. - `` with your database username. - `yugabyte` with the database name, if you're connecting to a database other than the default (yugabyte). -- `` with the path to the root certificate on your computer. +- `` with the path to the universe root certificate you downloaded to your computer. @@ -358,6 +354,7 @@ ycqlsh> SELECT * FROM ybdemo_keyspace.cassandrakeyvalue LIMIT 5; ## Learn more +- [Securing YugabyteDB: Client-to-Server Encryption in Transit](https://www.yugabyte.com/blog/securing-yugabytedb-client-to-server-encryption/#verification-of-server-certificates) - [ysqlsh](../../../admin/ysqlsh/) — Overview of the command line interface (CLI), syntax, and commands. - [YSQL API](../../../api/ysql/) — Reference for supported YSQL statements, data types, functions, and operators. - [ycqlsh](../../../admin/ycqlsh/) — Overview of the command line interface (CLI), syntax, and commands. diff --git a/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md b/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md index 7397c8cc116a..074366e4b4c7 100644 --- a/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md +++ b/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md @@ -71,8 +71,8 @@ Complete the **Security Configurations** section as follows: - **Enable YSQL Auth** - specify whether or not to enable the YSQL password authentication. - **Enable YCQL** - specify whether or not to enable the YCQL API endpoint for running Cassandra-compatible workloads. This setting is enabled by default. - **Enable YCQL Auth** - specify whether or not to enable the YCQL password authentication. -- **Enable Node-to-Node TLS** - specify whether or not to enable encryption-in-transit for communication between the database servers. This setting is enabled by default. -- **Enable Client-to-Node TLS** - specify whether or not to enable encryption-in-transit for communication between clients and the database servers. This setting is enabled by default. +- **Enable Node-to-Node TLS** - specify whether or not to enable encryption in transit for communication between the database servers. This setting is enabled by default. +- **Enable Client-to-Node TLS** - specify whether or not to enable encryption in transit for communication between clients and the database servers. This setting is enabled by default. - **Root Certificate** - select an existing security certificate or create a new one. - **Enable Encryption at Rest** - specify whether or not to enable encryption for data stored on the tablet servers. This setting is disabled by default. @@ -80,7 +80,7 @@ Complete the **Security Configurations** section as follows: Complete the **Advanced** section as follows: -- In the **DB Version** field, specify the YugabyteDB version. The default is either the same as the YugabyteDB Anywhere version or the latest YugabyteDB version available for YugabyteDB Anywhere. +- In the **DB Version** field, specify the YugabyteDB version. The default is either the same as the YugabyteDB Anywhere version or the latest YugabyteDB version available for YugabyteDB Anywhere. If the version you want to add is not listed, you can add it to YugabyteDB Anywhere. Refer to [Manage YugabyteDB releases](../../manage-deployments/ybdb-releases/). - Use the **Enable IPV6** field to specify whether or not you want to use IPV6 networking for connections between database servers. This setting is disabled by default. - Use the **Enable Public Network Access** field to specify whether or not to assign a load balancer or nodeport for connecting to the database endpoints over the internet. This setting is disabled by default. diff --git a/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone.md b/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone.md index edc814f8f993..61e6b6bb6c76 100644 --- a/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone.md +++ b/docs/content/preview/yugabyte-platform/create-deployments/create-universe-multi-zone.md @@ -84,19 +84,42 @@ Specify the instance to use for the universe nodes: ### Security Configurations +#### IP Settings + To enable public access to the universe, select the **Assign Public IP** option. -Enable the YSQL and YCQL endpoints and database authentication. You can also enable and disable authentication after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. +#### Authentication Settings + +Enable the YSQL and YCQL endpoints and database authentication. Enter the password to use for the default database admin superuser (yugabyte for YSQL, and cassandra for YCQL). For more information, refer to [Database authorization](../../security/authorization-platform/). -Enable encryption in transit to encrypt universe traffic. Refer to [Enable encryption in transit](../../security/enable-encryption-in-transit/). +You can also enable and disable the API endpoints and authentication after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. + +By default, the API endpoints use ports 5433 (YSQL) and 9042 (YCQL). You can [customize these ports](#advanced-configuration), and, after deployment, you can modify the YCQL API and admin UI endpoint ports. To change YCQL ports, navigate to your universe, click **Actions**, choose **Edit YCQL Configuration**, and select the **Override YCQL Default Ports** option. + +#### Encryption Settings + +Enable encryption in transit to encrypt universe traffic. You can enable the following: + +- **Node-to-Node TLS** to encrypt traffic between universe nodes. +- **Client-to-Node TLS** to encrypt traffic between universe nodes and external clients. + + Note that if you want to enable Client-to-Node encryption, you first must enable Node-to-Node encryption. + +Encryption requires a certificate. YugabyteDB Anywhere can generate a self-signed certificate automatically, or you can use your own certificate. + +To use your own, you must first add it to YugabyteDB Anywhere; refer to [Add certificates](../../security/enable-encryption-in-transit/add-certificate-self/). + +To have YugabyteDB Anywhere generate a certificate for the universe, use the default **Root Certificate** setting of **Create New Certificate**. To use a certificate you added or a previously generated certificate, select it from the **Root Certificate** menu. + +For more information on using and managing certificates, refer to [Encryption in transit](../../security/enable-encryption-in-transit/). -Enable encryption at rest to encrypt the universe data. Refer to [Enable encryption at rest](../../security/enable-encryption-at-rest/). +To encrypt the universe data, select the **Enable encryption at rest** option and select the [KMS configuration](../../security/create-kms-config/aws-kms/) to use for encryption. For more information, refer to [Encryption at rest](../../security/enable-encryption-at-rest/). ### Advanced Configuration -Choose the version of YugabyteDB to install on the nodes. +Choose the version of YugabyteDB to install on the nodes. If the version you want to add is not listed, you can add it to YugabyteDB Anywhere. Refer to [Manage YugabyteDB releases](../../manage-deployments/ybdb-releases/). The access key is the SSH key that is created in the provider. Usually, each provider has its own access key, but if you are reusing keys across providers, they are listed here. @@ -104,7 +127,7 @@ For AWS providers, you can assign an ARN to the nodes in the universe; this allo To use cron instead of systemd for managing nodes, you can disable systemd services. This not recommended. -To customize the ports used for the universe, select the **Override Deployment Ports** option and enter the custom port numbers for the services you want to change. +To customize the [ports used for the universe](../../prepare/networking/), select the **Override Deployment Ports** option and enter the custom port numbers for the services you want to change. Any value from `1024` to `65535` is valid, as long as it doesn't conflict with anything else running on nodes to be provisioned. ### G-Flags diff --git a/docs/content/preview/yugabyte-platform/manage-deployments/edit-universe.md b/docs/content/preview/yugabyte-platform/manage-deployments/edit-universe.md index da7a4534c467..f6a7b77cffb1 100644 --- a/docs/content/preview/yugabyte-platform/manage-deployments/edit-universe.md +++ b/docs/content/preview/yugabyte-platform/manage-deployments/edit-universe.md @@ -40,7 +40,7 @@ YugabyteDB Anywhere performs these modifications through the [YB-Masters](../../ Note that you can't change the replication factor of a universe. -To change the number of nodes of universes created with an on-premises cloud provider and secured with third-party certificates obtained from external certification authorities, follow the instructions in [Expand the universe](../../security/enable-encryption-in-transit#expand-the-universe). +To change the number of nodes of universes created with an on-premises cloud provider and secured with third-party certificates obtained from external certification authorities, you must first add the certificates to the nodes you will add to the universe. Refer to [Add certificates](../../security/enable-encryption-in-transit/add-certificate-ca/). Ensure that the certificates are signed by the same external CA and have the same root certificate. In addition, ensure that you copy the certificates to the same locations that you originally used when creating the universe. ### Smart resize diff --git a/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md b/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md index 972439ea0698..af8182a0bb25 100644 --- a/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md +++ b/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md @@ -123,8 +123,8 @@ If using a service account, record the following two pieces of information about | Save for later | To configure | | :--- | :--- | -| Access key ID | [AWS cloud provider](../../../configure-yugabyte-platform/aws/) | -| Secret Access Key | [AWS cloud provider](../../../configure-yugabyte-platform/aws/) | +| Access key ID | [AWS provider configuration](../../../configure-yugabyte-platform/aws/) | +| Secret Access Key | | ### IAM role @@ -178,4 +178,4 @@ If you will be using your own custom SSH keys, then ensure that you have them wh | Save for later | To configure | | :--- | :--- | -| Custom SSH keys | [AWS provider](../../../configure-yugabyte-platform/kubernetes/) | +| Custom SSH keys | [AWS provider configuration](../../../configure-yugabyte-platform/kubernetes/) | diff --git a/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md b/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md index 6ca1477ddcea..1bf892f14bc0 100644 --- a/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md +++ b/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md @@ -52,30 +52,46 @@ The more permissions that you can provide, the more YBA can automate. ## Azure -The following permissions are required for the Azure resource group where you will deploy. +### Application and resource group + +YugabyteDB Anywhere requires cloud permissions to create VMs. You grant YugabyteDB Anywhere access to manage Azure resources such as VMs by registering an application in the Azure portal so the Microsoft identity platform can provide authentication and authorization services for your application. Registering your application establishes a trust relationship between your application and the Microsoft identity platform. + +In addition, your Azure application needs to have a [resource group](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups) with the following permissions: ```sh Network Contributor Virtual Machine Contributor ``` -To grant the required access, you can do one of the following: +You can optionally create a resource group for network resources if you want network interfaces to be created separately. The network resource group must have the `Network Contributor` permission. + +For more information on registering applications, refer to [Register an application with the Microsoft identity platform](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app?tabs=certificate) in the Microsoft Entra documentation. + +For more information on roles, refer to [Assign Azure roles using the Azure portal](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) in the Microsoft Azure documentation. + +### Credentials + +YugabyteDB Anywhere can authenticate with Azure using one of the following methods: + +- [Add credentials](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app?tabs=client-secret#add-credentials), in the form of a client secret, to your registered application. -- [Register an application](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app) in the Azure portal so the Microsoft identity platform can provide authentication and authorization services for your application. Registering your application establishes a trust relationship between your application and the Microsoft identity platform. + For information on creating client secrets, see [Create a new client secret](https://learn.microsoft.com/en-us/entra/identity-platform/howto-create-service-principal-portal#option-3-create-a-new-client-secret) in the Microsoft Entra documentation. -- [Assign a managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/qs-configure-portal-windows-vm) to the Azure VM hosting YugabyteDB Anywhere. +- [Assign a managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/qs-configure-portal-windows-vm) to the Azure VM hosting YugabyteDB Anywhere. Azure will use the managed identity assigned to your instance to authenticate. -For information on assigning roles to applications, see [Assign a role to an application](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#assign-a-role-to-the-application); and assigning roles for managed identities, see [Assign Azure roles using the Azure portal](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) in the Microsoft Azure documentation. + For information on assigning roles for managed identities, see [Assign Azure roles using the Azure portal](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) in the Microsoft Azure documentation. -If you are registering an application, record the following information about your service account. You will need to provide this information later to YBA. +Record the following information about your service account. You will need to provide this information later when creating an Azure provider configuration. | Save for later | To configure | | :--- | :--- | -| **Service account details** | [Azure cloud provider](../../../configure-yugabyte-platform/azure/) | +| **Service account details** | [Azure provider configuration](../../../configure-yugabyte-platform/azure/) | | Client ID: | | -| Client Secret: | | +| Client Secret:
(not required when using managed identity) | | | Resource Group: | | | Subscription ID: | | +| (Optional) Network Resource Group: | | +| (Optional) Network Subscription ID: | | | Tenant ID: | | ## Managing SSH keys for VMs @@ -89,4 +105,4 @@ If you will be using your own custom SSH keys, then ensure that you have them wh | Save for later | To configure | | :--- | :--- | -| Custom SSH keys | [Azure provider](../../../configure-yugabyte-platform/azure/) | +| Custom SSH keys | [Azure provider configuration](../../../configure-yugabyte-platform/azure/) | diff --git a/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md b/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md index 502d178bc1d2..a54c68f08097 100644 --- a/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md +++ b/docs/content/preview/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md @@ -70,7 +70,7 @@ Then use one of the following methods: | Save for later | To configure | | :--- | :--- | -| Service account JSON | [GCP cloud provider](../../../configure-yugabyte-platform/gcp/) | +| Service account JSON | [GCP provider configuration](../../../configure-yugabyte-platform/gcp/) | ## Managing SSH keys for VMs @@ -83,4 +83,4 @@ If you will be using your own custom SSH keys, then ensure that you have them wh | Save for later | To configure | | :--- | :--- | -| Custom SSH keys | [GCP provider](../../../configure-yugabyte-platform/gcp/) | +| Custom SSH keys | [GCP provider configuration](../../../configure-yugabyte-platform/gcp/) | diff --git a/docs/content/preview/yugabyte-platform/prepare/networking.md b/docs/content/preview/yugabyte-platform/prepare/networking.md index 4049dc352211..f41f43bb2442 100644 --- a/docs/content/preview/yugabyte-platform/prepare/networking.md +++ b/docs/content/preview/yugabyte-platform/prepare/networking.md @@ -18,7 +18,7 @@ YugabyteDB Anywhere (YBA) needs to be able to access nodes that will be used to ![YugabyteDB Anywhere network and ports](/images/yb-platform/prepare/yba-networking.png) -The following ports need to be open. (The default port numbers can be customized.) +The following ports need to be open. | From | To | Requirements | | :--- | :--- | :--- | diff --git a/docs/content/preview/yugabyte-platform/security/_index.md b/docs/content/preview/yugabyte-platform/security/_index.md index 67b92d4da7fd..297937b36426 100644 --- a/docs/content/preview/yugabyte-platform/security/_index.md +++ b/docs/content/preview/yugabyte-platform/security/_index.md @@ -5,6 +5,10 @@ linkTitle: Security description: Secure YugabyteDB Anywhere and YugabyteDB universes. image: /images/section_icons/index/secure.png headcontent: Secure YugabyteDB Anywhere and your YugabyteDB universes. +aliases: + - /preview/yugabyte-platform/security/network-security/ + - /preview/yugabyte-platform/security/customize-ports/ + - /preview/yugabyte-platform/security/security-checklist-yp/ menu: preview_yugabyte-platform: parent: yugabytedb-anywhere @@ -13,48 +17,51 @@ weight: 660 type: indexpage --- -{{}} - - {{}} - - {{}} - - {{}} - - {{}} - - {{}} - - {{}} - - {{}} - -{{}} +You can apply security measures to protect your YugabyteDB Anywhere instance and YugabyteDB universes. + +## Network security + +You need to ensure that YugabyteDB Anywhere and the database run in a trusted network environment. You should restrict machine and port access, based on the following guidelines: + +- Servers running YugabyteDB services are directly accessible only by YugabyteDB Anywhere, servers running the application, and database administrators. +- Only YugabyteDB Anywhere and servers running applications can connect to YugabyteDB services on the RPC ports. Access to the YugabyteDB ports should be denied to everybody else. + +{{}} +For information on networking and port requirements, refer to [Networking](../prepare/networking/). +{{}} + +## Database authentication + +Authentication requires that all clients provide valid credentials before they can connect to a YugabyteDB universe. The authentication credentials in YugabyteDB are stored internally in the YB-Master system tables. The authentication mechanisms available to users depends on what is supported and exposed by the YSQL and YCQL APIs. + +You enable authentication for the YSQL and YCQL APIs when you deploy a universe. See [Enable database authentication](authorization-platform/#enable-database-authentication). + +YugabyteDB Anywhere and YugabyteDB also support LDAP and OIDC for managing authentication. See [Database authentication](authentication/). + +For more information on authentication in YugabyteDB, see [Enable authentication](../../secure/enable-authentication/). + +## Role-based access control + +Roles can be assigned to grant users only the essential privileges based on the operations they need to perform in YugabyteDB Anywhere, and in YugabyteDB universes. + +To manage access to your YugabyteDB Anywhere instance, typically you create a [Super Admin role first](../install-yugabyte-platform/create-admin-user/). The Super Admin can create additional admins and other users with fewer privileges. For information on how to manage YugabyteDB Anywhere users and roles, see [Manage YugabyteDB Anywhere users](../administer-yugabyte-platform/anywhere-rbac/). + +For information on how to manage database roles and users, see [Database authorization](authorization-platform/). + +## Encryption in transit + +Encryption in transit (TLS) ensures that network communication between servers is secure. You can configure YugabyteDB to use TLS to encrypt intra-cluster (Node-to-Node) and client to server (Client-to-Node) network communication. You should enable encryption in transit in YugabyteDB universes and clients to ensure the privacy and integrity of data transferred over the network. + +{{}} +For more information, see [Encryption in transit](enable-encryption-in-transit/). +{{}} + +## Encryption at rest + +Encryption at rest ensures that data at rest, stored on disk, is protected. You can configure YugabyteDB universes with a user-generated symmetric key to perform universe-wide encryption. + +Encryption at rest in YugabyteDB Anywhere uses a master key to encrypt and decrypt universe keys. The master key details are stored in YugabyteDB Anywhere in [key management service (KMS) configurations](create-kms-config/aws-kms/). You enable encryption at rest for a universe by assigning the universe a KMS configuration. The master key designated in the configuration is then used for generating the universe keys used for encrypting the universe data. + +{{}} +For more information, see [Enable encryption at rest](enable-encryption-at-rest/). +{{}} diff --git a/docs/content/preview/yugabyte-platform/security/authentication/oidc-authentication-aad.md b/docs/content/preview/yugabyte-platform/security/authentication/oidc-authentication-aad.md index 01ff490591f6..8f091bb5e35a 100644 --- a/docs/content/preview/yugabyte-platform/security/authentication/oidc-authentication-aad.md +++ b/docs/content/preview/yugabyte-platform/security/authentication/oidc-authentication-aad.md @@ -114,7 +114,11 @@ To register an application, do the following: 1. Select the tenant for the application. -1. Set the redirect URI. This is where the IdP redirects after authentication. +1. Set the redirect URI. This is where the IdP redirects after authentication. The URI is in the following form: + + ```sh + https:///api/v1/callback?client_name=OidcClient + ``` 1. Click **Register**. diff --git a/docs/content/preview/yugabyte-platform/security/authorization-platform.md b/docs/content/preview/yugabyte-platform/security/authorization-platform.md index 4cfa768ac1b1..84c0c7432b10 100644 --- a/docs/content/preview/yugabyte-platform/security/authorization-platform.md +++ b/docs/content/preview/yugabyte-platform/security/authorization-platform.md @@ -21,17 +21,17 @@ YugabyteDB uses [role-based access control](../../../secure/authorization/) (RBA (For information on managing access to your YugabyteDB Anywhere instance, refer to [Manage account users](../../administer-yugabyte-platform/anywhere-rbac/).) -## Enable database authentication +## Enable database authorization You enable the YSQL and YCQL endpoints and database authentication when deploying a universe. -On the **Create Universe > Primary Cluster** page, under **Security Configurations**, enable the **Authentication Settings** for the APIs you want to use, as shown in the following illustration. +On the **Create Universe > Primary Cluster** page, under **Security Configurations > Authentication Settings**, enable the endpoints and authorization for the APIs you want to use, as shown in the following illustration. ![Enable YSQL and YCQL endpoints](/images/yp/security/enable-endpoints.png) Enter the password to use for the default database admin superuser (`yugabyte` for YSQL, and `cassandra` for YCQL). -You can also enable and disable the endpoints and authentication after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. +You can also enable and disable the endpoints and authorization after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. ## Default roles and users diff --git a/docs/content/preview/yugabyte-platform/security/customize-ports.md b/docs/content/preview/yugabyte-platform/security/customize-ports.md deleted file mode 100644 index 14dc4f785a6b..000000000000 --- a/docs/content/preview/yugabyte-platform/security/customize-ports.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: Configure ports -headerTitle: Configure ports -linkTitle: Configure ports -description: Configure ports -menu: - preview_yugabyte-platform: - parent: security - identifier: customize-ports - weight: 20 -type: docs ---- - -YugabyteDB Anywhere and the universes it manages use a set of [default ports](../../prepare/networking/) to manage access to services. - -When deploying a universe, YugabyteDB Anywhere allows you to customize these ports. - -## Customize ports - -On the **Create Universe > Primary Cluster** page, under **Advanced Configuration**, enable the **Override Deployment Ports** option, as shown in the following illustration: - -![Override Deployment Ports](/images/yp/security/override-deployment-ports.png) - -Replace the default values with the values identifying the port that each process should use. Any value from `1024` to `65535` is valid, as long as this value does not conflict with anything else running on nodes to be provisioned. - -After deployment, you can modify the YCQL API and admin UI endpoint ports. To change ports, navigate to your universe, click **Actions**, choose **Edit YCQL Configuration**, and select the **Override YCQL Default Ports** option. - -If you change the YCQL API endpoint on an active universe, be sure to update your applications as appropriate. diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-at-rest.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-at-rest.md index 5a45fce41668..4258add7b3ef 100644 --- a/docs/content/preview/yugabyte-platform/security/enable-encryption-at-rest.md +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-at-rest.md @@ -1,8 +1,9 @@ --- -title: Enable encryption at rest -headerTitle: Enable encryption at rest -linkTitle: Enable encryption at rest -description: Enable encryption at rest +title: Encryption at rest in YugabyteDB Anywhere +headerTitle: Encryption at rest +linkTitle: Encryption at rest +description: Use encryption at rest in YugabyteDB Anywhere +headcontent: Encrypt your universes menu: preview_yugabyte-platform: parent: security @@ -18,7 +19,7 @@ YugabyteDB Anywhere uses the following types of keys for envelope encryption: | Key | Description | | :--- | :--- | | Data encryption keys (DEK) | Symmetric keys used to directly encrypt the data. Each file flushed from memory has a unique DEK. This key is generated in the database layer of YugabyteDB. | -| Universe key | Symmetric key used to encrypt and decrypt DEKs. A single universe key is used for all the DEKs in a universe. This key is generated by YugabyteDB Anywhere. +| Universe key | Symmetric key used to encrypt and decrypt DEKs. A single universe key is used for all the DEKs in a universe. This key is generated by YugabyteDB Anywhere. | | Master key | The key at the highest level in the key hierarchy. The master key is used to encrypt universe keys. This key is a customer managed key (CMK) stored and managed in a Key Management Service (KMS). | Master key details are stored in YugabyteDB Anywhere in KMS configurations, and YugabyteDB Anywhere supports CMKs in AWS KMS, GCP KMS, Azure Key Vault, and Hashicorp Vault. You enable encryption at rest for a universe by assigning the universe a KMS configuration. For instructions on creating a KMS configuration, see [Create a KMS configuration](../create-kms-config/aws-kms/). diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit.md deleted file mode 100644 index fe140665b8f7..000000000000 --- a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit.md +++ /dev/null @@ -1,704 +0,0 @@ ---- -title: Enable encryption in transit -headerTitle: Enable encryption in transit -linkTitle: Enable encryption in transit -description: Use YugabyteDB Anywhere to enable encryption in transit (TLS) on a YugabyteDB universe and connect to clients. -menu: - preview_yugabyte-platform: - parent: security - identifier: enable-encryption-in-transit - weight: 40 -rightNav: - hideH4: true -type: docs ---- - -YugabyteDB Anywhere allows you to protect data in transit by using the following: - -- Server-to-server encryption for intra-node communication between YB-Master and YB-TServer nodes. -- Client-to-server encryption for communication between clients and nodes when using CLIs, tools, and APIs for YSQL and YCQL. -- Encryption for communication between YugabyteDB Anywhere and other services, including LDAP, OIDC, Hashicorp Vault, Webhook, and S3 backup storage. - -{{< note title="Note" >}} - -Before you can enable client-to-server encryption, you first must enable server-to-server encryption. - -{{< /note >}} - -YugabyteDB Anywhere lets you create a new self-signed certificate, use an existing self-signed certificate, or upload a third-party certificate from external providers, such as Venafi or DigiCert (which is only available for an on-premises cloud provider). - -You can enable encryption in transit (TLS) during universe creation and change these settings for an existing universe. - -## Self-signed certificates generated by YugabyteDB Anywhere - -YugabyteDB Anywhere can create self-signed certificates for each universe. These certificates may be shared between universes in a single instance of YugabyteDB Anywhere. The certificate name has the following format: - -`yb-environment-universe_name`, where *environment* is the environment type (either `dev`, `stg`, `demo`, or `prod`) that was used during the tenant registration (admin user creation), and *universe-name* is the provided universe name. YugabyteDB Anywhere generates the root certificate, root private key, and node-level certificates (assuming node-to-node encryption is enabled), and then provisions those artifacts to the database nodes any time nodes are created or added to the cluster. The following three files are copied to each node: - -1. The root certificate (`ca.cert`). -1. The node certificate (`node.ip_address.crt`). -1. The node private key (`node.ip_address.key`). - -YugabyteDB Anywhere retains the root certificate and the root private key for all interactions with the cluster. - -### Customize the organization name in self-signed certificates - -YugabyteDB Anywhere automatically creates self-signed certificates when you run some workflows, such as create universe. The organization name in certificates is set to `example.com` by default. - -If you are using YugabyteDB Anywhere version 2.18.2 or later to manage universes with YugabyteDB version 2.18.2 or later, you can set a custom organization name using the global [runtime configuration](../../administer-yugabyte-platform/manage-runtime-config/) flag, `yb.tlsCertificate.organizationName`. - -Note that, for the change to take effect, you need to set the flag _before_ you run a workflow that generates a self-signed certificate. - -Customize the organization name as follows: - -1. In YugabyteDB Anywhere, navigate to **Admin** > **Advanced** and select the **Global Configuration** tab. -1. In the **Search** bar, enter `yb.tlsCertificate.organizationName` to view the flag, as per the following illustration: - - ![Custom Organization name](/images/yp/encryption-in-transit/custom-org-name.png) - -1. Click **Actions** > **Edit Configuration**, enter a new Config Value, and click **Save**. - -#### Validate custom organization name - -You can verify the organization name by running the following `openssl x509` command: - -```sh -openssl x509 -in ca.crt -text -``` - -```output {hl_lines=[6]} -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 1683277970271 (0x187eb2f7b5f) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=yb-dev-sb-ybdemo-univ1~2, O=example.com - Validity - Not Before: May 5 09:12:50 2023 GMT - Not After : May 5 09:12:50 2027 GMT -``` - -Notice that default value is `O=example.com`. - -After setting the runtime configuration to a value of your choice, (`org-foo` in this example), you should see output similar to the following: - -```sh -openssl x509 -in ca.crt -text -noout -``` - -```output -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 1689376612248 (0x18956b15f98) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo - Validity - Not Before: Jul 14 23:16:52 2023 GMT - Not After : Jul 14 23:16:52 2027 GMT - Subject: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: -``` - -### Use YugabyteDB Anywhere-generated certificates to enable TLS - -When you create a universe, you can enable TLS using certificates generated by YugabyteDB Anywhere, as follows: - -1. Create a new universe via **Universes > Create Universe** and then configure it. -1. Based on your requirements, select **Enable Node-to-Node TLS** or **Enable Client-to-Node TLS** or both. -1. Choose an existing certificate from the **Root Certificate** list or create a new certificate by accepting the default option **Create new certificate**. - -To view the certificate, navigate to **Configs > Security > Encryption in Transit > Self Signed**. - -You can also modify TLS settings for an existing universe, as follows: - -1. Navigate to either **Dashboard** or **Universes** and open a specific universe. - -1. Click **Actions > Edit Security > Encryption in-Transit** to open the **TLS Configuration** dialog and then proceed as follows: - - - If encryption in transit is currently disabled for the universe, enable it via the **Encryption in Transit for this Universe** field, as per the following illustration: - - ![TLS Configuration](/images/yp/encryption-in-transit/tls-config1.png) - - Use the expanded **TLS Configuration** dialog shown in the following illustration to change the settings to meet your requirements: - - ![TLS Configuration Expanded](/images/yp/encryption-in-transit/tls-config2.png) - - - If encryption in transit is currently enabled for the universe, you can either disable or modify it, as follows: - - - To disable encryption in transit, disable the **Encryption in Transit for this Universe** field and then click **OK**. - - - To modify encryption in-transit settings, leave the **Encryption in Transit for this Universe** field enabled and make the necessary changes to other fields. - - If you are changing certificates, you need to be aware that this requires restart of the YB-Master and YB-TServer processes and can result in downtime. To avoid downtime, you should accept the default value (enabled) for the **Rolling Upgrade** field to trigger a sequential node-by-node change with a specific delay between node upgrades (as opposed to a simultaneous change of certificates in every node which occurs when the **Rolling Upgrade** field is disabled). If you select the **Create new certificate** option when changing certificates, the corresponding certificates will be rotated, that is, replaced with new certificates. - -## Self-signed self-provided certificates - -Instead of using YugabyteDB Anywhere-provided certificates, you can use your own self-signed certificates that you upload to YugabyteDB Anywhere by following the procedure described in [Use self-signed self-provided certificates to enable TLS](#use-self-signed-self-provided-certificates-to-enable-tls). - -The certificates must meet the following criteria: - -- Be in the `.crt` format and the private key must be in the `.pem` format, with both of these artifacts available for upload. -- Contain IP addresses of the target database nodes or DNS names as the Subject Alternative Names (wildcards are acceptable). - -YugabyteDB Anywhere produces the node (leaf) certificates from the uploaded certificates and copies the certificate chain, leaf certificate, and private key to the nodes in the cluster. - -### Use self-signed self-provided certificates to enable TLS - -When you create a universe, you can enable TLS using your own certificates, as follows: - -1. Navigate to **Configs > Security > Encryption in Transit**. -1. Click **Add Certificate** to open the **Add Certificate** dialog. -1. Select **Self Signed**. -1. Click **Upload Root Certificate**, then browse to the root certificate file (`.crt`) and upload it. -1. Click **Upload Key**, then browse to the root certificate file (`.key`) and upload it. -1. In the **Certificate Name** field, enter a meaningful name for your certificate. -1. In the **Expiration Date** field, specify the expiration date of the root certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. -1. Click **Add** to make the certificate available. -1. Go to **Universes > Create Universe** to open the **Create Universe** dialog. -1. Configure the universe. -1. Based on your requirements, select **Enable Node-to-Node TLS** and **Enable Client-to-Node TLS**. -1. Select an existing certificate from the **Root Certificate** list and then select the certificate that you have uploaded. -1. Create the universe. - -You can also modify TLS settings for an existing universe by navigating to **Universes**, opening a specific universe, clicking **Actions > Edit Security > Encryption in-Transit** to open the **TLS Configuration** dialog, and then following the procedure described in [Use YugabyteDB Anywhere-generated certificates to enable TLS](#use-yugabytedb-anywhere-generated-certificates-to-enable-tls) for an existing universe. - -## Custom CA-signed self-provided certificates - -For universes created with an on-premise cloud provider, instead of using self-signed certificates, you can use third-party certificates from external CAs. The third-party CA root certificate must be configured in YugabyteDB Anywhere. You have to copy the custom CA root certificate, node certificate, and node key to the appropriate database nodes using the procedure described in [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls). - -The certificates must adhere to the following criteria: - -- Be stored in a `.crt` file, with both the certificate and the private key being in the PEM format. - - If your certificates and keys are stored in the PKCS12 format, you can [convert them to the PEM format](#convert-certificates-and-keys-from-pkcs12-to-pem-format). - -- Contain IP addresses of the database nodes or DNS names as the Subject Alternative Names (wildcards are acceptable). - -### Use custom CA-signed certificates to enable TLS - -The following procedure describes how to install certificates on the database nodes. You have to repeat these steps for every database node that is to be used in the creation of a universe. - -**Step 1:** Obtain the keys and the custom CA-signed certificates for each of the on-premise nodes for which you are configuring node-to-node TLS. In addition, obtain the keys and the custom signed certificates for client access for configuring client-to-node TLS. - -**Step 2**: For each on-premise node, copy the custom CA root certificate, node certificate, and node key to that node's file system. - -If you are enabling client-to-node TLS, make sure to copy the client certificate and client key to each of the nodes. - -In addition, ensure the following: - -- The file names and file paths of different certificates and keys are identical across all the database nodes. For example, if you name your CA root certificate as `ca.crt` on one node, then you must name it `ca.crt` on all the nodes. Similarly, if you copy `ca.crt` to `/opt/yugabyte/keys` on one node, then you must copy `ca.crt` to the same path on other nodes. -- The yugabyte system user has read permissions to all the certificates and keys. - -**Step 3**: Create a CA-signed certificate in YugabyteDB Anywhere, as follows: - -1. Navigate to **Configs > Security > Encryption in Transit**. - -1. Click **Add Certificate** to open the **Add Certificate** dialog. - -1. Select **CA Signed**, as per the following illustration: - - ![add-cert](/images/yp/encryption-in-transit/add-cert.png) - -1. Upload the custom CA root certificate as the root certificate. - - If you use an intermediate CA/issuer, but do not have the complete chain of certificates, then you need to create a bundle by executing the `cat intermediate-ca.crt root-ca.crt > bundle.crt` command, and then use this bundle as the root certificate. You might also want to [verify the certificate chain](#verify-certificate-chain). - -1. Enter the file paths for each of the certificates on the nodes. These are the paths from the previous step. - -1. In the **Certificate Name** field, enter a meaningful name for your certificate. - -1. Use the **Expiration Date** field to specify the expiration date of the certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. - -1. Click **Add** to make the certificate available. - -1. Go to **Universes > Create Universe** to open the **Create Universe** dialog. - -1. Configure the universe. - -1. Based on your requirements, select **Enable Node-to-Node TLS** and **Enable Client-to-Node TLS**. - -1. Select an existing certificate from the **Root Certificate** list and then select the certificate that you have uploaded. - -1. Create the universe. - -You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. - -#### Convert certificates and keys from PKCS12 to PEM format - -If your certificates and keys are stored in the PKCS12 format, you can convert them to the PEM format using OpenSSL. - -You start by extracting the certificate via the following command: - -```sh -openssl pkcs12 -in cert-archive.pfx -out cert.pem -clcerts -nokeys -``` - -To extract the key and write it to the PEM file unencrypted, execute the following command: - -```sh -openssl pkcs12 -in cert-archive.pfx -out key.pem -nocerts -nodes -``` - -If the key is protected by a passphrase in the PKCS12 archive, you are prompted for the passphrase. - -#### Verify certificate chain - -Perform the following steps to verify your certificates: - -1. Execute the following verify command which checks the database node certificate (node.crt) against the root CA certificate (ca.crt): - - ```sh - openssl verify ca.crt node.crt - ``` - -1. Verify that the node certificate (`node.crt`) and the node private key (`node.key`) match. See [How do I verify that a private key matches a certificate?](https://www.ssl247.com/knowledge-base/detail/how-do-i-verify-that-a-private-key-matches-a-certificate-openssl-1527076112539/ka03l0000015hscaay/) - -1. Verify that the node certificate and Root CA certificate expiration is at least 3 months by checking the validity field in the output of the following commands: - - ```sh - openssl x509 -in node.crt -text -noout - ``` - - ```sh - openssl x509 -in ca.crt -text -noout - ``` - -1. Verify that the node certificate Common Name (CN) or Subject Alternate Name (SAN) contains the IP address or DNS name of each on-prem node on which the nodes are deployed. - - {{< note >}} -Each entry you provide for the CN or SAN must match the on-prem node as entered in the provider configuration. For example, if the node address is entered as a DNS address in the on-prem provider configuration, you must use the same DNS entry in the CN or SAN, not the resolved IP address. - {{< /note >}} - - If you face any issue with the above verification, you can customize the level of certificate validation while creating a universe that uses these certificates. Refer to [Customizing the verification of RPC server certificate by the client](https://www.yugabyte.com/blog/yugabytedb-server-to-server-encryption/#customizing-the-verification-of-rpc-server-certificate-by-the-client). - -{{< note >}} -The client certificates and keys are required only if you intend to use [PostgreSQL certificate-based authentication](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html#:~:text=independent%20authentication%20option-,clientcert,-%2C%20which%20can%20be). -{{< /note >}} - -### Rotate custom CA-signed certificates - -You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. - -You rotate the existing custom certificates and replace them with new database node certificates issued by the same custom CA that issued the original certificates as follows: - -**Step 1**: Follow Step 1 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) to obtain a new set of certificates for each of the nodes. - -**Step 2**: Follow Step 2 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) to copy the certificates to the respective nodes. - -**Step 3**: Follow Step 3 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) to create a new CA-signed certificate in YugabyteDB Anywhere. - -**Step 4**: Edit the universe to use the new certificates, as follows: - -- Navigate to the universe for which you are rotating the keys. - -- Select **Actions > Edit Security**, as shown in the following illustration: - - ![edit-security](/images/yp/encryption-in-transit/edit-security.png) - -- Select **Encryption in-Transit** to open the **TLS Configuration** dialog. - -- Complete the **TLS Configuration** dialog shown in the following illustration: - - ![Configure TLS](/images/yp/encryption-in-transit/edit-tls-new.png) - - - Select the new certificate which you created in Step 3. - - - Modifying certificates requires restart of YB-Master and YB-TServer processes, which can result in downtime. To avoid downtime, you should accept the default value (enabled) for the **Rolling Upgrade** field to trigger a sequential node-by-node change with a specific delay between node upgrades (as opposed to a simultaneous change of certificates in every node which occurs when the **Rolling Upgrade** field is disabled). - - - Click **OK**. - - Typically, this process takes time, as it needs to wait for the specified delay interval after each node is upgraded. - -### Expand the universe - -You can expand universes configured with custom CA-signed certificates. - -Before adding new nodes to expand an existing universe, you need to prepare those nodes by repeating Step 2 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) for each of the new nodes you plan to add to the universe. You need to ensure that the certificates are signed by the same external CA and have the same root certificate. In addition, ensure that you copy the certificates to the same locations that you originally used when creating the universe. - -When the universe is ready for expansion, complete the **Edit Universe** dialog to add new nodes. - -## Custom HashiCorp Vault-provided certificates - -YugabyteDB Anywhere allows you to add an encryption in transit configuration using HashiCorp Vault with a public key infrastructure (PKI) secret engine. This configuration can be used to enable TLS for different clusters and YugabyteDB instances. You can apply this configuration to node-to-node encryption, client-to-node encryption, or both. - -For the correct configuration, the following criteria must be met: - -- HashiCorp Vault is unsealed. - -- HashiCorp Vault with the PKI secret engine is configured and enabled. -- HashiCorp Vault URL is accessible by YugabyteDB Anywhere. -- Because HashiCorp Vault is accessed via an authentication token mechanism, a token must be created beforehand while creating a key provider with appropriate permissions. -- HashiCorp Vault needs to be running and always accessible to YugabyteDB Anywhere. -- HashiCorp PKI certificate revocation list (CRL) or CA URLs must be accessible from each node server. -- Appropriate certificates and roles have been created for YugabyteDB Anywhere usage. -- Node servers are able to validate certificates. -- Required permissions have been provided to perform various key management operations. - -### Configure HashiCorp Vault - -Before you can start configuring HashiCorp Vault, install it on a virtual machine, as per instructions provided in [Install Vault](https://www.vaultproject.io/docs/install). The vault can be set up as a multi-node cluster. Ensure that your vault installation meets the following requirements: - -- Has transit secret engine enabled. -- Its seal and unseal mechanism is secure and repeatable. -- Its token creation mechanism is repeatable. - -You need to configure HashiCorp Vault in order to use it with YugabyteDB Anywhere, as follows: - -1. Create a vault configuration file that references your nodes and specifies the address, as follows: - - ```properties - storage "raft" { - path = "./vault/data/" - node_id = "node1" - } - - listener "tcp" { - address = "127.0.0.1:8200" - tls_disable = "true" - } - - api_addr = "http://127.0.0.1:8200" - cluster_addr = "https://127.0.0.1:8201" - ui = true - disable_mlock = true - default_lease_ttl = "768h" - max_lease_ttl = "8760h" - ``` - - Replace `127.0.0.1` with the vault web address. - - For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). - -1. Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). - -1. Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). - -1. Enable the secret engine by executing the following command: - - ```shell - vault secrets enable pki - ``` - -1. Configure the secret engine, as follows: - - - Create a root CA or configure the top-level CA. - - - Optionally, create an intermediate CA chain and sign them. - - - Create an intermediate CA for YugabyteDB, as per the following example: - - ```sh - export pki=pki - export pki_int="pki_int" - export role_i=RoleName - export ip="s.test.com" - - vault secrets enable -path=$pki_int pki - vault secrets tune -max-lease-ttl=43800h $pki_int - vault write $pki_int/intermediate/generate/internal common_name="test.com Intermediate Authority" ttl=43800h -format=json | jq -r '.data.csr' > pki_int.csr - - \# *** dump the output of the preceding command in pki_int.csr - - vault write $pki/root/sign-intermediate csr=@pki_int.csr format=pem_bundle ttl=43800h -format=json | jq -r .data.certificate > i_signed.pem - - \# *** dump the output in i_signed.pem - - vault write $pki_int/intermediate/set-signed certificate=@i_signed.pem - vault write $pki_int/config/urls issuing_certificates="http://127.0.0.1:8200/v1/pki_int/ca" crl_distribution_points="http://127.0.0.1:8200/v1/pki_int/crl" - ``` - -1. Create the vault policy, as per the following example: - - ```properties - # Enable secrets engine - path "sys/mounts/*" { - capabilities = ["create", "read", "update", "delete", "list"] - } - - # List enabled secrets engine - path "sys/mounts" { - capabilities = ["read", "list"] - } - - # Work with pki secrets engine - path "pki*" { - capabilities = ["create", "read", "update", "delete", "list", "sudo"] - } - ``` - -1. Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: - - ```shell - vault token create -no-default-policy -policy=pki_policy - ``` - - You may also specify the following for your token: - - - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. - - `period` — If specified, the token can be infinitely renewed. - - YugabyteDB Anywhere automatically tries to renew the token every 12 hours after it has passed 70% of its expiry window; as a result, you should set the TTL or period to be greater than 12 hours. - - For more information, refer to [Tokens](https://developer.hashicorp.com/vault/tutorials/tokens/tokens) in the Hashicorp documentation. - -1. Create a role that maps a name in the vault to a procedure for generating a certificate, as follows: - - ```sh - vault write /roles/ allow_any_name=true allow_subdomains=true max_ttl="8640h" - ``` - - Credentials are generated against this role. - -1. Issue certificates for nodes or a YugabyteDB client: - - - For a node, execute the following: - - ```sh - vault write /issue/ common_name="" ip_sans="" ttl="860h" - ``` - - - For YugabyteDB client, execute the following: - - ```sh - vault write /issue/ common_name="" - ``` - -### Use HashiCorp Vault-provided certificates to enable TLS - -When you create a universe, you can enable TLS using certificates provided by HashiCorp Vault, as follows: - -1. Navigate to **Configs > Security > Encryption in Transit**. -1. Click **Add Certificate** to open the **Add Certificate** dialog. -1. Select **Hashicorp**. -1. In the **Config Name** field, enter a meaningful name for your configuration. -1. In the **Vault Address** field, specify a valid URL that includes the port number. The format is `http://0.0.0.0:0000`, which corresponds to `VAULT_HOSTNAME:0000` -1. In the **Secret Token** field, specify the secret token for the vault. -1. In the **Role** field, specify the role used for creating certificates. -1. Optionally, provide the secret engine path on which the PKI is mounted. If you do not supply this information, `pki/` will be used. -1. Click **Add** to make the certificate available. -1. Go to **Universes > Create Universe** to open the **Create Universe** dialog. -1. Configure the universe. -1. Based on your requirements, select **Enable Node-to-Node TLS** and **Enable Client-to-Node TLS**. -1. Select an existing certificate from the **Root Certificate** list and then select the certificate that you have uploaded. -1. Create the universe. - -You can also edit TLS settings for an existing universe by navigating to **Universes**, opening a specific universe, clicking **Actions > Edit Security > Encryption in-Transit** to open the **TLS Configuration** dialog, and then modifying the required settings. - -## Kubernetes cert-manager - -For a universe created on Kubernetes, YugabyteDB Anywhere allows you to configure an existing running instance of the [cert-manager](https://cert-manager.io/) as a TLS certificate provider for a cluster, assuming that the following criteria are met: - -- The cert-manager is running in the Kubernetes cluster. -- A root or intermediate CA (either self-signed or external) is already configured on the cert-manager. The same root certificate file must be prepared for upload to YugabyteDB Anywhere. -- An Issuer or ClusterIssuer Kind is configured on the cert-manager and is ready to issue certificates using the previously-mentioned root or intermediate certificate. - -During the universe creation, you can enable TLS certificates issued by the cert-manager, as follows: - -1. Upload the root certificate to YugabyteDB Anywhere: - - - Prepare the root certificate in a file (for example, `root.crt`). - - Navigate to **Configs > Security > Encryption in Transit** and click **Add Certificate**. - - On the **Add Certificate** dialog shown in the following illustration, select **K8S cert-manager**: - - ![Add Certificate](/images/yp/security/kubernetes-cert-manager.png) - - - In the **Certificate Name** field, enter a meaningful name for your certificate configuration. - - Click **Upload Root Certificate** and select the root certificate file that you prepared. - - Click **Add** to make the certificate available. - -1. Configure the Kubernetes-based cloud provider by following instructions provided in [Configure region and zones](../../configure-yugabyte-platform/kubernetes/#configure-region-and-zones). In the **Add new region** dialog shown in the following illustration, you would be able to specify the Issuer name or the ClusterIssuer name for each zone. Because an Issuer Kind is a Kubernetes namespace-scoped resource, the zone definition should also set the **Namespace** field value if an Issuer Kind is selected: - - ![Add new region](/images/yp/security/kubernetes-cert-manager-add-region.png) - -1. Create the universe: - - - Navigate to **Universes** and click **Create Universe**. - - In the **Provider** field, select the cloud provider that you have configured in step 2. - - Complete the fields based on your requirements, and select **Enable Node-to-Node TLS** or **Enable Client-to-Node TLS**. - - Select the root certificate that you have uploaded in step 1. - - Click **Create**. - -### Troubleshoot - -If you encounter problems, you should verify the name of Issuer or ClusterIssuer in the Kubernetes cluster, as well as ensure that the Kubernetes cluster is in Ready state. You can use the following commands: - -```sh -kubectl get ClusterIssuer -``` - -```sh -kubectl -n Issuer -``` - -## Connect to clusters - -Using TLS, you can connect to the YSQL and YCQL endpoints. - -### Connect to a YSQL endpoint with TLS - -If you created your universe with the Client-to-Node TLS option enabled, then you must download client certificates to your client computer to establish connection to your database, as follows: - -- Navigate to the **Certificates** page and then to your universe's certificate. - -- Click **Actions** and select **Download YSQL Cert**, as shown in the following illustration. This triggers the download of the `yugabytedb.crt` and `yugabytedb.key` files. - - ![download-ysql-cert](/images/yp/encryption-in-transit/download-ysql-cert.png) - -- Optionally, when connecting to universes that are configured with custom CA-signed certificates, obtain the root CA and client YSQL certificate from your administrator. These certificates are not available on YugabyteDB Anywhere for downloading. - -- For testing with a `ysqlsh` client, paste the `yugabytedb.crt` and `yugabytedb.key` files into the `/.yugabytedb` directory and change the permissions to `0600`, as follows: - - ```sh - mkdir ~/.yugabytedb; cd ~/.yugabytedb - cp /yugabytedb.crt . - cp /yugabytedb.key . - chmod 600 yugabytedb.* - ``` - -- Run `ysqlsh` using the `sslmode=require` option, as follows: - - ```sh - cd - bin/ysqlsh -h 172.152.43.78 -p 5433 sslmode=require - ``` - - ```output - ysqlsh (11.2-YB-2.3.3.0-b0) - SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES256-GCM-SHA384, bits: 256, compression: off) - Type "help" for help. - - yugabyte=# - ``` - -To use TLS from a different client, consult the client-specific documentation. For example, if you are using a PostgreSQL JDBC driver to connect to YugabyteDB, see [Configuring the client](https://jdbc.postgresql.org/documentation/head/ssl-client.html) for more details. - -If you are using PostgreSQL/YugabyteDB JDBC driver with SSL, you need to convert the certificates to DER format. To do this, you need to perform only steps 6 and 7 from [Set up SSL certificates for Java applications](../../../reference/drivers/java/postgres-jdbc-reference/#set-up-ssl-certificates-for-java-applications) section after downloading the certificates. - -### Connect to a YCQL endpoint with TLS - -If you created your universe with the Client-to-Node TLS option enabled, then you must download client certificates to your client computer to establish connection to your database, as follows: - -- Navigate to the **Certificates** page and then to your universe's certificate. - -- Click **Actions** and select **Download Root Cert**, as shown in the following illustration. This triggers the download of the `root.crt` file. - - ![download-root-cert](/images/yp/encryption-in-transit/download-root-cert.png) - -- Optionally, when connecting to universes that are configured with custom CA-signed certificates, obtain the root CA and client YSQL certificate from your administrator. These certificates are not available on YugabyteDB Anywhere for downloading. - -- Set `SSL_CERTFILE` environment variable to point to the location of the downloaded root certificate. - -- Run `ycqlsh` using the `-ssl` option, as follows: - - ```sh - cp /root.crt ~/.yugabytedb/root.crt - export SSL_CERTFILE=~/.yugabytedb/root.crt - bin/ycqlsh 172.152.43.78 --ssl - ``` - - ```output - Connected to local cluster at 172.152.43.78:9042. - [ycqlsh 5.0.1 | Cassandra 3.9-SNAPSHOT | CQL spec 3.4.2 | Native protocol v4] - Use HELP for help. - ycqlsh> - ``` - -To use TLS from a different client, consult the client-specific documentation. For example, if you are using a Cassandra driver to connect to YugabyteDB, see [SSL](https://docs.datastax.com/en/developer/python-driver/3.19/security/#ssl). - -## Validate certificates - -When configuring and using certificates, SSL issues may occasionally arise. You can validate your certificates and keys as follows: - -1. Verify that the CA CRT and CA private key match by executing the following commands: - - ```shell - openssl rsa -noout -modulus -in ca.key | openssl md5 - openssl x509 -noout -modulus -in ca.crt | openssl md5 - - \# outputs should match - ``` - -2. Verify that the CA CRT is actually a certificate authority by executing the following command: - - ```shell - openssl x509 -text -noout -in ca.crt - - \# Look for fields - - X509v3 Basic Constraints: - - CA:TRUE - ``` - -3. Verify that certificates and keys are in PEM format (as opposed to the DER or other format). If these artifacts are not in the PEM format and you require assistance with converting them or identifying the format, consult [Converting certificates](https://support.globalsign.com/ssl/ssl-certificates-installation/converting-certificates-openssl). - -4. Ensure that the private key does not have a passphrase associated with it. For information on how to identify this condition, see [Decrypt an encrypted SSL RSA private key](https://techjourney.net/how-to-decrypt-an-enrypted-ssl-rsa-private-key-pem-key/). - -## Enforcing TLS versions - -As TLS 1.0 and 1.1 are no longer accepted by PCI compliance, and considering significant vulnerabilities around these versions of the protocol, it is recommended that you migrate to TLS 1.2 or later versions. - -You can set the TLS version for node-to-node and client-node communication. To enforce TLS 1.2, add the following flag for YB-TServer: - -```shell -ssl_protocols = tls12 -``` - -To enforce the minimum TLS version of 1.2, you need to specify all available subsequent versions for YB-TServer, as follows: - -```shell -ssl_protocols = tls12,tls13 -``` - -In addition, as the `ssl_protocols` setting does not propagate to PostgreSQL, it is recommended that you specify the minimum TLS version (`ssl_min_protocol_version`) for PostgreSQL by setting the following YB-TServer flag: - -```shell ---ysql_pg_conf_csv="ssl_min_protocol_version='TLSv1.2'" -``` - -## Use self-signed and custom CA certificates - -YugabyteDB Anywhere uses TLS to protect data in transit when connecting to other services, including: - -- LDAP -- OIDC -- Webhook -- [S3 backup storage](../../back-up-restore-universes/configure-backup-storage/) -- Hashicorp Vault -- [YugabyteDB Anywhere high availability](../../administer-yugabyte-platform/high-availability/) - -If you are using self-signed or custom CA certificates, YugabyteDB cannot verify your TLS connections unless you add the certificates to the YugabyteDB Anywhere Trust Store. - -### Add certificates to your trust store - -To add a certificate to the YugabyteDB Anywhere Trust Store, do the following: - -1. Navigate to **Admin > CA Certificates**. - -1. Click **Upload Trusted CA Certificate**. - -1. Enter a name for the certificate. - -1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. - -### Rotate a certificate in your trust store - -To rotate a certificate in your YugabyteDB Anywhere Trust Store, do the following: - -1. Navigate to **Admin > CA Certificates**. - -1. Click the **...** button for the certificate and choose **Update Certificate**. - -1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. - -### Delete a certificate in your trust store - -To delete a certificate in your YugabyteDB Anywhere Trust Store, do the following: - -1. Navigate to **Admin > CA Certificates**. - -1. Click the **...** button for the certificate and choose **Delete**, then click **Delete CA Certificate**. diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/_index.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/_index.md new file mode 100644 index 000000000000..0909c9f96b7c --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/_index.md @@ -0,0 +1,90 @@ +--- +title: Encryption in transit in YugabyteDB Anywhere +headerTitle: Encryption in transit +linkTitle: Encryption in transit +description: Use encryption in transit (TLS) to secure data traffic. +headcontent: Secure intra-node and application traffic +menu: + preview_yugabyte-platform: + parent: security + identifier: enable-encryption-in-transit + weight: 40 +type: indexpage +showRightNav: true +--- + +YugabyteDB Anywhere allows you to protect data in transit by using the following: + +- Node-to-Node TLS to encrypt intra-node communication between YB-Master and YB-TServer nodes. +- Client-to-Node TLS to encrypt communication between a universe and clients. This includes applications, shells (ysqlsh, ycqlsh, psql, and so on), and other tools, using the YSQL and YCQL APIs. + +## Manage certificates + +Use YugabyteDB Anywhere to manage certificates used for encryption in transit. + +{{}} + + {{}} + + {{}} + + {{}} + + {{}} + +{{}} + +## Enable encryption in transit + +You enable Node-to-Node and Client-to-Node encryption in transit when you [create a universe](../../create-deployments/create-universe-multi-zone/). + +You can also enable and disable encryption in transit for an existing universe as follows: + +1. Navigate to your universe. +1. Click **Actions > Edit Security > Encryption in-Transit** to open the **Manage encryption in transit** dialog. +1. Enable or disable the **Enable encryption in transit for this Universe** option. +1. Click **Apply**. + +### Enforce TLS versions + +As TLS 1.0 and 1.1 are no longer accepted by PCI compliance, and considering significant vulnerabilities around these versions of the protocol, it is recommended that you migrate to TLS 1.2 or later versions. + +You can set the TLS version for node-to-node and client-node communication. To enforce TLS 1.2, add the following flag for YB-TServer: + +```shell +ssl_protocols = tls12 +``` + +To enforce the minimum TLS version of 1.2, you need to specify all available subsequent versions for YB-TServer, as follows: + +```shell +ssl_protocols = tls12,tls13 +``` + +In addition, as the `ssl_protocols` setting does not propagate to PostgreSQL, it is recommended that you specify the minimum TLS version (`ssl_min_protocol_version`) for PostgreSQL by setting the following YB-TServer flag: + +```shell +--ysql_pg_conf_csv="ssl_min_protocol_version='TLSv1.2'" +``` + +## Learn more + +- [Securing YugabyteDB: Server-to-Server Encryption in Transit](https://www.yugabyte.com/blog/yugabytedb-server-to-server-encryption/) +- [Securing YugabyteDB: SQL Client-to-Server Encryption in Transit](https://www.yugabyte.com/blog/securing-yugabytedb-client-to-server-encryption/) +- [Securing YugabyteDB: CQL Client-to-Server Encryption in Transit](https://www.yugabyte.com/blog/securing-yugabytedb-part-3-cql-client-server-encryption-transit/) diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-ca.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-ca.md new file mode 100644 index 000000000000..3121eed05e01 --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-ca.md @@ -0,0 +1,113 @@ +--- +title: Add CA-signed certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add CA-signed certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + preview_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-2-ca + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +For universes created with an on-premises provider, instead of using self-signed certificates, you can use third-party certificates from external certificate authorities (CA). The third-party CA root certificate must be configured in YugabyteDB Anywhere. You also have to copy the custom CA root certificate, node certificate, and node key to the appropriate on-premises provider nodes. + +## Prerequisites + +The server and CA certificates must adhere to the following criteria: + +- Be stored in a `.crt` file, with both the certificate and the private key being in the PEM format. + + If your certificates and keys are stored in the PKCS12 format, you can [convert them to the PEM format](#convert-certificates-and-keys-from-pkcs12-to-pem-format). + +The server certificates must adhere to the following criteria: + +- Contain IP addresses of the database nodes in the Common Name or in the Subject Alternative Name. For on-premises universes where nodes are identified using DNS addresses, the server certificates should include the DNS names of the database nodes in the Common Name or Subject Alternate Name (wildcards are acceptable). + +## Add CA-signed certificates + +The following procedure describes how to install certificates on the database nodes. You have to repeat these steps for every database node that is to be used in the creation of a universe. + +### Obtain certificates and keys + +Obtain the keys and the custom CA-signed certificates for each of the on-premise nodes for which you are configuring node-to-node TLS. In addition, obtain the keys and the custom signed certificates for client access for configuring client-to-node TLS. + +### Copy the certificates to each node + +For each on-premises provider node, copy the custom CA certificate, node certificate, and node key to that node's file system. + +If you are enabling client-to-node TLS, make sure to copy the client-facing server certificate and client-facing server key to each of the nodes. + +In addition, ensure the following: + +- The file names and file paths of different certificates and keys are identical across all the database nodes. For example, if you name your CA root certificate as `ca.crt` on one node, then you must name it `ca.crt` on all the nodes. Similarly, if you copy `ca.crt` to `/opt/yugabyte/keys` on one node, then you must copy `ca.crt` to the same path on other nodes. +- The `yugabyte` system user has read permissions to all the certificates and keys. + +### Add the CA certificate to YugabyteDB Anywhere + +Add a CA-signed certificate to YugabyteDB Anywhere as follows: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **CA Signed**, as per the following illustration: + + ![Add CA certificate](/images/yp/encryption-in-transit/add-cert.png) + +1. In the **Certificate Name** field, enter a meaningful name for your certificate. + +1. Upload the custom CA certificate (including any intermediate certificates in the chain) as the Root CA certificate. + + If you use an intermediate CA/issuer, but do not have the complete chain of certificates, then you need to create a bundle by executing the `cat intermediate-ca.crt root-ca.crt > bundle.crt` command, and then use this bundle as the root certificate. You might also want to [verify the certificate chain](#verify-certificate-chain). + +1. Enter the file paths for each of the certificates on the nodes. These are the paths from the previous step. + +1. Use the **Expiration Date** field to specify the expiration date of the certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. + +1. Click **Add** to make the certificate available. + +You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. + +### Verify certificate chain + +Perform the following steps to verify your certificates: + +1. Execute the following verify command which checks the database node certificate (node.crt) against the root CA certificate (ca.crt): + + ```sh + openssl verify ca.crt node.crt + ``` + +1. Verify that the node certificate (`node.crt`) and the node private key (`node.key`) match. See [How do I verify that a private key matches a certificate?](https://www.ssl247.com/knowledge-base/detail/how-do-i-verify-that-a-private-key-matches-a-certificate-openssl-1527076112539/ka03l0000015hscaay/) + +1. Verify that the node certificate and Root CA certificate expiration is at least 3 months by checking the validity field in the output of the following commands: + + ```sh + openssl x509 -in node.crt -text -noout + ``` + + ```sh + openssl x509 -in ca.crt -text -noout + ``` + +1. Verify that the node certificate Common Name (CN) or Subject Alternate Name (SAN) contains the IP address or DNS name of each on-premises node on which the nodes are deployed. + + {{< note >}} +Each entry you provide for the CN or SAN must match the on-premises node as entered in the provider configuration. For example, if the node address is entered as a DNS address in the on-premises provider configuration, you must use the same DNS entry in the CN or SAN, not the resolved IP address. + {{< /note >}} + + If you face any issue with the above verification, you can customize the level of certificate validation while creating a universe that uses these certificates. Refer to [Customizing the verification of RPC server certificate by the client](https://www.yugabyte.com/blog/yugabytedb-server-to-server-encryption/#customizing-the-verification-of-rpc-server-certificate-by-the-client). + +{{< note >}} +The client certificates and keys are required only if you intend to use [PostgreSQL certificate-based authentication](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html#:~:text=independent%20authentication%20option-,clientcert,-%2C%20which%20can%20be). +{{< /note >}} diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-hashicorp.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-hashicorp.md new file mode 100644 index 000000000000..0de3ff36ffb2 --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-hashicorp.md @@ -0,0 +1,189 @@ +--- +title: Add Hashicorp Vault certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add Hashicorp Vault certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + preview_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-3-hashicorp + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +YugabyteDB Anywhere allows you to add an encryption in transit configuration using HashiCorp Vault with a public key infrastructure (PKI) secret engine. This configuration can be used to enable TLS for different clusters and YugabyteDB instances. You can apply this configuration to node-to-node encryption, client-to-node encryption, or both. + +## Prerequisites + +For the correct configuration, the following criteria must be met: + +- HashiCorp Vault is unsealed. +- HashiCorp Vault with the PKI secret engine is configured and enabled. +- HashiCorp Vault URL is accessible by YugabyteDB Anywhere. +- Because HashiCorp Vault is accessed via an authentication token mechanism, a token must be created beforehand while creating a key provider with appropriate permissions. +- HashiCorp Vault needs to be running and always accessible to YugabyteDB Anywhere. +- HashiCorp PKI certificate revocation list (CRL) or CA URLs must be accessible from each node server. +- Appropriate certificates and roles have been created for YugabyteDB Anywhere usage. +- Node servers are able to validate certificates. +- Required permissions have been provided to perform various key management operations. + +## Configure HashiCorp Vault + +Before you can start configuring HashiCorp Vault, install it on a virtual machine, as per instructions provided in [Install Vault](https://www.vaultproject.io/docs/install). The vault can be set up as a multi-node cluster. Ensure that your vault installation meets the following requirements: + +- Has transit secret engine enabled. +- Its seal and unseal mechanism is secure and repeatable. +- Its token creation mechanism is repeatable. + +You need to configure HashiCorp Vault in order to use it with YugabyteDB Anywhere, as follows: + +1. Create a vault configuration file that references your nodes and specifies the address, as follows: + + ```properties + storage "raft" { + path = "./vault/data/" + node_id = "node1" + } + + listener "tcp" { + address = "127.0.0.1:8200" + tls_disable = "true" + } + + api_addr = "http://127.0.0.1:8200" + cluster_addr = "https://127.0.0.1:8201" + ui = true + disable_mlock = true + default_lease_ttl = "768h" + max_lease_ttl = "8760h" + ``` + + Replace `127.0.0.1` with the vault web address. + + For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). + +1. Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). + +1. Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). + +1. Enable the secret engine by executing the following command: + + ```shell + vault secrets enable pki + ``` + +1. Configure the secret engine, as follows: + + - Create a root CA or configure the top-level CA. + + - Optionally, create an intermediate CA chain and sign them. + + - Create an intermediate CA for YugabyteDB, as per the following example: + + ```sh + export pki=pki + export pki_int="pki_int" + export role_i=RoleName + export ip="s.test.com" + + vault secrets enable -path=$pki_int pki + vault secrets tune -max-lease-ttl=43800h $pki_int + vault write $pki_int/intermediate/generate/internal common_name="test.com Intermediate Authority" ttl=43800h -format=json | jq -r '.data.csr' > pki_int.csr + + \# *** dump the output of the preceding command in pki_int.csr + + vault write $pki/root/sign-intermediate csr=@pki_int.csr format=pem_bundle ttl=43800h -format=json | jq -r .data.certificate > i_signed.pem + + \# *** dump the output in i_signed.pem + + vault write $pki_int/intermediate/set-signed certificate=@i_signed.pem + vault write $pki_int/config/urls issuing_certificates="http://127.0.0.1:8200/v1/pki_int/ca" crl_distribution_points="http://127.0.0.1:8200/v1/pki_int/crl" + ``` + +1. Create the vault policy, as per the following example: + + ```properties + # Enable secrets engine + path "sys/mounts/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } + + # List enabled secrets engine + path "sys/mounts" { + capabilities = ["read", "list"] + } + + # Work with pki secrets engine + path "pki*" { + capabilities = ["create", "read", "update", "delete", "list", "sudo"] + } + ``` + +1. Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: + + ```shell + vault token create -no-default-policy -policy=pki_policy + ``` + + You may also specify the following for your token: + + - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. + - `period` — If specified, the token can be infinitely renewed. + + YugabyteDB Anywhere automatically tries to renew the token every 12 hours after it has passed 70% of its expiry window; as a result, you should set the TTL or period to be greater than 12 hours. + + For more information, refer to [Tokens](https://developer.hashicorp.com/vault/tutorials/tokens/tokens) in the Hashicorp documentation. + +1. Create a role that maps a name in the vault to a procedure for generating a certificate, as follows: + + ```sh + vault write /roles/ allow_any_name=true allow_subdomains=true max_ttl="8640h" + ``` + + Credentials are generated against this role. + +1. Issue certificates for nodes or a YugabyteDB client: + + - For a node, execute the following: + + ```sh + vault write /issue/ common_name="" ip_sans="" ttl="860h" + ``` + + - For YugabyteDB client, execute the following: + + ```sh + vault write /issue/ common_name="" + ``` + +## Add HashiCorp Vault-provided certificates + +When you create a universe, you can enable TLS using certificates provided by HashiCorp Vault, as follows: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **Hashicorp**. + + ![Add Hashicorp certificate](/images/yp/encryption-in-transit/add-hashicorp-cert.png) + +1. In the **Config Name** field, enter a meaningful name for your configuration. + +1. In the **Vault Address** field, specify a valid URL that includes the port number. The format is `http://0.0.0.0:0000`, which corresponds to `VAULT_HOSTNAME:0000` + +1. In the **Secret Token** field, specify the secret token for the vault. + +1. In the **Role** field, specify the role used for creating certificates. + +1. Optionally, provide the secret engine path on which the PKI is mounted. If you do not supply this information, `pki/` will be used. + +1. Click **Add** to make the certificate available. diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-kubernetes.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-kubernetes.md new file mode 100644 index 000000000000..dfc98c52c593 --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-kubernetes.md @@ -0,0 +1,69 @@ +--- +title: Add cert-manager certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add cert-manager certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + preview_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-4-kubernetes + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +For a universe created on Kubernetes, YugabyteDB Anywhere allows you to configure an existing running instance of the [cert-manager](https://cert-manager.io/) as a TLS certificate provider for a cluster. + +## Prerequisites + +The following criteria must be met: + +- The cert-manager is running in the Kubernetes cluster. +- A root or intermediate CA (either self-signed or external) is already configured on the cert-manager. The same CA certificate file, including any intermediate CAs, must be prepared for upload to YugabyteDB Anywhere. For intermediate certificates, the chained CA certificate can be constructed using a command similar to `cat intermediate-ca.crt root-ca.crt > bundle.crt`. +- An Issuer or ClusterIssuer Kind is configured on the cert-manager and is ready to issue certificates using the previously-mentioned root or intermediate certificate. +- Prepare the root certificate in a file (for example, `root.crt`). + +## Add certificates using cert-manager + +Add TLS certificates issued by the cert-manager as follows: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **K8S cert-manager**. + + ![Add Kubernetes Certificate](/images/yp/encryption-in-transit/add-k8s-cert.png) + +1. In the **Certificate Name** field, enter a meaningful name for your certificate. + +1. Click **Upload Root Certificate** and select the CA certificate file that you prepared. + +1. Click **Add** to make the certificate available. + +## Configure the provider + +After the certificate is added to YugabyteDB Anywhere, configure the Kubernetes provider configuration by following instructions provided in [Configure region and zones](../../../configure-yugabyte-platform/kubernetes/#configure-region-and-zones). + +In the **Add new region** dialog shown in the following illustration, you would be able to specify the Issuer name or the ClusterIssuer name for each zone. Because an Issuer Kind is a Kubernetes namespace-scoped resource, the zone definition should also set the **Namespace** field value if an Issuer Kind is selected. + +![Add new region](/images/yp/security/kubernetes-cert-manager-add-region.png) + +## Troubleshoot + +If you encounter problems, you should verify the name of Issuer or ClusterIssuer in the Kubernetes cluster, as well as ensure that the Kubernetes cluster is in Ready state. You can use the following commands: + +```sh +kubectl get ClusterIssuer +``` + +```sh +kubectl -n Issuer +``` diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-self.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-self.md new file mode 100644 index 000000000000..7af7acd877b4 --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-self.md @@ -0,0 +1,99 @@ +--- +title: Add self-signed certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add self-signed certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + preview_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-1-self + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +Instead of using YugabyteDB Anywhere-provided certificates, you can use your own self-signed certificates that you upload to YugabyteDB Anywhere. + +## Prerequisites + +The certificates must meet the following criteria: + +- Be in the `.crt` format and the private key must be in the `.pem` format, with both of these artifacts available for upload. + +YugabyteDB Anywhere produces the node (leaf) certificates from the uploaded certificates and copies the certificate chain, leaf certificate, and private key to the nodes in the cluster. + +### Convert certificates and keys from PKCS12 to PEM format + +If your certificates and keys are stored in the PKCS12 format, you can convert them to the PEM format using OpenSSL. + +Start by extracting the certificate via the following command: + +```sh +openssl pkcs12 -in cert-archive.pfx -out cert.pem -clcerts -nokeys +``` + +To extract the key and write it to the PEM file unencrypted, execute the following command: + +```sh +openssl pkcs12 -in cert-archive.pfx -out key.pem -nocerts -nodes +``` + +If the key is protected by a passphrase in the PKCS12 archive, you are prompted for the passphrase. + +## Add self-signed certificates + +To add self-signed certificates to YugabyteDB Anywhere: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **Self Signed**. + + ![Add Self Signed certificate](/images/yp/encryption-in-transit/add-self-cert.png) + +1. In the **Certificate Name** field, enter a meaningful name for your certificate. + +1. Click **Upload Root Certificate**, then browse to the root certificate file (`.crt`) and upload it. + +1. Click **Upload Key**, then browse to the root certificate file (`.key`) and upload it. + +1. In the **Expiration Date** field, specify the expiration date of the root certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. + +1. Click **Add** to make the certificate available. + +## Validate certificates + +When configuring and using certificates, SSL issues may occasionally arise. You can validate your certificates and keys as follows: + +- Verify that the CA CRT and CA private key match by executing the following commands: + + ```shell + openssl rsa -noout -modulus -in ca.key | openssl md5 + openssl x509 -noout -modulus -in ca.crt | openssl md5 + + \# outputs should match + ``` + +- Verify that the CA CRT is actually a certificate authority by executing the following command: + + ```shell + openssl x509 -text -noout -in ca.crt + + \# Look for fields + + X509v3 Basic Constraints: + + CA:TRUE + ``` + +- Verify that certificates and keys are in PEM format (as opposed to the DER or other format). If these artifacts are not in the PEM format and you require assistance with converting them or identifying the format, consult [Converting certificates](https://support.globalsign.com/ssl/ssl-certificates-installation/converting-certificates-openssl). + +- Ensure that the private key does not have a passphrase associated with it. For information on how to identify this condition, see [Decrypt an encrypted SSL RSA private key](https://techjourney.net/how-to-decrypt-an-enrypted-ssl-rsa-private-key-pem-key/). diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/auto-certificate.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/auto-certificate.md new file mode 100644 index 000000000000..b5c1c7bdc66e --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/auto-certificate.md @@ -0,0 +1,95 @@ +--- +title: Automatically generated certificates on YugabyteDB Anywhere +headerTitle: Auto-generated certificates +linkTitle: Auto-generated certificates +description: YugabyteDB Anywhere-generated self-signed certificates. +headcontent: Let YugabyteDB Anywhere manage certificates for your universe +menu: + preview_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: auto-certificate + weight: 10 +type: docs +--- + +YugabyteDB Anywhere can automatically create and manage self-signed certificates for universes when you create them. These certificates may be shared between universes in a single instance of YugabyteDB Anywhere. + +Automatically generated certificates are named using the following convention: + +```sh +yb-environment-universe_name +``` + +where *environment* is the environment type (either `dev`, `stg`, `demo`, or `prod`) that was used during the tenant registration (admin user creation), and *universe_name* is the provided universe name. + +YugabyteDB Anywhere generates the root CA certificate, root private key, and node-level certificates (assuming node-to-node or client-to-node encryption is enabled), and then provisions those artifacts to the database nodes any time nodes are created or added to the cluster. The following three files are copied to each node: + +1. The root certificate (`ca.cert`). +1. The node certificate (`node.ip_address.crt`). +1. The node private key (`node.ip_address.key`). + +YugabyteDB Anywhere retains the root certificate and the root private key for all interactions with the cluster. + +To view the certificate details, navigate to **Configs > Security > Encryption in Transit** and click **Show details**. + +## Customize the organization name in self-signed certificates + +YugabyteDB Anywhere automatically creates self-signed certificates when you run some workflows, such as create universe. The organization name in certificates is set to `example.com` by default. + +If you are using YugabyteDB Anywhere version 2.18.2 or later to manage universes with YugabyteDB version 2.18.2 or later, you can set a custom organization name using the global [runtime configuration](../../../administer-yugabyte-platform/manage-runtime-config/) flag, `yb.tlsCertificate.organizationName`. + +Note that, for the change to take effect, you need to set the flag _before_ you run a workflow that generates a self-signed certificate. + +Customize the organization name as follows: + +1. In YugabyteDB Anywhere, navigate to **Admin** > **Advanced** and select the **Global Configuration** tab. +1. In the **Search** bar, enter `yb.tlsCertificate.organizationName` to view the flag, as per the following illustration: + + ![Custom Organization name](/images/yp/encryption-in-transit/custom-org-name.png) + +1. Click **Actions** > **Edit Configuration**, enter a new Config Value, and click **Save**. + +## Validate custom organization name + +You can verify the organization name by running the following `openssl x509` command: + +```sh +openssl x509 -in ca.crt -text +``` + +```output {hl_lines=[6]} +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1683277970271 (0x187eb2f7b5f) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=yb-dev-sb-ybdemo-univ1~2, O=example.com + Validity + Not Before: May 5 09:12:50 2023 GMT + Not After : May 5 09:12:50 2027 GMT +``` + +Notice that default value is `O=example.com`. + +After setting the runtime configuration to a value of your choice, (`org-foo` in this example), you should see output similar to the following: + +```sh +openssl x509 -in ca.crt -text -noout +``` + +```output +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1689376612248 (0x18956b15f98) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo + Validity + Not Before: Jul 14 23:16:52 2023 GMT + Not After : Jul 14 23:16:52 2027 GMT + Subject: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: +``` diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/rotate-certificates.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/rotate-certificates.md new file mode 100644 index 000000000000..9f130d51782a --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/rotate-certificates.md @@ -0,0 +1,55 @@ +--- +title: Rotate certificates on YugabyteDB Anywhere +headerTitle: Rotate certificates +linkTitle: Rotate certificates +description: Rotate certificates on YugabyteDB Anywhere. +headcontent: Rotate certificates used by a universe +menu: + preview_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: rotate-certificates + weight: 30 +type: docs +--- + +You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. + +Before rotating certificates, ensure that you have added the certificates to YugabyteDB Anywhere. Refer to [Add certificates](../add-certificate-self/). + +**Client-to-node certificates** + +Regardless of whether the client-to-node certificates are expired or not expired, you can always trigger a rolling upgrade to rotate the certificates. + +- If the universe was created before v2.16.6, then the rotation requires a restart, which can be done in a rolling manner with no downtime. +- If the universe was created after v2.16.6, then the rotation can be done without a restart and no downtime. + +**Node-to-node certificates** + +If the certificate has expired, the rotation requires a simultaneous restart of all nodes, resulting in some downtime. + +If the certificate has not expired, the rotation can be done using a rolling upgrade. + +- If the universe was created before v2.16.6, then the rotation requires a restart, which can be done in a rolling manner with no downtime. +- If the universe is created after v2.16.6, then the rotation can be done without a restart and no downtime. + +You can always opt to not perform rolling updates to update all nodes at the same time, but this will result in downtime. + +## Rotate certificates + +To modify encryption in transit settings and rotate certificates for a universe, do the following: + +1. Navigate to your universe. + +1. Click **Actions > Edit Security > Encryption in-Transit** to open the **Manage encryption in transit** dialog. + + ![Rotate certificates](/images/yp/encryption-in-transit/rotate-cert.png) + +1. To rotate the CA certificate, on the **Certificate Authority** tab, select the new CA certificate(s). + + If you wish to have YBA generate a new self-signed CA certificate [automatically](../auto-certificate/), delete the root certificate field. + +1. To rotate the server certificates, on the **Server Certificate** tab, select the **Rotate Node-to-Node Server Certificate** and **Rotate Client-to-Node Server Certificate** options as appropriate. + +1. Select the **Use rolling upgrade to apply this change** option to perform the upgrade in a rolling update (recommended) and enter the number of seconds to wait between node upgrades. + +1. Click **Apply**. diff --git a/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/trust-store.md b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/trust-store.md new file mode 100644 index 000000000000..7b1c3df608a0 --- /dev/null +++ b/docs/content/preview/yugabyte-platform/security/enable-encryption-in-transit/trust-store.md @@ -0,0 +1,54 @@ +--- +title: Add CA-signed certificates to YugabyteDB Anywhere +headerTitle: Add certificates to your trust store +linkTitle: Trust store +description: Add certificates to the YugabyteDB Anywhere trust store. +headcontent: Add certificates for third-party services +menu: + preview_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: trust-store + weight: 40 +type: docs +--- + +YugabyteDB Anywhere uses certificates to validate connections between YugabyteDB Anywhere and other external services, including: + +- [LDAP](../../../administer-yugabyte-platform/ldap-authentication/) +- [OIDC](../../../administer-yugabyte-platform/oidc-authentication/) +- [Webhook](../../../alerts-monitoring/set-up-alerts-health-check/) +- [S3 backup storage](../../../back-up-restore-universes/configure-backup-storage/) +- [Hashicorp Vault](../../create-kms-config/hashicorp-kms/) +- Other [YugabyteDB Anywhere high availability](../../../administer-yugabyte-platform/high-availability/) replicas. + +When using self-signed or custom CA certificates, to enable YugabyteDB Anywhere to validate your TLS connections, you _must_ add the certificates to the YugabyteDB Anywhere Trust Store + +## Add certificates to your trust store + +To add a certificate to the YugabyteDB Anywhere Trust Store, do the following: + +1. Navigate to **Admin > CA Certificates**. + +1. Click **Upload Trusted CA Certificate**. + +1. Enter a name for the certificate. + +1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. + +## Rotate a certificate in your trust store + +To rotate a certificate in your YugabyteDB Anywhere Trust Store, do the following: + +1. Navigate to **Admin > CA Certificates**. + +1. Click the **...** button for the certificate and choose **Update Certificate**. + +1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. + +## Delete a certificate in your trust store + +To delete a certificate in your YugabyteDB Anywhere Trust Store, do the following: + +1. Navigate to **Admin > CA Certificates**. + +1. Click the **...** button for the certificate and choose **Delete**, then click **Delete CA Certificate**. diff --git a/docs/content/preview/yugabyte-platform/security/security-checklist-yp.md b/docs/content/preview/yugabyte-platform/security/security-checklist-yp.md deleted file mode 100644 index d880a0561307..000000000000 --- a/docs/content/preview/yugabyte-platform/security/security-checklist-yp.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -title: Security checklist for YugabyteDB Anywhere -headerTitle: Security checklist -linkTitle: Security checklist -description: Security measures that can be implemented to protect your YugabyteDB Anywhere and YugabyteDB universes. -aliases: - - /preview/yugabyte-platform/security/network-security/ -menu: - preview_yugabyte-platform: - parent: security - identifier: security-checklist-yp - weight: 10 -type: docs ---- - -You can apply security measures to protect your YugabyteDB Anywhere instance and YugabyteDB universes. - -## Network security - -You need to ensure that YugabyteDB Anywhere and the database run in a trusted network environment. You should restrict machine and port access, based on the following guidelines: - -- Servers running YugabyteDB services are directly accessible only by YugabyteDB Anywhere, servers running the application, and database administrators. -- Only YugabyteDB Anywhere and servers running applications can connect to YugabyteDB services on the RPC ports. Access to the YugabyteDB ports should be denied to everybody else. - -For information on configuring ports, refer to [Configure ports](../customize-ports/). - -## Database authentication - -Authentication requires that all clients provide valid credentials before they can connect to a YugabyteDB universe. The authentication credentials in YugabyteDB are stored internally in the YB-Master system tables. The authentication mechanisms available to users depends on what is supported and exposed by the YSQL and YCQL APIs. - -You enable authentication for the YSQL and YCQL APIs when you deploy a universe. See [Enable database authentication](../authorization-platform/#enable-database-authentication). - -YugabyteDB Anywhere and YugabyteDB also support LDAP and OIDC for managing authentication. See [Database authentication](../authentication/). - -For more information on authentication in YugabyteDB, see [Enable authentication](../../../secure/enable-authentication/). - -## Role-based access control - -Roles can be assigned to grant users only the essential privileges based on the operations they need to perform in YugabyteDB Anywhere, and in YugabyteDB universes. - -To manage access to your YugabyteDB Anywhere instance, typically you create a [Super Admin role first](../../install-yugabyte-platform/create-admin-user/). The Super Admin can create additional admins and other users with fewer privileges. For information on how to manage YugabyteDB Anywhere users and roles, see [Manage YugabyteDB Anywhere users](../../administer-yugabyte-platform/anywhere-rbac/). - -For information on how to manage database roles and users, see [Database authorization](../authorization-platform). - -## Encryption in transit - -Encryption in transit (TLS) ensures that network communication between servers is secure. You can configure YugabyteDB to use TLS to encrypt intra-cluster and client to server network communication. You should enable encryption in transit in YugabyteDB universes and clients to ensure the privacy and integrity of data transferred over the network. - -For more information, see [Enable encryption in transit](../enable-encryption-in-transit). - -## Encryption at rest - -Encryption at rest ensures that data at rest, stored on disk, is protected. You can configure YugabyteDB universes with a user-generated symmetric key to perform universe-wide encryption. - -Encryption at rest in YugabyteDB Anywhere uses a master key to encrypt and decrypt universe keys. The master key details are stored in YugabyteDB Anywhere in [key management service (KMS) configurations](../create-kms-config/aws-kms/). You enable encryption at rest for a universe by assigning the universe a KMS configuration. The master key designated in the configuration is then used for generating the universe keys used for encrypting the universe data. - -For more information, see [Enable encryption at rest](../enable-encryption-at-rest). diff --git a/docs/content/preview/yugabyte-platform/yba-overview.md b/docs/content/preview/yugabyte-platform/yba-overview.md index dd28ef75d07a..9d60f6722330 100644 --- a/docs/content/preview/yugabyte-platform/yba-overview.md +++ b/docs/content/preview/yugabyte-platform/yba-overview.md @@ -14,7 +14,7 @@ type: docs YugabyteDB Anywhere (YBA) is a self-managed database-as-a-service that allows you to deploy and operate YugabyteDB database clusters (also known as universes) at scale. -In YBA, a database cluster is called a [universe](../../architecture/key-concepts/#universe), and the terms are used interchangeably. More precisely, a universe in YBA always consists of one (and only one) primary cluster, and can optionally also include a single [read replica](../../architecture/docdb-replication/read-replicas/) cluster attached to the primary cluster. +In YBA, a database cluster is called a [universe](../../architecture/key-concepts/#universe), and the terms are used interchangeably. More precisely, a universe in YBA always consists of one (and only one) [primary cluster](../../architecture/key-concepts/#primary-cluster), and can optionally also include a single [read replica](../../architecture/key-concepts/#read-replica-cluster/) cluster attached to the primary cluster. ## Features diff --git a/docs/content/stable/explore/fault-tolerance/_index.md b/docs/content/stable/explore/fault-tolerance/_index.md index 986fa265a985..ba165019c705 100644 --- a/docs/content/stable/explore/fault-tolerance/_index.md +++ b/docs/content/stable/explore/fault-tolerance/_index.md @@ -14,7 +14,7 @@ type: indexpage showRightNav: true --- -Resiliency, in the context of cloud databases, refers to the ability to withstand and recover from various types of failures, ranging from hardware malfunctions and software bugs to network outages and natural disasters. A resilient database system is designed to maintain data integrity, accessibility, and continuity of operations, even in the face of adverse events. Achieving resilience in cloud databases requires a multi-faceted approach, involving robust architectural design, effective data replication and backup strategies, load balancing, failover mechanisms, and comprehensive monitoring and incident response procedures. +Resiliency, in the context of cloud databases, refers to the ability to withstand and recover from various types of failures. These can range from hardware malfunctions and software bugs to network outages and natural disasters. A resilient database system is designed to maintain data integrity, accessibility, and continuity of operations, even in the face of adverse events. Achieving resilience in cloud databases requires a multi-faceted approach, involving robust architectural design, effective data replication and backup strategies, load balancing, failover mechanisms, and comprehensive monitoring and incident response procedures. YugabyteDB has been designed ground up to be resilient. YugabyteDB can continuously serve requests in the event of planned or unplanned outages, such as system upgrades and outages related to a node, availability zone, or region. YugabyteDB's High availability is achieved through a combination of distributed architecture, data replication, consensus algorithms, automatic rebalancing, and failure detection mechanisms, ensuring that the database remains available, consistent, and resilient to failures of fault domains. diff --git a/docs/content/stable/reference/configuration/yb-tserver.md b/docs/content/stable/reference/configuration/yb-tserver.md index f4c285516396..15ad7966c670 100644 --- a/docs/content/stable/reference/configuration/yb-tserver.md +++ b/docs/content/stable/reference/configuration/yb-tserver.md @@ -855,6 +855,14 @@ Default: `-1` (disables logging statement durations) Specifies the lowest YSQL message level to log. +##### --ysql_output_buffer_size + +Size of YSQL layer output buffer, in bytes. YSQL buffers query responses in this output buffer until either a buffer flush is requested by the client or the buffer overflows. + +As long as no data has been flushed from the buffer, the database can retry queries on retryable errors. For example, you can increase the size of the buffer so that YSQL can retry [read restart errors](../../../architecture/transactions/read-restart-error). + +Default: `262144` (256kB, type: int32) + ### YCQL The following flags support the use of the [YCQL API](../../../api/ycql/): diff --git a/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/high-availability.md b/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/high-availability.md index 8ab6bb467915..7e89080a5f2c 100644 --- a/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/high-availability.md +++ b/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/high-availability.md @@ -141,7 +141,7 @@ For example, if your metrics retention is 14 days on your active instance, and y After HA is operational, it is recommended that you enable certificate validation to improve security of communication between the active and any standby instances. Enable certificate validation as follows: -1. Add certificates for the active and all standbys to the active instance [trust store](../../security/enable-encryption-in-transit/#add-certificates-to-your-trust-store). +1. Add certificates for the active and all standbys to the active instance [trust store](../../security/enable-encryption-in-transit/trust-store/). - If YBA was set up to use a custom server certificate, locate the corresponding Certificate Authority (CA) certificate. - If YBA was set up to use automatically generated self-signed certificates and you installed YBA using YBA Installer, locate the CA certificate at `/opt/yugabyte/data/yba-installer/certs/ca_cert.pem` on both the YBA active and standby instances. (If you configured a custom install root, replace `/opt/yugabyte` with the path you configured.) diff --git a/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md b/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md index 1e5ed15ac66a..0fef6b0d8cee 100644 --- a/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md +++ b/docs/content/stable/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md @@ -94,6 +94,7 @@ You configure OIDC as follows: - In the **Scope** field, enter your identity provider OIDC scope that is allowed to be requested. This field accepts a space-separated list of values. If left blank, all scopes will be considered. - In the **Email Attribute** field, enter the OIDC scope containing the user email identifier. This field accepts a case-sensitive custom configuration. Typically, this field is left blank. - If you have configured OIDC to use [refresh tokens](https://openid.net/specs/openid-connect-core-1_0.html#RefreshTokens), in the **Refresh Token URL** field, enter the URL of the refresh token endpoint. + - If you have configured [OIDC enhancements](../../security/authentication/oidc-authentication-aad/#enable-oidc-enhancements), you can select the **Display JWT token on login** option to allow users to access their JWT from the YugabyteDB Anywhere sign in page. See [Set up OIDC with Azure AD on YugabyteDB Anywhere](../../security/authentication/oidc-authentication-aad/#set-up-oidc-with-azure-ad-on-yugabytedb-anywhere). 1. You can assign the default [role](../anywhere-rbac/#built-in-roles) for OIDC users to be ReadOnly or ConnectOnly. diff --git a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/aws.md b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/aws.md index 3a56d91f9a0f..ce3499b4cbf2 100644 --- a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/aws.md +++ b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/aws.md @@ -98,7 +98,7 @@ Enter a Provider name. The Provider name is an internal tag used for organizing **Credential Type**. YBA requires the ability to create VMs in AWS. To do this, you can do one of the following: -- **Specify Access ID and Secret Key** - Create an AWS Service Account with the required permissions (refer to [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes/)), and provide your AWS Access Key ID and Secret Access Key. +- **Specify Access ID and Secret Key** - Create an AWS Service Account with the required permissions (refer to [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes-aws/)), and provide your AWS Access Key ID and Secret Access Key. - **Use IAM Role from this YBA host's instance** - Provision the YBA VM instance with an IAM role that has sufficient permissions by attaching an [IAM role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html) to the YBA VM in the **EC2** tab. This option is only available if YBA is installed on AWS. **Use AWS Route 53 DNS Server**. Choose whether to use the cloud DNS Server / load balancer for universes deployed using this provider. Generally, SQL clients should prefer to use [smart client drivers](../../../drivers-orms/smart-drivers/) to connect to cluster nodes, rather than load balancers. However, in some cases (for example, if no smart driver is available in the language), you may use a DNS Server or load-balancer. The DNS Server acts as a load-balancer that routes clients to various nodes in the database universe. YBA integrates with [Amazon Route53](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/Welcome.html) to provide managed Canonical Name (CNAME) entries for your YugabyteDB universes, and automatically updates the DNS entry as nodes get created, removed, or undergo maintenance. diff --git a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/azure.md b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/azure.md index 0e8b120ed1c9..4284d16e7781 100644 --- a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/azure.md +++ b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/azure.md @@ -49,13 +49,13 @@ When deploying a universe, YBA uses the provider configuration settings to do th ## Prerequisites -You need to add the following Azure cloud provider credentials via YBA: +You need to add the following Azure cloud provider credentials: +- Application client ID and (if using credentials) client secret +- Resource group name - Subscription ID - Tenant ID - SSH port and user -- Application client ID and secret -- Resource group YBA uses the credentials to automatically provision and deprovision YugabyteDB instances. @@ -105,11 +105,29 @@ Enter a Provider name. The Provider name is an internal tag used for organizing ### Cloud Info -- **Client ID** represents the [ID of an application](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#option-2-create-a-new-application-secret) registered in your Azure Active Directory. -- **Client Secret** represents the secret of an application registered in your Azure Active Directory. You need to enter the `Value` of the secret (not the `Secret ID`). -- **Resource Group** represents the group in which YugabyteDB nodes compute and network resources are created. Your Azure Active Directory application (client ID and client secret) needs to have `Network Contributor` and `Virtual Machine Contributor` roles assigned for this resource group. -- **Subscription ID** is required for cost management. The virtual machine resources managed by YBA are tagged with this subscription. -- **Tenant ID** represents the Azure Active Directory tenant ID which belongs to an active subscription. To find your tenant ID, follow instructions provided in [How to find your Azure Active Directory tenant ID](https://learn.microsoft.com/en-us/azure/active-directory/fundamentals/how-to-find-tenant). +Enter the following details of your Azure cloud account, as described in [Azure cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes-azure/). + +#### Client ID + +Provide the ID of the application you registered with the Microsoft Identity Platform. + +#### Credential type + +If you [added credentials](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app?tabs=client-secret#add-credentials) in the form of a client secret to your registered application: + +1. Select **Specify Client Secret**. +1. Enter the Client Secret of the application associated with the Client ID you provided. You need to enter the `Value` of the secret (not the `Secret ID`). + +If you are using the [managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/qs-configure-portal-windows-vm) of the Azure VM hosting YugabyteDB Anywhere to authenticate: + +- Select **Use Managed Identity from this YBA host's instance**. + +#### Additional fields + +- **Resource Group** is the name of the resource group you created for your application, and in which YugabyteDB node compute and network resources will be created. +- **Subscription ID** is required for cost management. The virtual machine resources managed by YBA are tagged with this subscription. To get the subscription ID, open Subscriptions in Azure portal and find your subscription. Then, copy the Subscription ID. +- Optionally, if you created a different resource group for your network interfaces, provide the **Network Resource Group** name and the associated **Network Subscription ID**. If you do not provide a Network Resource Group or Subscription ID, network resources will be created in the default resource group. +- **Tenant ID** represents the tenant ID which belongs to an active subscription. To find your tenant ID, follow instructions provided in [How to find your Microsoft Entra tenant ID](https://learn.microsoft.com/en-us/entra/fundamentals/how-to-find-tenant). - **Private DNS zone** lets you use a custom domain name for the nodes in your universe. For details and instructions, see [Define a private DNS zone](#define-a-private-dns-zone). ### Regions diff --git a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/gcp.md b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/gcp.md index d282e002d077..8fae19a40c8d 100644 --- a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/gcp.md +++ b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/gcp.md @@ -108,7 +108,7 @@ Enter a Provider name. The Provider name is an internal tag used for organizing ### Cloud Info -If your YBA instance is not running inside GCP, you need to supply YBA with credentials to the desired GCP project by uploading a configuration file. To do this, set **Credential Type** to **Upload Service Account config** and proceed to upload the JSON file that you obtained when you created your service account, as described in [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes/). +If your YBA instance is not running inside GCP, you need to supply YBA with credentials to the desired GCP project by uploading a configuration file. To do this, set **Credential Type** to **Upload Service Account config** and proceed to upload the JSON file that you obtained when you created your service account, as described in [Cloud permissions](../../prepare/cloud-permissions/cloud-permissions-nodes-gcp/). If your YBA instance is running inside GCP, the preferred method for authentication to the GCP APIs is to add a service account role to the GCP instance running YBA and then configure YBA to use the instance's service account. To do this, set **Credential Type** to **Use service account from this YBA host's instance**. diff --git a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/kubernetes.md b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/kubernetes.md index c0d803b1aa22..85104b90c75d 100644 --- a/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/kubernetes.md +++ b/docs/content/stable/yugabyte-platform/configure-yugabyte-platform/kubernetes.md @@ -120,7 +120,7 @@ Continue configuring your Kubernetes provider by clicking **Add region** and com 1. Complete the **Overrides** field using one of the provided [options](#overrides). If you do not specify anything, YBA uses defaults specified inside the Helm chart. For additional information, see [Open source Kubernetes](../../../deploy/kubernetes/single-zone/oss/helm-chart/). -1. If you are using [Kubernetes cert-manager](https://cert-manager.io) to manage TLS certificates, specify the issuer type and enter the issuer name. For more information, refer to [Enable encryption in transit](../../security/enable-encryption-in-transit/#kubernetes-cert-manager). +1. If you are using [Kubernetes cert-manager](https://cert-manager.io) to manage TLS certificates, specify the issuer type and enter the issuer name. For more information, refer to [Enable encryption in transit](../../security/enable-encryption-in-transit/add-certificate-kubernetes/). If required, add a new zone by clicking **Add Zone**, as your configuration may have multiple zones. diff --git a/docs/content/stable/yugabyte-platform/create-deployments/connect-to-universe.md b/docs/content/stable/yugabyte-platform/create-deployments/connect-to-universe.md index 786964e574ef..f7db84b41830 100644 --- a/docs/content/stable/yugabyte-platform/create-deployments/connect-to-universe.md +++ b/docs/content/stable/yugabyte-platform/create-deployments/connect-to-universe.md @@ -21,13 +21,21 @@ You can connect to the database on a universe in the following ways: ## Download the universe certificate -If the universe uses encryption in transit, to connect you need to first download the universe TLS root certificate. Do the following: +If the universe uses Client-to-Node encryption in transit, to connect you need to first download the universe TLS certificate. Do the following: 1. Navigate to **Configs > Security > Encryption in Transit**. -1. Find the certificate for your universe in the list and click **Actions** and download the certificate. +1. Find your universe in the list. -For more information on connecting to TLS-enabled universes, refer to [Connect to clusters](../../security/enable-encryption-in-transit/#connect-to-clusters). +1. Click **Actions** and choose **Download Root CA Cert**. + + This downloads the `root.crt` file. + +For information on connecting using a client shell using this certificate, see [Connect from your desktop](#connect-from-your-desktop). + +To use TLS to connect an application, refer to the [driver documentation](../../../reference/drivers/). If you are using a PostgreSQL JDBC driver to connect to YugabyteDB, you can also refer to [Configuring the client](https://jdbc.postgresql.org/documentation/head/ssl-client.html) for more details. + +If you are using PostgreSQL/YugabyteDB JDBC driver with SSL, you need to convert the certificates to DER format. To do this, you need to perform only steps 6 and 7 from [Set up SSL certificates for Java applications](../../../reference/drivers/java/postgres-jdbc-reference/#set-up-ssl-certificates-for-java-applications) section after downloading the certificates. ## Connect to a universe node @@ -97,13 +105,13 @@ To run a shell from a universe node, do the following: ### Enable Tectia SSH -By default, YBA uses OpenSSH for SSH to remote nodes. YBA also supports the use of Tectia SSH that is based on the latest SSH G3 protocol. +By default, YugabyteDB Anywhere uses OpenSSH for SSH to remote nodes. YugabyteDB Anywhere also supports the use of Tectia SSH that is based on the latest SSH G3 protocol. -[Tectia SSH](https://www.ssh.com/products/tectia-ssh/) is used for secure file transfer, secure remote access and tunnelling. YBA is shipped with a trial version of Tectia SSH client that requires a license to notify YBA to permanently use Tectia instead of OpenSSH. +[Tectia SSH](https://www.ssh.com/products/tectia-ssh/) is used for secure file transfer, secure remote access and tunnelling. YugabyteDB Anywhere is shipped with a trial version of Tectia SSH client that requires a license to notify YugabyteDB Anywhere to permanently use Tectia instead of OpenSSH. To upload the Tectia license, manually copy it at `${storage_path}/yugaware/data/licenses/`, where _storage_path_ is the path provided during the Replicated installation. -After the license is uploaded, YBA exposes the runtime flag `yb.security.ssh2_enabled` that you need to enable, as per the following example: +After the license is uploaded, YugabyteDB Anywhere exposes the runtime flag `yb.security.ssh2_enabled` that you need to enable, as per the following example: ```shell curl --location --request PUT 'http:///api/v1/customers//runtime_config/00000000-0000-0000-0000-000000000000/key/yb.security.ssh2_enabled' @@ -118,21 +126,9 @@ curl --location --request PUT 'http:///api/v1/customers//runt ### Prerequisites -- If you are using a Yugabyte client shell, ensure you are running the latest versions of the shells (Yugabyte Client 2.6 or later). - - You can download using the following command on Linux or macOS: - - ```sh - $ curl -sSL https://downloads.yugabyte.com/get_clients.sh | bash - ``` - - Windows client shells require Docker. For example: - - ```sh - docker run -it yugabytedb/yugabyte-client ysqlsh -h -p - ``` +- If you are using [ysqlsh](../../../admin/ysqlsh/) or [ycqlsh](../../../admin/ycqlsh/), ensure you are running the latest versions of the shells. -- If your universe has TLS/SSL (encryption in-transit) enabled, you need to [download the certificate](#download-the-universe-certificate) to your computer. +- If your universe has Client-to-Node encryption in transit enabled, you need to [download the certificate](#download-the-universe-certificate) to your computer. - The host address of an endpoint on your universe. @@ -181,7 +177,7 @@ Replace the following: - `` with the IP address of an endpoint on your universe. - `` with your database username. - `yugabyte` with the database name, if you're connecting to a database other than the default (yugabyte). -- `` with the path to the root certificate on your computer. +- `` with the path to the universe root certificate you downloaded to your computer. To load sample data and explore an example using ysqlsh, follow the instructions in [Install the Retail Analytics sample database](../../../sample-data/retail-analytics/#install-the-retail-analytics-sample-database). @@ -203,7 +199,7 @@ Replace the following: - `` with the IP address of an endpoint on your universe. - `` with your database username. -- `` with the path to the root certificate on your computer. +- `` with the path to the universe root certificate you downloaded to your computer. @@ -224,7 +220,7 @@ Replace the following: - `` with the IP address of an endpoint on your universe. - `` with your database username. - `yugabyte` with the database name, if you're connecting to a database other than the default (yugabyte). -- `` with the path to the root certificate on your computer. +- `` with the path to the universe root certificate you downloaded to your computer. @@ -358,6 +354,7 @@ ycqlsh> SELECT * FROM ybdemo_keyspace.cassandrakeyvalue LIMIT 5; ## Learn more +- [Securing YugabyteDB: Client-to-Server Encryption in Transit](https://www.yugabyte.com/blog/securing-yugabytedb-client-to-server-encryption/#verification-of-server-certificates) - [ysqlsh](../../../admin/ysqlsh/) — Overview of the command line interface (CLI), syntax, and commands. - [YSQL API](../../../api/ysql/) — Reference for supported YSQL statements, data types, functions, and operators. - [ycqlsh](../../../admin/ycqlsh/) — Overview of the command line interface (CLI), syntax, and commands. diff --git a/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md b/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md index e78bd637938b..41c69725009b 100644 --- a/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md +++ b/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone-kubernetes.md @@ -71,8 +71,8 @@ Complete the **Security Configurations** section as follows: - **Enable YSQL Auth** - specify whether or not to enable the YSQL password authentication. - **Enable YCQL** - specify whether or not to enable the YCQL API endpoint for running Cassandra-compatible workloads. This setting is enabled by default. - **Enable YCQL Auth** - specify whether or not to enable the YCQL password authentication. -- **Enable Node-to-Node TLS** - specify whether or not to enable encryption-in-transit for communication between the database servers. This setting is enabled by default. -- **Enable Client-to-Node TLS** - specify whether or not to enable encryption-in-transit for communication between clients and the database servers. This setting is enabled by default. +- **Enable Node-to-Node TLS** - specify whether or not to enable encryption in transit for communication between the database servers. This setting is enabled by default. +- **Enable Client-to-Node TLS** - specify whether or not to enable encryption in transit for communication between clients and the database servers. This setting is enabled by default. - **Root Certificate** - select an existing security certificate or create a new one. - **Enable Encryption at Rest** - specify whether or not to enable encryption for data stored on the tablet servers. This setting is disabled by default. @@ -80,7 +80,7 @@ Complete the **Security Configurations** section as follows: Complete the **Advanced** section as follows: -- In the **DB Version** field, specify the YugabyteDB version. The default is either the same as the YugabyteDB Anywhere version or the latest YugabyteDB version available for YugabyteDB Anywhere. +- In the **DB Version** field, specify the YugabyteDB version. The default is either the same as the YugabyteDB Anywhere version or the latest YugabyteDB version available for YugabyteDB Anywhere. If the version you want to add is not listed, you can add it to YugabyteDB Anywhere. Refer to [Manage YugabyteDB releases](../../manage-deployments/ybdb-releases/). - Use the **Enable IPV6** field to specify whether or not you want to use IPV6 networking for connections between database servers. This setting is disabled by default. - Use the **Enable Public Network Access** field to specify whether or not to assign a load balancer or nodeport for connecting to the database endpoints over the internet. This setting is disabled by default. diff --git a/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone.md b/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone.md index 926bf2ce40cd..0c5e307fb2ca 100644 --- a/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone.md +++ b/docs/content/stable/yugabyte-platform/create-deployments/create-universe-multi-zone.md @@ -84,19 +84,42 @@ Specify the instance to use for the universe nodes: ### Security Configurations +#### IP Settings + To enable public access to the universe, select the **Assign Public IP** option. -Enable the YSQL and YCQL endpoints and database authentication. You can also enable and disable authentication after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. +#### Authentication Settings + +Enable the YSQL and YCQL endpoints and database authentication. Enter the password to use for the default database admin superuser (yugabyte for YSQL, and cassandra for YCQL). For more information, refer to [Database authorization](../../security/authorization-platform/). -Enable encryption in transit to encrypt universe traffic. Refer to [Enable encryption in transit](../../security/enable-encryption-in-transit/). +You can also enable and disable the API endpoints and authentication after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. + +By default, the API endpoints use ports 5433 (YSQL) and 9042 (YCQL). You can [customize these ports](#advanced-configuration), and, after deployment, you can modify the YCQL API and admin UI endpoint ports. To change YCQL ports, navigate to your universe, click **Actions**, choose **Edit YCQL Configuration**, and select the **Override YCQL Default Ports** option. + +#### Encryption Settings + +Enable encryption in transit to encrypt universe traffic. You can enable the following: + +- **Node-to-Node TLS** to encrypt traffic between universe nodes. +- **Client-to-Node TLS** to encrypt traffic between universe nodes and external clients. + + Note that if you want to enable Client-to-Node encryption, you first must enable Node-to-Node encryption. + +Encryption requires a certificate. YugabyteDB Anywhere can generate a self-signed certificate automatically, or you can use your own certificate. + +To use your own, you must first add it to YugabyteDB Anywhere; refer to [Add certificates](../../security/enable-encryption-in-transit/add-certificate-self/). + +To have YugabyteDB Anywhere generate a certificate for the universe, use the default **Root Certificate** setting of **Create New Certificate**. To use a certificate you added or a previously generated certificate, select it from the **Root Certificate** menu. + +For more information on using and managing certificates, refer to [Encryption in transit](../../security/enable-encryption-in-transit/). -Enable encryption at rest to encrypt the universe data. Refer to [Enable encryption at rest](../../security/enable-encryption-at-rest/). +To encrypt the universe data, select the **Enable encryption at rest** option and select the [KMS configuration](../../security/create-kms-config/aws-kms/) to use for encryption. For more information, refer to [Encryption at rest](../../security/enable-encryption-at-rest/). ### Advanced Configuration -Choose the version of YugabyteDB to install on the nodes. +Choose the version of YugabyteDB to install on the nodes. If the version you want to add is not listed, you can add it to YugabyteDB Anywhere. Refer to [Manage YugabyteDB releases](../../manage-deployments/ybdb-releases/). The access key is the SSH key that is created in the provider. Usually, each provider has its own access key, but if you are reusing keys across providers, they are listed here. @@ -104,7 +127,7 @@ For AWS providers, you can assign an ARN to the nodes in the universe; this allo To use cron instead of systemd for managing nodes, you can disable systemd services. This not recommended. -To customize the ports used for the universe, select the **Override Deployment Ports** option and enter the custom port numbers for the services you want to change. +To customize the [ports used for the universe](../../prepare/networking/), select the **Override Deployment Ports** option and enter the custom port numbers for the services you want to change. Any value from `1024` to `65535` is valid, as long as it doesn't conflict with anything else running on nodes to be provisioned. ### G-Flags diff --git a/docs/content/stable/yugabyte-platform/manage-deployments/edit-universe.md b/docs/content/stable/yugabyte-platform/manage-deployments/edit-universe.md index ba49842f7bd8..80666a5cd90f 100644 --- a/docs/content/stable/yugabyte-platform/manage-deployments/edit-universe.md +++ b/docs/content/stable/yugabyte-platform/manage-deployments/edit-universe.md @@ -38,7 +38,7 @@ YugabyteDB Anywhere performs these modifications through the [YB-Masters](../../ Note that you can't change the replication factor of a universe. -To change the number of nodes of universes created with an on-premises cloud provider and secured with third-party certificates obtained from external certification authorities, follow the instructions in [Expand the universe](../../security/enable-encryption-in-transit#expand-the-universe). +To change the number of nodes of universes created with an on-premises cloud provider and secured with third-party certificates obtained from external certification authorities, you must first add the certificates to the nodes you will add to the universe. Refer to [Add certificates](../../security/enable-encryption-in-transit/add-certificate-ca/). Ensure that the certificates are signed by the same external CA and have the same root certificate. In addition, ensure that you copy the certificates to the same locations that you originally used when creating the universe. ### Smart resize diff --git a/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md b/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md index 3525642e82ac..c66935dc4710 100644 --- a/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md +++ b/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-aws.md @@ -123,8 +123,8 @@ If using a service account, record the following two pieces of information about | Save for later | To configure | | :--- | :--- | -| Access key ID | [AWS cloud provider](../../../configure-yugabyte-platform/aws/) | -| Secret Access Key | [AWS cloud provider](../../../configure-yugabyte-platform/aws/) | +| Access key ID | [AWS provider configuration](../../../configure-yugabyte-platform/aws/) | +| Secret Access Key | | ### IAM role @@ -178,4 +178,4 @@ If you will be using your own custom SSH keys, then ensure that you have them wh | Save for later | To configure | | :--- | :--- | -| Custom SSH keys | [AWS provider](../../../configure-yugabyte-platform/kubernetes/) | +| Custom SSH keys | [AWS provider configuration](../../../configure-yugabyte-platform/kubernetes/) | diff --git a/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md b/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md index dfc94e1b63bc..87b68c7f7b70 100644 --- a/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md +++ b/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-azure.md @@ -52,30 +52,46 @@ The more permissions that you can provide, the more YBA can automate. ## Azure -The following permissions are required for the Azure resource group where you will deploy. +### Application and resource group + +YugabyteDB Anywhere requires cloud permissions to create VMs. You grant YugabyteDB Anywhere access to manage Azure resources such as VMs by registering an application in the Azure portal so the Microsoft identity platform can provide authentication and authorization services for your application. Registering your application establishes a trust relationship between your application and the Microsoft identity platform. + +In addition, your Azure application needs to have a [resource group](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/overview#resource-groups) with the following permissions: ```sh Network Contributor -Virtual Machine Contributor +Virtual Machine Contributor ``` -To grant the required access, you can do one of the following: +You can optionally create a resource group for network resources if you want network interfaces to be created separately. The network resource group must have the `Network Contributor` permission. + +For more information on registering applications, refer to [Register an application with the Microsoft identity platform](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app?tabs=certificate) in the Microsoft Entra documentation. + +For more information on roles, refer to [Assign Azure roles using the Azure portal](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) in the Microsoft Azure documentation. + +### Credentials + +YugabyteDB Anywhere can authenticate with Azure using one of the following methods: + +- [Add credentials](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app?tabs=client-secret#add-credentials), in the form of a client secret, to your registered application. -- [Register an application](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app) in the Azure portal so the Microsoft identity platform can provide authentication and authorization services for your application. Registering your application establishes a trust relationship between your application and the Microsoft identity platform. + For information on creating client secrets, see [Create a new client secret](https://learn.microsoft.com/en-us/entra/identity-platform/howto-create-service-principal-portal#option-3-create-a-new-client-secret) in the Microsoft Entra documentation. -- [Assign a managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/qs-configure-portal-windows-vm) to the Azure VM hosting YugabyteDB Anywhere. +- [Assign a managed identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/qs-configure-portal-windows-vm) to the Azure VM hosting YugabyteDB Anywhere. Azure will use the managed identity assigned to your instance to authenticate. -For information on assigning roles to applications, see [Assign a role to an application](https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal#assign-a-role-to-the-application); and assigning roles for managed identities, see [Assign Azure roles using the Azure portal](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) in the Microsoft Azure documentation. + For information on assigning roles for managed identities, see [Assign Azure roles using the Azure portal](https://learn.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=delegate-condition) in the Microsoft Azure documentation. -If you are registering an application, record the following information about your service account. You will need to provide this information later to YBA. +Record the following information about your service account. You will need to provide this information later when creating an Azure provider configuration. | Save for later | To configure | | :--- | :--- | -| **Service account details** | [Azure cloud provider](../../../configure-yugabyte-platform/azure/) | +| **Service account details** | [Azure provider configuration](../../../configure-yugabyte-platform/azure/) | | Client ID: | | -| Client Secret: | | +| Client Secret:
(not required when using managed identity) | | | Resource Group: | | | Subscription ID: | | +| (Optional) Network Resource Group: | | +| (Optional) Network Subscription ID: | | | Tenant ID: | | ## Managing SSH keys for VMs @@ -89,4 +105,4 @@ If you will be using your own custom SSH keys, then ensure that you have them wh | Save for later | To configure | | :--- | :--- | -| Custom SSH keys | [Azure provider](../../../configure-yugabyte-platform/azure/) | +| Custom SSH keys | [Azure provider configuration](../../../configure-yugabyte-platform/azure/) | diff --git a/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md b/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md index 87f5929a0ec0..3ce2bbefaaec 100644 --- a/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md +++ b/docs/content/stable/yugabyte-platform/prepare/cloud-permissions/cloud-permissions-nodes-gcp.md @@ -70,7 +70,7 @@ Then use one of the following methods: | Save for later | To configure | | :--- | :--- | -| Service account JSON | [GCP cloud provider](../../../configure-yugabyte-platform/gcp/) | +| Service account JSON | [GCP provider configuration](../../../configure-yugabyte-platform/gcp/) | ## Managing SSH keys for VMs @@ -83,4 +83,4 @@ If you will be using your own custom SSH keys, then ensure that you have them wh | Save for later | To configure | | :--- | :--- | -| Custom SSH keys | [GCP provider](../../../configure-yugabyte-platform/gcp/) | +| Custom SSH keys | [GCP provider configuration](../../../configure-yugabyte-platform/gcp/) | diff --git a/docs/content/stable/yugabyte-platform/prepare/networking.md b/docs/content/stable/yugabyte-platform/prepare/networking.md index 60481addf5f1..84510fa85577 100644 --- a/docs/content/stable/yugabyte-platform/prepare/networking.md +++ b/docs/content/stable/yugabyte-platform/prepare/networking.md @@ -18,7 +18,7 @@ YugabyteDB Anywhere (YBA) needs to be able to access nodes that will be used to ![YugabyteDB Anywhere network and ports](/images/yb-platform/prepare/yba-networking.png) -The following ports need to be open. (The default port numbers can be customized.) +The following ports need to be open. | From | To | Requirements | | :--- | :--- | :--- | diff --git a/docs/content/stable/yugabyte-platform/security/_index.md b/docs/content/stable/yugabyte-platform/security/_index.md index 238de3dbdb5e..adecab2fae6b 100644 --- a/docs/content/stable/yugabyte-platform/security/_index.md +++ b/docs/content/stable/yugabyte-platform/security/_index.md @@ -13,48 +13,51 @@ weight: 660 type: indexpage --- -{{}} - - {{}} - - {{}} - - {{}} - - {{}} - - {{}} - - {{}} - - {{}} - -{{}} +You can apply security measures to protect your YugabyteDB Anywhere instance and YugabyteDB universes. + +## Network security + +You need to ensure that YugabyteDB Anywhere and the database run in a trusted network environment. You should restrict machine and port access, based on the following guidelines: + +- Servers running YugabyteDB services are directly accessible only by YugabyteDB Anywhere, servers running the application, and database administrators. +- Only YugabyteDB Anywhere and servers running applications can connect to YugabyteDB services on the RPC ports. Access to the YugabyteDB ports should be denied to everybody else. + +{{}} +For information on networking and port requirements, refer to [Networking](../prepare/networking/). +{{}} + +## Database authentication + +Authentication requires that all clients provide valid credentials before they can connect to a YugabyteDB universe. The authentication credentials in YugabyteDB are stored internally in the YB-Master system tables. The authentication mechanisms available to users depends on what is supported and exposed by the YSQL and YCQL APIs. + +You enable authentication for the YSQL and YCQL APIs when you deploy a universe. See [Enable database authentication](authorization-platform/#enable-database-authentication). + +YugabyteDB Anywhere and YugabyteDB also support LDAP and OIDC for managing authentication. See [Database authentication](authentication/). + +For more information on authentication in YugabyteDB, see [Enable authentication](../../secure/enable-authentication/). + +## Role-based access control + +Roles can be assigned to grant users only the essential privileges based on the operations they need to perform in YugabyteDB Anywhere, and in YugabyteDB universes. + +To manage access to your YugabyteDB Anywhere instance, typically you create a [Super Admin role first](../install-yugabyte-platform/create-admin-user/). The Super Admin can create additional admins and other users with fewer privileges. For information on how to manage YugabyteDB Anywhere users and roles, see [Manage YugabyteDB Anywhere users](../administer-yugabyte-platform/anywhere-rbac/). + +For information on how to manage database roles and users, see [Database authorization](authorization-platform/). + +## Encryption in transit + +Encryption in transit (TLS) ensures that network communication between servers is secure. You can configure YugabyteDB to use TLS to encrypt intra-cluster (Node-to-Node) and client to server (Client-to-Node) network communication. You should enable encryption in transit in YugabyteDB universes and clients to ensure the privacy and integrity of data transferred over the network. + +{{}} +For more information, see [Encryption in transit](enable-encryption-in-transit/). +{{}} + +## Encryption at rest + +Encryption at rest ensures that data at rest, stored on disk, is protected. You can configure YugabyteDB universes with a user-generated symmetric key to perform universe-wide encryption. + +Encryption at rest in YugabyteDB Anywhere uses a master key to encrypt and decrypt universe keys. The master key details are stored in YugabyteDB Anywhere in [key management service (KMS) configurations](create-kms-config/aws-kms/). You enable encryption at rest for a universe by assigning the universe a KMS configuration. The master key designated in the configuration is then used for generating the universe keys used for encrypting the universe data. + +{{}} +For more information, see [Enable encryption at rest](enable-encryption-at-rest/). +{{}} diff --git a/docs/content/stable/yugabyte-platform/security/authentication/oidc-authentication-aad.md b/docs/content/stable/yugabyte-platform/security/authentication/oidc-authentication-aad.md index b8cd1fd5cdb6..f95191fe0f3c 100644 --- a/docs/content/stable/yugabyte-platform/security/authentication/oidc-authentication-aad.md +++ b/docs/content/stable/yugabyte-platform/security/authentication/oidc-authentication-aad.md @@ -114,7 +114,11 @@ To register an application, do the following: 1. Select the tenant for the application. -1. Set the redirect URI. This is where the IdP redirects after authentication. +1. Set the redirect URI. This is where the IdP redirects after authentication. The URI is in the following form: + + ```sh + https:///api/v1/callback?client_name=OidcClient + ``` 1. Click **Register**. diff --git a/docs/content/stable/yugabyte-platform/security/authorization-platform.md b/docs/content/stable/yugabyte-platform/security/authorization-platform.md index 628d466fc7e8..e5e9f14a3c45 100644 --- a/docs/content/stable/yugabyte-platform/security/authorization-platform.md +++ b/docs/content/stable/yugabyte-platform/security/authorization-platform.md @@ -21,17 +21,17 @@ YugabyteDB uses [role-based access control](../../../secure/authorization/) (RBA (For information on managing access to your YugabyteDB Anywhere instance, refer to [Manage account users](../../administer-yugabyte-platform/anywhere-rbac/).) -## Enable database authentication +## Enable database authorization You enable the YSQL and YCQL endpoints and database authentication when deploying a universe. -On the **Create Universe > Primary Cluster** page, under **Security Configurations**, enable the **Authentication Settings** for the APIs you want to use, as shown in the following illustration. +On the **Create Universe > Primary Cluster** page, under **Security Configurations > Authentication Settings**, enable the endpoints and authorization for the APIs you want to use, as shown in the following illustration. ![Enable YSQL and YCQL endpoints](/images/yp/security/enable-endpoints.png) Enter the password to use for the default database admin superuser (`yugabyte` for YSQL, and `cassandra` for YCQL). -You can also enable and disable the endpoints and authentication after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. +You can also enable and disable the endpoints and authorization after deployment. Navigate to your universe, click **Actions**, and choose **Edit YSQL Configuration** or **Edit YCQL Configuration**. ## Default roles and users @@ -47,7 +47,7 @@ yugabyte=> \du ```output List of roles - Role name | Attributes | Member of + Role name | Attributes | Member of --------------+------------------------------------------------------------+----------- postgres | Superuser, Create role, Create DB, Replication, Bypass RLS | {} yb_db_admin | No inheritance, Cannot login | {} diff --git a/docs/content/stable/yugabyte-platform/security/customize-ports.md b/docs/content/stable/yugabyte-platform/security/customize-ports.md deleted file mode 100644 index 96e31ce9b2c5..000000000000 --- a/docs/content/stable/yugabyte-platform/security/customize-ports.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: Configure ports -headerTitle: Configure ports -linkTitle: Configure ports -description: Configure ports -menu: - stable_yugabyte-platform: - parent: security - identifier: customize-ports - weight: 20 -type: docs ---- - -YugabyteDB Anywhere and the universes it manages use a set of [default ports](../../prepare/networking/) to manage access to services. - -When deploying a universe, YugabyteDB Anywhere allows you to customize these ports. - -## Customize ports - -On the **Create Universe > Primary Cluster** page, under **Advanced Configuration**, enable the **Override Deployment Ports** option, as shown in the following illustration: - -![Override Deployment Ports](/images/yp/security/override-deployment-ports.png) - -Replace the default values with the values identifying the port that each process should use. Any value from `1024` to `65535` is valid, as long as this value does not conflict with anything else running on nodes to be provisioned. - -After deployment, you can modify the YCQL API and admin UI endpoint ports. To change ports, navigate to your universe, click **Actions**, choose **Edit YCQL Configuration**, and select the **Override YCQL Default Ports** option. - -If you change the YCQL API endpoint on an active universe, be sure to update your applications as appropriate. diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-at-rest.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-at-rest.md index 939ee6e46291..eb6210b5d269 100644 --- a/docs/content/stable/yugabyte-platform/security/enable-encryption-at-rest.md +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-at-rest.md @@ -1,8 +1,9 @@ --- -title: Enable encryption at rest -headerTitle: Enable encryption at rest -linkTitle: Enable encryption at rest -description: Enable encryption at rest +title: Encryption at rest in YugabyteDB Anywhere +headerTitle: Encryption at rest +linkTitle: Encryption at rest +description: Use encryption at rest in YugabyteDB Anywhere +headcontent: Encrypt your universes menu: stable_yugabyte-platform: parent: security @@ -18,7 +19,7 @@ YugabyteDB Anywhere uses the following types of keys for envelope encryption: | Key | Description | | :--- | :--- | | Data encryption keys (DEK) | Symmetric keys used to directly encrypt the data. Each file flushed from memory has a unique DEK. This key is generated in the database layer of YugabyteDB. | -| Universe key | Symmetric key used to encrypt and decrypt DEKs. A single universe key is used for all the DEKs in a universe. This key is generated by YugabyteDB Anywhere. +| Universe key | Symmetric key used to encrypt and decrypt DEKs. A single universe key is used for all the DEKs in a universe. This key is generated by YugabyteDB Anywhere. | | Master key | The key at the highest level in the key hierarchy. The master key is used to encrypt universe keys. This key is a customer managed key (CMK) stored and managed in a Key Management Service (KMS). | Master key details are stored in YugabyteDB Anywhere in KMS configurations, and YugabyteDB Anywhere supports CMKs in AWS KMS, GCP KMS, Azure Key Vault, and Hashicorp Vault. You enable encryption at rest for a universe by assigning the universe a KMS configuration. For instructions on creating a KMS configuration, see [Create a KMS configuration](../create-kms-config/aws-kms/). diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit.md deleted file mode 100644 index 81ac79f914aa..000000000000 --- a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit.md +++ /dev/null @@ -1,704 +0,0 @@ ---- -title: Enable encryption in transit -headerTitle: Enable encryption in transit -linkTitle: Enable encryption in transit -description: Use YugabyteDB Anywhere to enable encryption in transit (TLS) on a YugabyteDB universe and connect to clients. -menu: - stable_yugabyte-platform: - parent: security - identifier: enable-encryption-in-transit - weight: 40 -rightNav: - hideH4: true -type: docs ---- - -YugabyteDB Anywhere allows you to protect data in transit by using the following: - -- Server-to-server encryption for intra-node communication between YB-Master and YB-TServer nodes. -- Client-to-server encryption for communication between clients and nodes when using CLIs, tools, and APIs for YSQL and YCQL. -- Encryption for communication between YugabyteDB Anywhere and other services, including LDAP, OIDC, Hashicorp Vault, Webhook, and S3 backup storage. - -{{< note title="Note" >}} - -Before you can enable client-to-server encryption, you first must enable server-to-server encryption. - -{{< /note >}} - -YugabyteDB Anywhere lets you create a new self-signed certificate, use an existing self-signed certificate, or upload a third-party certificate from external providers, such as Venafi or DigiCert (which is only available for an on-premises cloud provider). - -You can enable encryption in transit (TLS) during universe creation and change these settings for an existing universe. - -## Self-signed certificates generated by YugabyteDB Anywhere - -YugabyteDB Anywhere can create self-signed certificates for each universe. These certificates may be shared between universes in a single instance of YugabyteDB Anywhere. The certificate name has the following format: - -`yb-environment-universe_name`, where *environment* is the environment type (either `dev`, `stg`, `demo`, or `prod`) that was used during the tenant registration (admin user creation), and *universe-name* is the provided universe name. YugabyteDB Anywhere generates the root certificate, root private key, and node-level certificates (assuming node-to-node encryption is enabled), and then provisions those artifacts to the database nodes any time nodes are created or added to the cluster. The following three files are copied to each node: - -1. The root certificate (`ca.cert`). -1. The node certificate (`node.ip_address.crt`). -1. The node private key (`node.ip_address.key`). - -YugabyteDB Anywhere retains the root certificate and the root private key for all interactions with the cluster. - -### Customize the organization name in self-signed certificates - -YugabyteDB Anywhere automatically creates self-signed certificates when you run some workflows, such as create universe. The organization name in certificates is set to `example.com` by default. - -If you are using YBA version 2.18.2 or later to manage universes with YugabyteDB version 2.18.2 or later, you can set a custom organization name using the global [runtime configuration](../../administer-yugabyte-platform/manage-runtime-config/) flag, `yb.tlsCertificate.organizationName`. - -Note that, for the change to take effect, you need to set the flag _before_ you run a workflow that generates a self-signed certificate. - -Customize the organization name as follows: - -1. In YugabyteDB Anywhere, navigate to **Admin** > **Advanced** and select the **Global Configuration** tab. -1. In the **Search** bar, enter `yb.tlsCertificate.organizationName` to view the flag, as per the following illustration: - - ![Custom Organization name](/images/yp/encryption-in-transit/custom-org-name.png) - -1. Click **Actions** > **Edit Configuration**, enter a new Config Value, and click **Save**. - -#### Validate custom organization name - -You can verify the organization name by running the following `openssl x509` command: - -```sh -openssl x509 -in ca.crt -text -``` - -```output {hl_lines=[6]} -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 1683277970271 (0x187eb2f7b5f) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN=yb-dev-sb-ybdemo-univ1~2, O=example.com - Validity - Not Before: May 5 09:12:50 2023 GMT - Not After : May 5 09:12:50 2027 GMT -``` - -Notice that default value is `O=example.com`. - -After setting the runtime configuration to a value of your choice, (`org-foo` in this example), you should see output similar to the following: - -```sh -openssl x509 -in ca.crt -text -noout -``` - -```output -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 1689376612248 (0x18956b15f98) - Signature Algorithm: sha256WithRSAEncryption - Issuer: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo - Validity - Not Before: Jul 14 23:16:52 2023 GMT - Not After : Jul 14 23:16:52 2027 GMT - Subject: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: -``` - -### Use YugabyteDB Anywhere-generated certificates to enable TLS - -When you create a universe, you can enable TLS using certificates generated by YugabyteDB Anywhere, as follows: - -1. Create a new universe via **Universes > Create Universe** and then configure it. -1. Based on your requirements, select **Enable Node-to-Node TLS** or **Enable Client-to-Node TLS** or both. -1. Choose an existing certificate from the **Root Certificate** list or create a new certificate by accepting the default option **Create new certificate**. - -To view the certificate, navigate to **Configs > Security > Encryption in Transit > Self Signed**. - -You can also modify TLS settings for an existing universe, as follows: - -1. Navigate to either **Dashboard** or **Universes** and open a specific universe. - -1. Click **Actions > Edit Security > Encryption in-Transit** to open the **TLS Configuration** dialog and then proceed as follows: - - - If encryption in transit is currently disabled for the universe, enable it via the **Encryption in Transit for this Universe** field, as per the following illustration: - - ![TLS Configuration](/images/yp/encryption-in-transit/tls-config1.png) - - Use the expanded **TLS Configuration** dialog shown in the following illustration to change the settings to meet your requirements: - - ![TLS Configuration Expanded](/images/yp/encryption-in-transit/tls-config2.png) - - - If encryption in transit is currently enabled for the universe, you can either disable or modify it, as follows: - - - To disable encryption in transit, disable the **Encryption in Transit for this Universe** field and then click **OK**. - - - To modify encryption in-transit settings, leave the **Encryption in Transit for this Universe** field enabled and make the necessary changes to other fields. - - If you are changing certificates, you need to be aware that this requires restart of the YB-Master and YB-TServer processes and can result in downtime. To avoid downtime, you should accept the default value (enabled) for the **Rolling Upgrade** field to trigger a sequential node-by-node change with a specific delay between node upgrades (as opposed to a simultaneous change of certificates in every node which occurs when the **Rolling Upgrade** field is disabled). If you select the **Create new certificate** option when changing certificates, the corresponding certificates will be rotated, that is, replaced with new certificates. - -## Self-signed self-provided certificates - -Instead of using YugabyteDB Anywhere-provided certificates, you can use your own self-signed certificates that you upload to YugabyteDB Anywhere by following the procedure described in [Use self-signed self-provided certificates to enable TLS](#use-self-signed-self-provided-certificates-to-enable-tls). - -The certificates must meet the following criteria: - -- Be in the `.crt` format and the private key must be in the `.pem` format, with both of these artifacts available for upload. -- Contain IP addresses of the target database nodes or DNS names as the Subject Alternative Names (wildcards are acceptable). - -YugabyteDB Anywhere produces the node (leaf) certificates from the uploaded certificates and copies the certificate chain, leaf certificate, and private key to the nodes in the cluster. - -### Use self-signed self-provided certificates to enable TLS - -When you create a universe, you can enable TLS using your own certificates, as follows: - -1. Navigate to **Configs > Security > Encryption in Transit**. -1. Click **Add Certificate** to open the **Add Certificate** dialog. -1. Select **Self Signed**. -1. Click **Upload Root Certificate**, then browse to the root certificate file (`.crt`) and upload it. -1. Click **Upload Key**, then browse to the root certificate file (`.key`) and upload it. -1. In the **Certificate Name** field, enter a meaningful name for your certificate. -1. In the **Expiration Date** field, specify the expiration date of the root certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. -1. Click **Add** to make the certificate available. -1. Go to **Universes > Create Universe** to open the **Create Universe** dialog. -1. Configure the universe. -1. Based on your requirements, select **Enable Node-to-Node TLS** and **Enable Client-to-Node TLS**. -1. Select an existing certificate from the **Root Certificate** list and then select the certificate that you have uploaded. -1. Create the universe. - -You can also modify TLS settings for an existing universe by navigating to **Universes**, opening a specific universe, clicking **Actions > Edit Security > Encryption in-Transit** to open the **TLS Configuration** dialog, and then following the procedure described in [Use YugabyteDB Anywhere-generated certificates to enable TLS](#use-yugabytedb-anywhere-generated-certificates-to-enable-tls) for an existing universe. - -## Custom CA-signed self-provided certificates - -For universes created with an on-premise cloud provider, instead of using self-signed certificates, you can use third-party certificates from external CAs. The third-party CA root certificate must be configured in YugabyteDB Anywhere. You have to copy the custom CA root certificate, node certificate, and node key to the appropriate database nodes using the procedure described in [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls). - -The certificates must adhere to the following criteria: - -- Be stored in a `.crt` file, with both the certificate and the private key being in the PEM format. - - If your certificates and keys are stored in the PKCS12 format, you can [convert them to the PEM format](#convert-certificates-and-keys-from-pkcs12-to-pem-format). - -- Contain IP addresses of the database nodes or DNS names as the Subject Alternative Names (wildcards are acceptable). - -### Use custom CA-signed certificates to enable TLS - -The following procedure describes how to install certificates on the database nodes. You have to repeat these steps for every database node that is to be used in the creation of a universe. - -**Step 1:** Obtain the keys and the custom CA-signed certificates for each of the on-premise nodes for which you are configuring node-to-node TLS. In addition, obtain the keys and the custom signed certificates for client access for configuring client-to-node TLS. - -**Step 2**: For each on-premise node, copy the custom CA root certificate, node certificate, and node key to that node's file system. - -If you are enabling client-to-node TLS, make sure to copy the client certificate and client key to each of the nodes. - -In addition, ensure the following: - -- The file names and file paths of different certificates and keys are identical across all the database nodes. For example, if you name your CA root certificate as `ca.crt` on one node, then you must name it `ca.crt` on all the nodes. Similarly, if you copy `ca.crt` to `/opt/yugabyte/keys` on one node, then you must copy `ca.crt` to the same path on other nodes. -- The yugabyte system user has read permissions to all the certificates and keys. - -**Step 3**: Create a CA-signed certificate in YugabyteDB Anywhere, as follows: - -1. Navigate to **Configs > Security > Encryption in Transit**. - -1. Click **Add Certificate** to open the **Add Certificate** dialog. - -1. Select **CA Signed**, as per the following illustration: - - ![add-cert](/images/yp/encryption-in-transit/add-cert.png) - -1. Upload the custom CA root certificate as the root certificate. - - If you use an intermediate CA/issuer, but do not have the complete chain of certificates, then you need to create a bundle by executing the `cat intermediate-ca.crt root-ca.crt > bundle.crt` command, and then use this bundle as the root certificate. You might also want to [verify the certificate chain](#verify-certificate-chain). - -1. Enter the file paths for each of the certificates on the nodes. These are the paths from the previous step. - -1. In the **Certificate Name** field, enter a meaningful name for your certificate. - -1. Use the **Expiration Date** field to specify the expiration date of the certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. - -1. Click **Add** to make the certificate available. - -1. Go to **Universes > Create Universe** to open the **Create Universe** dialog. - -1. Configure the universe. - -1. Based on your requirements, select **Enable Node-to-Node TLS** and **Enable Client-to-Node TLS**. - -1. Select an existing certificate from the **Root Certificate** list and then select the certificate that you have uploaded. - -1. Create the universe. - -You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. - -#### Convert certificates and keys from PKCS12 to PEM format - -If your certificates and keys are stored in the PKCS12 format, you can convert them to the PEM format using OpenSSL. - -You start by extracting the certificate via the following command: - -```sh -openssl pkcs12 -in cert-archive.pfx -out cert.pem -clcerts -nokeys -``` - -To extract the key and write it to the PEM file unencrypted, execute the following command: - -```sh -openssl pkcs12 -in cert-archive.pfx -out key.pem -nocerts -nodes -``` - -If the key is protected by a passphrase in the PKCS12 archive, you are prompted for the passphrase. - -#### Verify certificate chain - -Perform the following steps to verify your certificates: - -1. Execute the following verify command which checks the database node certificate (node.crt) against the root CA certificate (ca.crt): - - ```sh - openssl verify ca.crt node.crt - ``` - -1. Verify that the node certificate (`node.crt`) and the node private key (`node.key`) match. See [How do I verify that a private key matches a certificate?](https://www.ssl247.com/knowledge-base/detail/how-do-i-verify-that-a-private-key-matches-a-certificate-openssl-1527076112539/ka03l0000015hscaay/) - -1. Verify that the node certificate and Root CA certificate expiration is at least 3 months by checking the validity field in the output of the following commands: - - ```sh - openssl x509 -in node.crt -text -noout - ``` - - ```sh - openssl x509 -in ca.crt -text -noout - ``` - -1. Verify that the node certificate Common Name (CN) or Subject Alternate Name (SAN) contains the IP address or DNS name of each on-prem node on which the nodes are deployed. - - {{< note >}} -Each entry you provide for the CN or SAN must match the on-prem node as entered in the provider configuration. For example, if the node address is entered as a DNS address in the on-prem provider configuration, you must use the same DNS entry in the CN or SAN, not the resolved IP address. - {{< /note >}} - - If you face any issue with the above verification, you can customize the level of certificate validation while creating a universe that uses these certificates. Refer to [Customizing the verification of RPC server certificate by the client](https://www.yugabyte.com/blog/yugabytedb-server-to-server-encryption/#customizing-the-verification-of-rpc-server-certificate-by-the-client). - -{{< note >}} -The client certificates and keys are required only if you intend to use [PostgreSQL certificate-based authentication](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html#:~:text=independent%20authentication%20option-,clientcert,-%2C%20which%20can%20be). -{{< /note >}} - -### Rotate custom CA-signed certificates - -You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. - -You rotate the existing custom certificates and replace them with new database node certificates issued by the same custom CA that issued the original certificates as follows: - -**Step 1**: Follow Step 1 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) to obtain a new set of certificates for each of the nodes. - -**Step 2**: Follow Step 2 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) to copy the certificates to the respective nodes. - -**Step 3**: Follow Step 3 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) to create a new CA-signed certificate in YugabyteDB Anywhere. - -**Step 4**: Edit the universe to use the new certificates, as follows: - -- Navigate to the universe for which you are rotating the keys. - -- Select **Actions > Edit Security**, as shown in the following illustration: - - ![edit-security](/images/yp/encryption-in-transit/edit-security.png) - -- Select **Encryption in-Transit** to open the **TLS Configuration** dialog. - -- Complete the **TLS Configuration** dialog shown in the following illustration: - - ![Configure TLS](/images/yp/encryption-in-transit/edit-tls-new.png) - - - Select the new certificate which you created in Step 3. - - - Modifying certificates requires restart of YB-Master and YB-TServer processes, which can result in downtime. To avoid downtime, you should accept the default value (enabled) for the **Rolling Upgrade** field to trigger a sequential node-by-node change with a specific delay between node upgrades (as opposed to a simultaneous change of certificates in every node which occurs when the **Rolling Upgrade** field is disabled). - - - Click **OK**. - - Typically, this process takes time, as it needs to wait for the specified delay interval after each node is upgraded. - -### Expand the universe - -You can expand universes configured with custom CA-signed certificates. - -Before adding new nodes to expand an existing universe, you need to prepare those nodes by repeating Step 2 of [Use custom CA-signed certificates to enable TLS](#use-custom-ca-signed-certificates-to-enable-tls) for each of the new nodes you plan to add to the universe. You need to ensure that the certificates are signed by the same external CA and have the same root certificate. In addition, ensure that you copy the certificates to the same locations that you originally used when creating the universe. - -When the universe is ready for expansion, complete the **Edit Universe** dialog to add new nodes. - -## Custom HashiCorp Vault-provided certificates - -YugabyteDB Anywhere allows you to add an encryption in transit configuration using HashiCorp Vault with a public key infrastructure (PKI) secret engine. This configuration can be used to enable TLS for different clusters and YugabyteDB instances. You can apply this configuration to node-to-node encryption, client-to-node encryption, or both. - -For the correct configuration, the following criteria must be met: - -- HashiCorp Vault is unsealed. - -- HashiCorp Vault with the PKI secret engine is configured and enabled. -- HashiCorp Vault URL is accessible by YugabyteDB Anywhere. -- Because HashiCorp Vault is accessed via an authentication token mechanism, a token must be created beforehand while creating a key provider with appropriate permissions. -- HashiCorp Vault needs to be running and always accessible to YugabyteDB Anywhere. -- HashiCorp PKI certificate revocation list (CRL) or CA URLs must be accessible from each node server. -- Appropriate certificates and roles have been created for YugabyteDB Anywhere usage. -- Node servers are able to validate certificates. -- Required permissions have been provided to perform various key management operations. - -### Configure HashiCorp Vault - -Before you can start configuring HashiCorp Vault, install it on a virtual machine, as per instructions provided in [Install Vault](https://www.vaultproject.io/docs/install). The vault can be set up as a multi-node cluster. Ensure that your vault installation meets the following requirements: - -- Has transit secret engine enabled. -- Its seal and unseal mechanism is secure and repeatable. -- Its token creation mechanism is repeatable. - -You need to configure HashiCorp Vault in order to use it with YugabyteDB Anywhere, as follows: - -1. Create a vault configuration file that references your nodes and specifies the address, as follows: - - ```properties - storage "raft" { - path = "./vault/data/" - node_id = "node1" - } - - listener "tcp" { - address = "127.0.0.1:8200" - tls_disable = "true" - } - - api_addr = "http://127.0.0.1:8200" - cluster_addr = "https://127.0.0.1:8201" - ui = true - disable_mlock = true - default_lease_ttl = "768h" - max_lease_ttl = "8760h" - ``` - - Replace `127.0.0.1` with the vault web address. - - For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). - -1. Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). - -1. Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). - -1. Enable the secret engine by executing the following command: - - ```shell - vault secrets enable pki - ``` - -1. Configure the secret engine, as follows: - - - Create a root CA or configure the top-level CA. - - - Optionally, create an intermediate CA chain and sign them. - - - Create an intermediate CA for YugabyteDB, as per the following example: - - ```sh - export pki=pki - export pki_int="pki_int" - export role_i=RoleName - export ip="s.test.com" - - vault secrets enable -path=$pki_int pki - vault secrets tune -max-lease-ttl=43800h $pki_int - vault write $pki_int/intermediate/generate/internal common_name="test.com Intermediate Authority" ttl=43800h -format=json | jq -r '.data.csr' > pki_int.csr - - \# *** dump the output of the preceding command in pki_int.csr - - vault write $pki/root/sign-intermediate csr=@pki_int.csr format=pem_bundle ttl=43800h -format=json | jq -r .data.certificate > i_signed.pem - - \# *** dump the output in i_signed.pem - - vault write $pki_int/intermediate/set-signed certificate=@i_signed.pem - vault write $pki_int/config/urls issuing_certificates="http://127.0.0.1:8200/v1/pki_int/ca" crl_distribution_points="http://127.0.0.1:8200/v1/pki_int/crl" - ``` - -1. Create the vault policy, as per the following example: - - ```properties - # Enable secrets engine - path "sys/mounts/*" { - capabilities = ["create", "read", "update", "delete", "list"] - } - - # List enabled secrets engine - path "sys/mounts" { - capabilities = ["read", "list"] - } - - # Work with pki secrets engine - path "pki*" { - capabilities = ["create", "read", "update", "delete", "list", "sudo"] - } - ``` - -1. Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: - - ```shell - vault token create -no-default-policy -policy=pki_policy - ``` - - You may also specify the following for your token: - - - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. - - `period` — If specified, the token can be infinitely renewed. - - YBA automatically tries to renew the token every 12 hours after it has passed 70% of its expiry window; as a result, you should set the TTL or period to be greater than 12 hours. - - For more information, refer to [Tokens](https://developer.hashicorp.com/vault/tutorials/tokens/tokens) in the Hashicorp documentation. - -1. Create a role that maps a name in the vault to a procedure for generating a certificate, as follows: - - ```sh - vault write /roles/ allow_any_name=true allow_subdomains=true max_ttl="8640h" - ``` - - Credentials are generated against this role. - -1. Issue certificates for nodes or a YugabyteDB client: - - - For a node, execute the following: - - ```sh - vault write /issue/ common_name="" ip_sans="" ttl="860h" - ``` - - - For YugabyteDB client, execute the following: - - ```sh - vault write /issue/ common_name="" - ``` - -### Use HashiCorp Vault-provided certificates to enable TLS - -When you create a universe, you can enable TLS using certificates provided by HashiCorp Vault, as follows: - -1. Navigate to **Configs > Security > Encryption in Transit**. -1. Click **Add Certificate** to open the **Add Certificate** dialog. -1. Select **Hashicorp**. -1. In the **Config Name** field, enter a meaningful name for your configuration. -1. In the **Vault Address** field, specify a valid URL that includes the port number. The format is `http://0.0.0.0:0000`, which corresponds to `VAULT_HOSTNAME:0000` -1. In the **Secret Token** field, specify the secret token for the vault. -1. In the **Role** field, specify the role used for creating certificates. -1. Optionally, provide the secret engine path on which the PKI is mounted. If you do not supply this information, `pki/` will be used. -1. Click **Add** to make the certificate available. -1. Go to **Universes > Create Universe** to open the **Create Universe** dialog. -1. Configure the universe. -1. Based on your requirements, select **Enable Node-to-Node TLS** and **Enable Client-to-Node TLS**. -1. Select an existing certificate from the **Root Certificate** list and then select the certificate that you have uploaded. -1. Create the universe. - -You can also edit TLS settings for an existing universe by navigating to **Universes**, opening a specific universe, clicking **Actions > Edit Security > Encryption in-Transit** to open the **TLS Configuration** dialog, and then modifying the required settings. - -## Kubernetes cert-manager - -For a universe created on Kubernetes, YugabyteDB Anywhere allows you to configure an existing running instance of the [cert-manager](https://cert-manager.io/) as a TLS certificate provider for a cluster, assuming that the following criteria are met: - -- The cert-manager is running in the Kubernetes cluster. -- A root or intermediate CA (either self-signed or external) is already configured on the cert-manager. The same root certificate file must be prepared for upload to YugabyteDB Anywhere. -- An Issuer or ClusterIssuer Kind is configured on the cert-manager and is ready to issue certificates using the previously-mentioned root or intermediate certificate. - -During the universe creation, you can enable TLS certificates issued by the cert-manager, as follows: - -1. Upload the root certificate to YugabyteDB Anywhere: - - - Prepare the root certificate in a file (for example, `root.crt`). - - Navigate to **Configs > Security > Encryption in Transit** and click **Add Certificate**. - - On the **Add Certificate** dialog shown in the following illustration, select **K8S cert-manager**: - - ![Add Certificate](/images/yp/security/kubernetes-cert-manager.png) - - - In the **Certificate Name** field, enter a meaningful name for your certificate configuration. - - Click **Upload Root Certificate** and select the root certificate file that you prepared. - - Click **Add** to make the certificate available. - -1. Configure the Kubernetes-based cloud provider by following instructions provided in [Configure region and zones](../../configure-yugabyte-platform/kubernetes/#configure-region-and-zones). In the **Add new region** dialog shown in the following illustration, you would be able to specify the Issuer name or the ClusterIssuer name for each zone. Because an Issuer Kind is a Kubernetes namespace-scoped resource, the zone definition should also set the **Namespace** field value if an Issuer Kind is selected: - - ![Add new region](/images/yp/security/kubernetes-cert-manager-add-region.png) - -1. Create the universe: - - - Navigate to **Universes** and click **Create Universe**. - - In the **Provider** field, select the cloud provider that you have configured in step 2. - - Complete the fields based on your requirements, and select **Enable Node-to-Node TLS** or **Enable Client-to-Node TLS**. - - Select the root certificate that you have uploaded in step 1. - - Click **Create**. - -### Troubleshoot - -If you encounter problems, you should verify the name of Issuer or ClusterIssuer in the Kubernetes cluster, as well as ensure that the Kubernetes cluster is in Ready state. You can use the following commands: - -```sh -kubectl get ClusterIssuer -``` - -```sh -kubectl -n Issuer -``` - -## Connect to clusters - -Using TLS, you can connect to the YSQL and YCQL endpoints. - -### Connect to a YSQL endpoint with TLS - -If you created your universe with the Client-to-Node TLS option enabled, then you must download client certificates to your client computer to establish connection to your database, as follows: - -- Navigate to the **Certificates** page and then to your universe's certificate. - -- Click **Actions** and select **Download YSQL Cert**, as shown in the following illustration. This triggers the download of the `yugabytedb.crt` and `yugabytedb.key` files. - - ![download-ysql-cert](/images/yp/encryption-in-transit/download-ysql-cert.png) - -- Optionally, when connecting to universes that are configured with custom CA-signed certificates, obtain the root CA and client YSQL certificate from your administrator. These certificates are not available on YugabyteDB Anywhere for downloading. - -- For testing with a `ysqlsh` client, paste the `yugabytedb.crt` and `yugabytedb.key` files into the `/.yugabytedb` directory and change the permissions to `0600`, as follows: - - ```sh - mkdir ~/.yugabytedb; cd ~/.yugabytedb - cp /yugabytedb.crt . - cp /yugabytedb.key . - chmod 600 yugabytedb.* - ``` - -- Run `ysqlsh` using the `sslmode=require` option, as follows: - - ```sh - cd - bin/ysqlsh -h 172.152.43.78 -p 5433 sslmode=require - ``` - - ```output - ysqlsh (11.2-YB-2.3.3.0-b0) - SSL connection (protocol: TLSv1.2, cipher: ECDHE-RSA-AES256-GCM-SHA384, bits: 256, compression: off) - Type "help" for help. - - yugabyte=# - ``` - -To use TLS from a different client, consult the client-specific documentation. For example, if you are using a PostgreSQL JDBC driver to connect to YugabyteDB, see [Configuring the client](https://jdbc.postgresql.org/documentation/head/ssl-client.html) for more details. - -If you are using PostgreSQL/YugabyteDB JDBC driver with SSL, you need to convert the certificates to DER format. To do this, you need to perform only steps 6 and 7 from [Set up SSL certificates for Java applications](../../../reference/drivers/java/postgres-jdbc-reference/#set-up-ssl-certificates-for-java-applications) section after downloading the certificates. - -### Connect to a YCQL endpoint with TLS - -If you created your universe with the Client-to-Node TLS option enabled, then you must download client certificates to your client computer to establish connection to your database, as follows: - -- Navigate to the **Certificates** page and then to your universe's certificate. - -- Click **Actions** and select **Download Root Cert**, as shown in the following illustration. This triggers the download of the `root.crt` file. - - ![download-root-cert](/images/yp/encryption-in-transit/download-root-cert.png) - -- Optionally, when connecting to universes that are configured with custom CA-signed certificates, obtain the root CA and client YSQL certificate from your administrator. These certificates are not available on YugabyteDB Anywhere for downloading. - -- Set `SSL_CERTFILE` environment variable to point to the location of the downloaded root certificate. - -- Run `ycqlsh` using the `-ssl` option, as follows: - - ```sh - cp /root.crt ~/.yugabytedb/root.crt - export SSL_CERTFILE=~/.yugabytedb/root.crt - bin/ycqlsh 172.152.43.78 --ssl - ``` - - ```output - Connected to local cluster at 172.152.43.78:9042. - [ycqlsh 5.0.1 | Cassandra 3.9-SNAPSHOT | CQL spec 3.4.2 | Native protocol v4] - Use HELP for help. - ycqlsh> - ``` - -To use TLS from a different client, consult the client-specific documentation. For example, if you are using a Cassandra driver to connect to YugabyteDB, see [SSL](https://docs.datastax.com/en/developer/python-driver/3.19/security/#ssl). - -## Validate certificates - -When configuring and using certificates, SSL issues may occasionally arise. You can validate your certificates and keys as follows: - -1. Verify that the CA CRT and CA private key match by executing the following commands: - - ```shell - openssl rsa -noout -modulus -in ca.key | openssl md5 - openssl x509 -noout -modulus -in ca.crt | openssl md5 - - \# outputs should match - ``` - -2. Verify that the CA CRT is actually a certificate authority by executing the following command: - - ```shell - openssl x509 -text -noout -in ca.crt - - \# Look for fields - - X509v3 Basic Constraints: - - CA:TRUE - ``` - -3. Verify that certificates and keys are in PEM format (as opposed to the DER or other format). If these artifacts are not in the PEM format and you require assistance with converting them or identifying the format, consult [Converting certificates](https://support.globalsign.com/ssl/ssl-certificates-installation/converting-certificates-openssl). - -4. Ensure that the private key does not have a passphrase associated with it. For information on how to identify this condition, see [Decrypt an encrypted SSL RSA private key](https://techjourney.net/how-to-decrypt-an-enrypted-ssl-rsa-private-key-pem-key/). - -## Enforcing TLS versions - -As TLS 1.0 and 1.1 are no longer accepted by PCI compliance, and considering significant vulnerabilities around these versions of the protocol, it is recommended that you migrate to TLS 1.2 or later versions. - -You can set the TLS version for node-to-node and client-node communication. To enforce TLS 1.2, add the following flag for YB-TServer: - -```shell -ssl_protocols = tls12 -``` - -To enforce the minimum TLS version of 1.2, you need to specify all available subsequent versions for YB-TServer, as follows: - -```shell -ssl_protocols = tls12,tls13 -``` - -In addition, as the `ssl_protocols` setting does not propagate to PostgreSQL, it is recommended that you specify the minimum TLS version (`ssl_min_protocol_version`) for PostgreSQL by setting the following YB-TServer flag: - -```shell ---ysql_pg_conf_csv="ssl_min_protocol_version='TLSv1.2'" -``` - -## Use self-signed and custom CA certificates - -YugabyteDB Anywhere uses TLS to protect data in transit when connecting to other services, including: - -- LDAP -- OIDC -- Webhook -- [S3 backup storage](../../back-up-restore-universes/configure-backup-storage/) -- Hashicorp Vault -- [YBA high availability](../../administer-yugabyte-platform/high-availability/) - -If you are using self-signed or custom CA certificates, YugabyteDB cannot verify your TLS connections unless you add the certificates to the YugabyteDB Anywhere Trust Store. - -### Add certificates to your trust store - -To add a certificate to the YugabyteDB Anywhere Trust Store, do the following: - -1. Navigate to **Admin > CA Certificates**. - -1. Click **Upload Trusted CA Certificate**. - -1. Enter a name for the certificate. - -1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. - -### Rotate a certificate in your trust store - -To rotate a certificate in your YugabyteDB Anywhere Trust Store, do the following: - -1. Navigate to **Admin > CA Certificates**. - -1. Click the **...** button for the certificate and choose **Update Certificate**. - -1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. - -### Delete a certificate in your trust store - -To delete a certificate in your YugabyteDB Anywhere Trust Store, do the following: - -1. Navigate to **Admin > CA Certificates**. - -1. Click the **...** button for the certificate and choose **Delete**, then click **Delete CA Certificate**. diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/_index.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/_index.md new file mode 100644 index 000000000000..410e552ad5de --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/_index.md @@ -0,0 +1,90 @@ +--- +title: Encryption in transit in YugabyteDB Anywhere +headerTitle: Encryption in transit +linkTitle: Encryption in transit +description: Use encryption in transit (TLS) to secure data traffic. +headcontent: Secure intra-node and application traffic +menu: + stable_yugabyte-platform: + parent: security + identifier: enable-encryption-in-transit + weight: 40 +type: indexpage +showRightNav: true +--- + +YugabyteDB Anywhere allows you to protect data in transit by using the following: + +- Node-to-Node TLS to encrypt intra-node communication between YB-Master and YB-TServer nodes. +- Client-to-Node TLS to encrypt communication between a universe and clients. This includes applications, shells (ysqlsh, ycqlsh, psql, and so on), and other tools, using the YSQL and YCQL APIs. + +## Manage certificates + +Use YugabyteDB Anywhere to manage certificates used for encryption in transit. + +{{}} + + {{}} + + {{}} + + {{}} + + {{}} + +{{}} + +## Enable encryption in transit + +You enable Node-to-Node and Client-to-Node encryption in transit when you [create a universe](../../create-deployments/create-universe-multi-zone/). + +You can also enable and disable encryption in transit for an existing universe as follows: + +1. Navigate to your universe. +1. Click **Actions > Edit Security > Encryption in-Transit** to open the **Manage encryption in transit** dialog. +1. Enable or disable the **Enable encryption in transit for this Universe** option. +1. Click **Apply**. + +### Enforce TLS versions + +As TLS 1.0 and 1.1 are no longer accepted by PCI compliance, and considering significant vulnerabilities around these versions of the protocol, it is recommended that you migrate to TLS 1.2 or later versions. + +You can set the TLS version for node-to-node and client-node communication. To enforce TLS 1.2, add the following flag for YB-TServer: + +```shell +ssl_protocols = tls12 +``` + +To enforce the minimum TLS version of 1.2, you need to specify all available subsequent versions for YB-TServer, as follows: + +```shell +ssl_protocols = tls12,tls13 +``` + +In addition, as the `ssl_protocols` setting does not propagate to PostgreSQL, it is recommended that you specify the minimum TLS version (`ssl_min_protocol_version`) for PostgreSQL by setting the following YB-TServer flag: + +```shell +--ysql_pg_conf_csv="ssl_min_protocol_version='TLSv1.2'" +``` + +## Learn more + +- [Securing YugabyteDB: Server-to-Server Encryption in Transit](https://www.yugabyte.com/blog/yugabytedb-server-to-server-encryption/) +- [Securing YugabyteDB: SQL Client-to-Server Encryption in Transit](https://www.yugabyte.com/blog/securing-yugabytedb-client-to-server-encryption/) +- [Securing YugabyteDB: CQL Client-to-Server Encryption in Transit](https://www.yugabyte.com/blog/securing-yugabytedb-part-3-cql-client-server-encryption-transit/) diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-ca.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-ca.md new file mode 100644 index 000000000000..0a2dd3fe5b0f --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-ca.md @@ -0,0 +1,113 @@ +--- +title: Add CA-signed certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add CA-signed certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + stable_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-2-ca + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +For universes created with an on-premises provider, instead of using self-signed certificates, you can use third-party certificates from external certificate authorities (CA). The third-party CA root certificate must be configured in YugabyteDB Anywhere. You also have to copy the custom CA root certificate, node certificate, and node key to the appropriate on-premises provider nodes. + +## Prerequisites + +The server and CA certificates must adhere to the following criteria: + +- Be stored in a `.crt` file, with both the certificate and the private key being in the PEM format. + + If your certificates and keys are stored in the PKCS12 format, you can [convert them to the PEM format](#convert-certificates-and-keys-from-pkcs12-to-pem-format). + +The server certificates must adhere to the following criteria: + +- Contain IP addresses of the database nodes in the Common Name or in the Subject Alternative Name. For on-premises universes where nodes are identified using DNS addresses, the server certificates should include the DNS names of the database nodes in the Common Name or Subject Alternate Name (wildcards are acceptable). + +## Add CA-signed certificates + +The following procedure describes how to install certificates on the database nodes. You have to repeat these steps for every database node that is to be used in the creation of a universe. + +### Obtain certificates and keys + +Obtain the keys and the custom CA-signed certificates for each of the on-premise nodes for which you are configuring node-to-node TLS. In addition, obtain the keys and the custom signed certificates for client access for configuring client-to-node TLS. + +### Copy the certificates to each node + +For each on-premises provider node, copy the custom CA certificate, node certificate, and node key to that node's file system. + +If you are enabling client-to-node TLS, make sure to copy the client-facing server certificate and client-facing server key to each of the nodes. + +In addition, ensure the following: + +- The file names and file paths of different certificates and keys are identical across all the database nodes. For example, if you name your CA root certificate as `ca.crt` on one node, then you must name it `ca.crt` on all the nodes. Similarly, if you copy `ca.crt` to `/opt/yugabyte/keys` on one node, then you must copy `ca.crt` to the same path on other nodes. +- The `yugabyte` system user has read permissions to all the certificates and keys. + +### Add the CA certificate to YugabyteDB Anywhere + +Add a CA-signed certificate to YugabyteDB Anywhere as follows: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **CA Signed**, as per the following illustration: + + ![Add CA certificate](/images/yp/encryption-in-transit/add-cert.png) + +1. In the **Certificate Name** field, enter a meaningful name for your certificate. + +1. Upload the custom CA certificate (including any intermediate certificates in the chain) as the Root CA certificate. + + If you use an intermediate CA/issuer, but do not have the complete chain of certificates, then you need to create a bundle by executing the `cat intermediate-ca.crt root-ca.crt > bundle.crt` command, and then use this bundle as the root certificate. You might also want to [verify the certificate chain](#verify-certificate-chain). + +1. Enter the file paths for each of the certificates on the nodes. These are the paths from the previous step. + +1. Use the **Expiration Date** field to specify the expiration date of the certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. + +1. Click **Add** to make the certificate available. + +You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. + +### Verify certificate chain + +Perform the following steps to verify your certificates: + +1. Execute the following verify command which checks the database node certificate (node.crt) against the root CA certificate (ca.crt): + + ```sh + openssl verify ca.crt node.crt + ``` + +1. Verify that the node certificate (`node.crt`) and the node private key (`node.key`) match. See [How do I verify that a private key matches a certificate?](https://www.ssl247.com/knowledge-base/detail/how-do-i-verify-that-a-private-key-matches-a-certificate-openssl-1527076112539/ka03l0000015hscaay/) + +1. Verify that the node certificate and Root CA certificate expiration is at least 3 months by checking the validity field in the output of the following commands: + + ```sh + openssl x509 -in node.crt -text -noout + ``` + + ```sh + openssl x509 -in ca.crt -text -noout + ``` + +1. Verify that the node certificate Common Name (CN) or Subject Alternate Name (SAN) contains the IP address or DNS name of each on-premises node on which the nodes are deployed. + + {{< note >}} +Each entry you provide for the CN or SAN must match the on-premises node as entered in the provider configuration. For example, if the node address is entered as a DNS address in the on-premises provider configuration, you must use the same DNS entry in the CN or SAN, not the resolved IP address. + {{< /note >}} + + If you face any issue with the above verification, you can customize the level of certificate validation while creating a universe that uses these certificates. Refer to [Customizing the verification of RPC server certificate by the client](https://www.yugabyte.com/blog/yugabytedb-server-to-server-encryption/#customizing-the-verification-of-rpc-server-certificate-by-the-client). + +{{< note >}} +The client certificates and keys are required only if you intend to use [PostgreSQL certificate-based authentication](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html#:~:text=independent%20authentication%20option-,clientcert,-%2C%20which%20can%20be). +{{< /note >}} diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-hashicorp.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-hashicorp.md new file mode 100644 index 000000000000..3ed069276439 --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-hashicorp.md @@ -0,0 +1,189 @@ +--- +title: Add Hashicorp Vault certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add Hashicorp Vault certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + stable_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-3-hashicorp + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +YugabyteDB Anywhere allows you to add an encryption in transit configuration using HashiCorp Vault with a public key infrastructure (PKI) secret engine. This configuration can be used to enable TLS for different clusters and YugabyteDB instances. You can apply this configuration to node-to-node encryption, client-to-node encryption, or both. + +## Prerequisites + +For the correct configuration, the following criteria must be met: + +- HashiCorp Vault is unsealed. +- HashiCorp Vault with the PKI secret engine is configured and enabled. +- HashiCorp Vault URL is accessible by YugabyteDB Anywhere. +- Because HashiCorp Vault is accessed via an authentication token mechanism, a token must be created beforehand while creating a key provider with appropriate permissions. +- HashiCorp Vault needs to be running and always accessible to YugabyteDB Anywhere. +- HashiCorp PKI certificate revocation list (CRL) or CA URLs must be accessible from each node server. +- Appropriate certificates and roles have been created for YugabyteDB Anywhere usage. +- Node servers are able to validate certificates. +- Required permissions have been provided to perform various key management operations. + +## Configure HashiCorp Vault + +Before you can start configuring HashiCorp Vault, install it on a virtual machine, as per instructions provided in [Install Vault](https://www.vaultproject.io/docs/install). The vault can be set up as a multi-node cluster. Ensure that your vault installation meets the following requirements: + +- Has transit secret engine enabled. +- Its seal and unseal mechanism is secure and repeatable. +- Its token creation mechanism is repeatable. + +You need to configure HashiCorp Vault in order to use it with YugabyteDB Anywhere, as follows: + +1. Create a vault configuration file that references your nodes and specifies the address, as follows: + + ```properties + storage "raft" { + path = "./vault/data/" + node_id = "node1" + } + + listener "tcp" { + address = "127.0.0.1:8200" + tls_disable = "true" + } + + api_addr = "http://127.0.0.1:8200" + cluster_addr = "https://127.0.0.1:8201" + ui = true + disable_mlock = true + default_lease_ttl = "768h" + max_lease_ttl = "8760h" + ``` + + Replace `127.0.0.1` with the vault web address. + + For additional configuration options, see [Parameters](https://www.vaultproject.io/docs/configuration#parameters). + +1. Initialize the vault server by following instructions provided in [Operator init](https://www.vaultproject.io/docs/commands/operator/init). + +1. Allow access to the vault by following instructions provided in [Unsealing](https://www.vaultproject.io/docs/concepts/seal#unsealing). + +1. Enable the secret engine by executing the following command: + + ```shell + vault secrets enable pki + ``` + +1. Configure the secret engine, as follows: + + - Create a root CA or configure the top-level CA. + + - Optionally, create an intermediate CA chain and sign them. + + - Create an intermediate CA for YugabyteDB, as per the following example: + + ```sh + export pki=pki + export pki_int="pki_int" + export role_i=RoleName + export ip="s.test.com" + + vault secrets enable -path=$pki_int pki + vault secrets tune -max-lease-ttl=43800h $pki_int + vault write $pki_int/intermediate/generate/internal common_name="test.com Intermediate Authority" ttl=43800h -format=json | jq -r '.data.csr' > pki_int.csr + + \# *** dump the output of the preceding command in pki_int.csr + + vault write $pki/root/sign-intermediate csr=@pki_int.csr format=pem_bundle ttl=43800h -format=json | jq -r .data.certificate > i_signed.pem + + \# *** dump the output in i_signed.pem + + vault write $pki_int/intermediate/set-signed certificate=@i_signed.pem + vault write $pki_int/config/urls issuing_certificates="http://127.0.0.1:8200/v1/pki_int/ca" crl_distribution_points="http://127.0.0.1:8200/v1/pki_int/crl" + ``` + +1. Create the vault policy, as per the following example: + + ```properties + # Enable secrets engine + path "sys/mounts/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } + + # List enabled secrets engine + path "sys/mounts" { + capabilities = ["read", "list"] + } + + # Work with pki secrets engine + path "pki*" { + capabilities = ["create", "read", "update", "delete", "list", "sudo"] + } + ``` + +1. Generate a token with appropriate permissions (as per the referenced policy) by executing the following command: + + ```shell + vault token create -no-default-policy -policy=pki_policy + ``` + + You may also specify the following for your token: + + - `ttl` — Time to live (TTL). If not specified, the default TTL of 32 days is used, which means that the generated token will expire after 32 days. + - `period` — If specified, the token can be infinitely renewed. + + YugabyteDB Anywhere automatically tries to renew the token every 12 hours after it has passed 70% of its expiry window; as a result, you should set the TTL or period to be greater than 12 hours. + + For more information, refer to [Tokens](https://developer.hashicorp.com/vault/tutorials/tokens/tokens) in the Hashicorp documentation. + +1. Create a role that maps a name in the vault to a procedure for generating a certificate, as follows: + + ```sh + vault write /roles/ allow_any_name=true allow_subdomains=true max_ttl="8640h" + ``` + + Credentials are generated against this role. + +1. Issue certificates for nodes or a YugabyteDB client: + + - For a node, execute the following: + + ```sh + vault write /issue/ common_name="" ip_sans="" ttl="860h" + ``` + + - For YugabyteDB client, execute the following: + + ```sh + vault write /issue/ common_name="" + ``` + +## Add HashiCorp Vault-provided certificates + +When you create a universe, you can enable TLS using certificates provided by HashiCorp Vault, as follows: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **Hashicorp**. + + ![Add Hashicorp certificate](/images/yp/encryption-in-transit/add-hashicorp-cert.png) + +1. In the **Config Name** field, enter a meaningful name for your configuration. + +1. In the **Vault Address** field, specify a valid URL that includes the port number. The format is `http://0.0.0.0:0000`, which corresponds to `VAULT_HOSTNAME:0000` + +1. In the **Secret Token** field, specify the secret token for the vault. + +1. In the **Role** field, specify the role used for creating certificates. + +1. Optionally, provide the secret engine path on which the PKI is mounted. If you do not supply this information, `pki/` will be used. + +1. Click **Add** to make the certificate available. diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-kubernetes.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-kubernetes.md new file mode 100644 index 000000000000..b899acfdf666 --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-kubernetes.md @@ -0,0 +1,69 @@ +--- +title: Add cert-manager certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add cert-manager certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + stable_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-4-kubernetes + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +For a universe created on Kubernetes, YugabyteDB Anywhere allows you to configure an existing running instance of the [cert-manager](https://cert-manager.io/) as a TLS certificate provider for a cluster. + +## Prerequisites + +The following criteria must be met: + +- The cert-manager is running in the Kubernetes cluster. +- A root or intermediate CA (either self-signed or external) is already configured on the cert-manager. The same CA certificate file, including any intermediate CAs, must be prepared for upload to YugabyteDB Anywhere. For intermediate certificates, the chained CA certificate can be constructed using a command similar to `cat intermediate-ca.crt root-ca.crt > bundle.crt`. +- An Issuer or ClusterIssuer Kind is configured on the cert-manager and is ready to issue certificates using the previously-mentioned root or intermediate certificate. +- Prepare the root certificate in a file (for example, `root.crt`). + +## Add certificates using cert-manager + +Add TLS certificates issued by the cert-manager as follows: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **K8S cert-manager**. + + ![Add Kubernetes Certificate](/images/yp/encryption-in-transit/add-k8s-cert.png) + +1. In the **Certificate Name** field, enter a meaningful name for your certificate. + +1. Click **Upload Root Certificate** and select the CA certificate file that you prepared. + +1. Click **Add** to make the certificate available. + +## Configure the provider + +After the certificate is added to YugabyteDB Anywhere, configure the Kubernetes provider configuration by following instructions provided in [Configure region and zones](../../../configure-yugabyte-platform/kubernetes/#configure-region-and-zones). + +In the **Add new region** dialog shown in the following illustration, you would be able to specify the Issuer name or the ClusterIssuer name for each zone. Because an Issuer Kind is a Kubernetes namespace-scoped resource, the zone definition should also set the **Namespace** field value if an Issuer Kind is selected. + +![Add new region](/images/yp/security/kubernetes-cert-manager-add-region.png) + +## Troubleshoot + +If you encounter problems, you should verify the name of Issuer or ClusterIssuer in the Kubernetes cluster, as well as ensure that the Kubernetes cluster is in Ready state. You can use the following commands: + +```sh +kubectl get ClusterIssuer +``` + +```sh +kubectl -n Issuer +``` diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-self.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-self.md new file mode 100644 index 000000000000..f2600a6cf505 --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/add-certificate-self.md @@ -0,0 +1,99 @@ +--- +title: Add self-signed certificates to YugabyteDB Anywhere +headerTitle: Add certificates +linkTitle: Add certificates +description: Add self-signed certificates to YugabyteDB Anywhere. +headcontent: Use your own certificates for encryption in transit +menu: + stable_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: add-certificate-1-self + weight: 20 +type: docs +--- + +{{}} +{{}} +{{}} +{{}} +{{}} +{{}} + +Instead of using YugabyteDB Anywhere-provided certificates, you can use your own self-signed certificates that you upload to YugabyteDB Anywhere. + +## Prerequisites + +The certificates must meet the following criteria: + +- Be in the `.crt` format and the private key must be in the `.pem` format, with both of these artifacts available for upload. + +YugabyteDB Anywhere produces the node (leaf) certificates from the uploaded certificates and copies the certificate chain, leaf certificate, and private key to the nodes in the cluster. + +### Convert certificates and keys from PKCS12 to PEM format + +If your certificates and keys are stored in the PKCS12 format, you can convert them to the PEM format using OpenSSL. + +Start by extracting the certificate via the following command: + +```sh +openssl pkcs12 -in cert-archive.pfx -out cert.pem -clcerts -nokeys +``` + +To extract the key and write it to the PEM file unencrypted, execute the following command: + +```sh +openssl pkcs12 -in cert-archive.pfx -out key.pem -nocerts -nodes +``` + +If the key is protected by a passphrase in the PKCS12 archive, you are prompted for the passphrase. + +## Add self-signed certificates + +To add self-signed certificates to YugabyteDB Anywhere: + +1. Navigate to **Configs > Security > Encryption in Transit**. + +1. Click **Add Certificate** to open the **Add Certificate** dialog. + +1. Select **Self Signed**. + + ![Add Self Signed certificate](/images/yp/encryption-in-transit/add-self-cert.png) + +1. In the **Certificate Name** field, enter a meaningful name for your certificate. + +1. Click **Upload Root Certificate**, then browse to the root certificate file (`.crt`) and upload it. + +1. Click **Upload Key**, then browse to the root certificate file (`.key`) and upload it. + +1. In the **Expiration Date** field, specify the expiration date of the root certificate. To find this information, execute the `openssl x509 -in -text -noout` command and note the **Validity Not After** date. + +1. Click **Add** to make the certificate available. + +## Validate certificates + +When configuring and using certificates, SSL issues may occasionally arise. You can validate your certificates and keys as follows: + +- Verify that the CA CRT and CA private key match by executing the following commands: + + ```shell + openssl rsa -noout -modulus -in ca.key | openssl md5 + openssl x509 -noout -modulus -in ca.crt | openssl md5 + + \# outputs should match + ``` + +- Verify that the CA CRT is actually a certificate authority by executing the following command: + + ```shell + openssl x509 -text -noout -in ca.crt + + \# Look for fields + + X509v3 Basic Constraints: + + CA:TRUE + ``` + +- Verify that certificates and keys are in PEM format (as opposed to the DER or other format). If these artifacts are not in the PEM format and you require assistance with converting them or identifying the format, consult [Converting certificates](https://support.globalsign.com/ssl/ssl-certificates-installation/converting-certificates-openssl). + +- Ensure that the private key does not have a passphrase associated with it. For information on how to identify this condition, see [Decrypt an encrypted SSL RSA private key](https://techjourney.net/how-to-decrypt-an-enrypted-ssl-rsa-private-key-pem-key/). diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/auto-certificate.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/auto-certificate.md new file mode 100644 index 000000000000..be142ccab0a1 --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/auto-certificate.md @@ -0,0 +1,95 @@ +--- +title: Automatically generated certificates on YugabyteDB Anywhere +headerTitle: Auto-generated certificates +linkTitle: Auto-generated certificates +description: YugabyteDB Anywhere-generated self-signed certificates. +headcontent: Let YugabyteDB Anywhere manage certificates for your universe +menu: + stable_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: auto-certificate + weight: 10 +type: docs +--- + +YugabyteDB Anywhere can automatically create and manage self-signed certificates for universes when you create them. These certificates may be shared between universes in a single instance of YugabyteDB Anywhere. + +Automatically generated certificates are named using the following convention: + +```sh +yb-environment-universe_name +``` + +where *environment* is the environment type (either `dev`, `stg`, `demo`, or `prod`) that was used during the tenant registration (admin user creation), and *universe_name* is the provided universe name. + +YugabyteDB Anywhere generates the root CA certificate, root private key, and node-level certificates (assuming node-to-node or client-to-node encryption is enabled), and then provisions those artifacts to the database nodes any time nodes are created or added to the cluster. The following three files are copied to each node: + +1. The root certificate (`ca.cert`). +1. The node certificate (`node.ip_address.crt`). +1. The node private key (`node.ip_address.key`). + +YugabyteDB Anywhere retains the root certificate and the root private key for all interactions with the cluster. + +To view the certificate details, navigate to **Configs > Security > Encryption in Transit** and click **Show details**. + +## Customize the organization name in self-signed certificates + +YugabyteDB Anywhere automatically creates self-signed certificates when you run some workflows, such as create universe. The organization name in certificates is set to `example.com` by default. + +If you are using YugabyteDB Anywhere version 2.18.2 or later to manage universes with YugabyteDB version 2.18.2 or later, you can set a custom organization name using the global [runtime configuration](../../../administer-yugabyte-platform/manage-runtime-config/) flag, `yb.tlsCertificate.organizationName`. + +Note that, for the change to take effect, you need to set the flag _before_ you run a workflow that generates a self-signed certificate. + +Customize the organization name as follows: + +1. In YugabyteDB Anywhere, navigate to **Admin** > **Advanced** and select the **Global Configuration** tab. +1. In the **Search** bar, enter `yb.tlsCertificate.organizationName` to view the flag, as per the following illustration: + + ![Custom Organization name](/images/yp/encryption-in-transit/custom-org-name.png) + +1. Click **Actions** > **Edit Configuration**, enter a new Config Value, and click **Save**. + +## Validate custom organization name + +You can verify the organization name by running the following `openssl x509` command: + +```sh +openssl x509 -in ca.crt -text +``` + +```output {hl_lines=[6]} +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1683277970271 (0x187eb2f7b5f) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=yb-dev-sb-ybdemo-univ1~2, O=example.com + Validity + Not Before: May 5 09:12:50 2023 GMT + Not After : May 5 09:12:50 2027 GMT +``` + +Notice that default value is `O=example.com`. + +After setting the runtime configuration to a value of your choice, (`org-foo` in this example), you should see output similar to the following: + +```sh +openssl x509 -in ca.crt -text -noout +``` + +```output +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1689376612248 (0x18956b15f98) + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo + Validity + Not Before: Jul 14 23:16:52 2023 GMT + Not After : Jul 14 23:16:52 2027 GMT + Subject: CN = yb-dev-sb-ybdemo-univ1~2, O = org-foo + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: +``` diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/rotate-certificates.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/rotate-certificates.md new file mode 100644 index 000000000000..e1500da8997a --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/rotate-certificates.md @@ -0,0 +1,55 @@ +--- +title: Rotate certificates on YugabyteDB Anywhere +headerTitle: Rotate certificates +linkTitle: Rotate certificates +description: Rotate certificates on YugabyteDB Anywhere. +headcontent: Rotate certificates used by a universe +menu: + stable_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: rotate-certificates + weight: 30 +type: docs +--- + +You can rotate certificates for universes configured with the same type of certificates. This involves replacing existing certificates with new database node certificates. + +Before rotating certificates, ensure that you have added the certificates to YugabyteDB Anywhere. Refer to [Add certificates](../add-certificate-self/). + +**Client-to-node certificates** + +Regardless of whether the client-to-node certificates are expired or not expired, you can always trigger a rolling upgrade to rotate the certificates. + +- If the universe was created before v2.16.6, then the rotation requires a restart, which can be done in a rolling manner with no downtime. +- If the universe was created after v2.16.6, then the rotation can be done without a restart and no downtime. + +**Node-to-node certificates** + +If the certificate has expired, the rotation requires a simultaneous restart of all nodes, resulting in some downtime. + +If the certificate has not expired, the rotation can be done using a rolling upgrade. + +- If the universe was created before v2.16.6, then the rotation requires a restart, which can be done in a rolling manner with no downtime. +- If the universe is created after v2.16.6, then the rotation can be done without a restart and no downtime. + +You can always opt to not perform rolling updates to update all nodes at the same time, but this will result in downtime. + +## Rotate certificates + +To modify encryption in transit settings and rotate certificates for a universe, do the following: + +1. Navigate to your universe. + +1. Click **Actions > Edit Security > Encryption in-Transit** to open the **Manage encryption in transit** dialog. + + ![Rotate certificates](/images/yp/encryption-in-transit/rotate-cert.png) + +1. To rotate the CA certificate, on the **Certificate Authority** tab, select the new CA certificate(s). + + If you wish to have YBA generate a new self-signed CA certificate [automatically](../auto-certificate/), delete the root certificate field. + +1. To rotate the server certificates, on the **Server Certificate** tab, select the **Rotate Node-to-Node Server Certificate** and **Rotate Client-to-Node Server Certificate** options as appropriate. + +1. Select the **Use rolling upgrade to apply this change** option to perform the upgrade in a rolling update (recommended) and enter the number of seconds to wait between node upgrades. + +1. Click **Apply**. diff --git a/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/trust-store.md b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/trust-store.md new file mode 100644 index 000000000000..286799bcd215 --- /dev/null +++ b/docs/content/stable/yugabyte-platform/security/enable-encryption-in-transit/trust-store.md @@ -0,0 +1,54 @@ +--- +title: Add CA-signed certificates to YugabyteDB Anywhere +headerTitle: Add certificates to your trust store +linkTitle: Trust store +description: Add certificates to the YugabyteDB Anywhere trust store. +headcontent: Add certificates for third-party services +menu: + stable_yugabyte-platform: + parent: enable-encryption-in-transit + identifier: trust-store + weight: 40 +type: docs +--- + +YugabyteDB Anywhere uses certificates to validate connections between YugabyteDB Anywhere and other external services, including: + +- [LDAP](../../../administer-yugabyte-platform/ldap-authentication/) +- [OIDC](../../../administer-yugabyte-platform/oidc-authentication/) +- [Webhook](../../../alerts-monitoring/set-up-alerts-health-check/) +- [S3 backup storage](../../../back-up-restore-universes/configure-backup-storage/) +- [Hashicorp Vault](../../create-kms-config/hashicorp-kms/) +- Other [YugabyteDB Anywhere high availability](../../../administer-yugabyte-platform/high-availability/) replicas. + +When using self-signed or custom CA certificates, to enable YugabyteDB Anywhere to validate your TLS connections, you _must_ add the certificates to the YugabyteDB Anywhere Trust Store + +## Add certificates to your trust store + +To add a certificate to the YugabyteDB Anywhere Trust Store, do the following: + +1. Navigate to **Admin > CA Certificates**. + +1. Click **Upload Trusted CA Certificate**. + +1. Enter a name for the certificate. + +1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. + +## Rotate a certificate in your trust store + +To rotate a certificate in your YugabyteDB Anywhere Trust Store, do the following: + +1. Navigate to **Admin > CA Certificates**. + +1. Click the **...** button for the certificate and choose **Update Certificate**. + +1. Click **Upload**, select your certificate (in .crt format) and click **Save CA Certificate**. + +## Delete a certificate in your trust store + +To delete a certificate in your YugabyteDB Anywhere Trust Store, do the following: + +1. Navigate to **Admin > CA Certificates**. + +1. Click the **...** button for the certificate and choose **Delete**, then click **Delete CA Certificate**. diff --git a/docs/content/stable/yugabyte-platform/security/security-checklist-yp.md b/docs/content/stable/yugabyte-platform/security/security-checklist-yp.md deleted file mode 100644 index 28fe48937101..000000000000 --- a/docs/content/stable/yugabyte-platform/security/security-checklist-yp.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -title: Security checklist for YugabyteDB Anywhere -headerTitle: Security checklist -linkTitle: Security checklist -description: Security measures that can be implemented to protect your YugabyteDB Anywhere and YugabyteDB universes. -menu: - stable_yugabyte-platform: - parent: security - identifier: security-checklist-yp - weight: 10 -type: docs ---- - -You can apply security measures to protect your YugabyteDB Anywhere instance and YugabyteDB universes. - -## Network Security - -You need to ensure that YugabyteDB Anywhere and the database run in a trusted network environment. You should restrict machine and port access, based on the following guidelines: - -- Servers running YugabyteDB services are directly accessible only by YugabyteDB Anywhere, servers running the application, and database administrators. -- Only YugabyteDB Anywhere and servers running applications can connect to YugabyteDB services on the RPC ports. Access to the YugabyteDB ports should be denied to everybody else. - -For information on configuring ports, refer to [Configure ports](../customize-ports/). - -## Database authentication - -Authentication requires that all clients provide valid credentials before they can connect to a YugabyteDB universe. The authentication credentials in YugabyteDB are stored internally in the YB-Master system tables. The authentication mechanisms available to users depends on what is supported and exposed by the YSQL and YCQL APIs. - -You enable authentication for the YSQL and YCQL APIs when you deploy a universe. See [Enable database authentication](../authorization-platform/#enable-database-authentication). - -YugabyteDB Anywhere and YugabyteDB also support LDAP and OIDC for managing authentication. See [Database authentication](../authentication/). - -For more information on authentication in YugabyteDB, see [Enable authentication](../../../secure/enable-authentication/). - -## Role-based access control - -Roles can be assigned to grant users only the essential privileges based on the operations they need to perform in YugabyteDB Anywhere, and in YugabyteDB universes. - -To manage access to your YugabyteDB Anywhere instance, typically you create a [Super Admin role first](../../install-yugabyte-platform/create-admin-user/). The Super Admin can create additional admins and other users with fewer privileges. For information on how to manage YugabyteDB Anywhere users and roles, see [Manage YugabyteDB Anywhere users](../../administer-yugabyte-platform/anywhere-rbac/). - -For information on how to manage database roles and users, see [Database authorization](../authorization-platform). - -## Encryption in transit - -Encryption in transit (TLS) ensures that network communication between servers is secure. You can configure YugabyteDB to use TLS to encrypt intra-cluster and client to server network communication. You should enable encryption in transit in YugabyteDB universes and clients to ensure the privacy and integrity of data transferred over the network. - -For more information, see [Enable encryption in transit](../enable-encryption-in-transit). - -## Encryption at rest - -Encryption at rest ensures that data at rest, stored on disk, is protected. You can configure YugabyteDB universes with a user-generated symmetric key to perform universe-wide encryption. - -Encryption at rest in YugabyteDB Anywhere uses a master key to encrypt and decrypt universe keys. The master key details are stored in YugabyteDB Anywhere in [key management service (KMS) configurations](../create-kms-config/aws-kms/). You enable encryption at rest for a universe by assigning the universe a KMS configuration. The master key designated in the configuration is then used for generating the universe keys used for encrypting the universe data. - -For more information, see [Enable encryption at rest](../enable-encryption-at-rest). diff --git a/docs/content/stable/yugabyte-platform/yba-overview.md b/docs/content/stable/yugabyte-platform/yba-overview.md index 858604670f65..fea817704bb9 100644 --- a/docs/content/stable/yugabyte-platform/yba-overview.md +++ b/docs/content/stable/yugabyte-platform/yba-overview.md @@ -14,7 +14,7 @@ type: docs YugabyteDB Anywhere (YBA) is a self-managed database-as-a-service that allows you to deploy and operate YugabyteDB database clusters (also known as universes) at scale. -In YBA, a database cluster is called a [universe](../../architecture/key-concepts/#universe), and the terms are used interchangeably. More precisely, a universe in YBA always consists of one (and only one) primary cluster, and can optionally also include a single [read replica](../../architecture/docdb-replication/read-replicas/) cluster attached to the primary cluster. +In YBA, a database cluster is called a [universe](../../architecture/key-concepts/#universe), and the terms are used interchangeably. More precisely, a universe in YBA always consists of one (and only one) [primary cluster](../../architecture/key-concepts/#primary-cluster), and can optionally also include a single [read replica](../../architecture/key-concepts/#read-replica-cluster/) cluster attached to the primary cluster. ## Features diff --git a/docs/content/v2.20/reference/configuration/yb-tserver.md b/docs/content/v2.20/reference/configuration/yb-tserver.md index ffd6cdf57e58..75073fcda92e 100644 --- a/docs/content/v2.20/reference/configuration/yb-tserver.md +++ b/docs/content/v2.20/reference/configuration/yb-tserver.md @@ -732,6 +732,14 @@ Default: `-1` (disables logging statement durations) Specifies the lowest YSQL message level to log. +##### --ysql_output_buffer_size + +Size of YSQL layer output buffer, in bytes. YSQL buffers query responses in this output buffer until either a buffer flush is requested by the client or the buffer overflows. + +As long as no data has been flushed from the buffer, the database can retry queries on retryable errors. For example, you can increase the size of the buffer so that YSQL can retry [read restart errors](../../../architecture/transactions/read-restart-error). + +Default: `262144` (256kB, type: int32) + ### YCQL The following flags support the use of the [YCQL API](../../../api/ycql/): diff --git a/docs/content/v2.20/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md b/docs/content/v2.20/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md index af9794133d52..b9f886475bda 100644 --- a/docs/content/v2.20/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md +++ b/docs/content/v2.20/yugabyte-platform/administer-yugabyte-platform/oidc-authentication.md @@ -63,10 +63,13 @@ You configure OIDC as follows: [Google OIDC discovery endpoint](https://developers.google.com/identity/protocols/oauth2/openid-connect#an-id-tokens-payload) is an example of such file. For most identity providers, `/.well-known/openid-configuration` is appended to the issuer to generate the metadata URL for OIDC specifications. - - In the **Scope** field, enter your identity provider OIDC scope that is allowed to be requested. This field accepts a space-separated list of values. If left blank, all scopes will be considered. - - In the **Email Attribute** field, enter the OIDC scope containing the user email identifier. This field accepts a case-sensitive custom configuration. Typically, this field is left blank. - - If you have an airgapped installation, where YBA cannot access the Discovery URL, provide the OIDC configuration for the identity provider directly. + If you have an airgapped installation, where YugabyteDB Anywhere cannot access the Discovery URL, provide the OIDC configuration for the identity provider directly. To do this, click **Configure OIDC Provider Metadata** and paste the OIDC configuration document from your identity provider (in JSON format) into the field. + - In the **Scope** field, enter your identity provider OIDC scope that is allowed to be requested. This field accepts a space-separated list of values. If left blank, all scopes will be considered. + - In the **Email Attribute** field, enter the OIDC scope containing the user email identifier. This field accepts a case-sensitive custom configuration. Typically, this field is left blank. + - If you have configured OIDC to use [refresh tokens](https://openid.net/specs/openid-connect-core-1_0.html#RefreshTokens), in the **Refresh Token URL** field, enter the URL of the refresh token endpoint. + - If you have configured [OIDC enhancements](../../security/authentication/oidc-authentication-aad/#enable-oidc-enhancements), you can select the **Display JWT token on login** option to allow users to access their JWT from the YugabyteDB Anywhere sign in page. See [Set up OIDC with Azure AD on YugabyteDB Anywhere](../../security/authentication/oidc-authentication-aad/#set-up-oidc-with-azure-ad-on-yugabytedb-anywhere). + 1. Click **Save**. diff --git a/docs/content/v2.20/yugabyte-platform/security/authentication/oidc-authentication-aad.md b/docs/content/v2.20/yugabyte-platform/security/authentication/oidc-authentication-aad.md index 6110d7667b98..24c04335aab6 100644 --- a/docs/content/v2.20/yugabyte-platform/security/authentication/oidc-authentication-aad.md +++ b/docs/content/v2.20/yugabyte-platform/security/authentication/oidc-authentication-aad.md @@ -114,7 +114,11 @@ To register an application, do the following: 1. Select the tenant for the application. -1. Set the redirect URI. This is where the IdP redirects after authentication. +1. Set the redirect URI. This is where the IdP redirects after authentication. The URI is in the following form: + + ```sh + https:///api/v1/callback?client_name=OidcClient + ``` 1. Click **Register**. diff --git a/docs/data/currentVersions.json b/docs/data/currentVersions.json index e2dafb06bb82..b9ac51386077 100644 --- a/docs/data/currentVersions.json +++ b/docs/data/currentVersions.json @@ -40,9 +40,9 @@ { "series": "v2.18", "display": "v2.18 (STS)", - "version": "2.18.8.0", + "version": "2.18.8.1", "versionShort": "2.18.8", - "appVersion": "2.18.8.0-b42", + "appVersion": "2.18.8.1-b3", "isStable": true, "isLTS": false, "isSTS": true, diff --git a/docs/netlify.toml b/docs/netlify.toml index 50c2bf78fada..34c83c4c29c4 100644 --- a/docs/netlify.toml +++ b/docs/netlify.toml @@ -747,7 +747,7 @@ [[redirects]] from = "/:version/reference/connectors/*" - to = "/preview/integrations/apache-kafka/" + to = "/preview/explore/change-data-capture/" # Redirect for troubleshoot diff --git a/docs/static/images/architecture/cdc-logical-replication-architecture.png b/docs/static/images/architecture/cdc-logical-replication-architecture.png new file mode 100644 index 000000000000..558d7967d17c Binary files /dev/null and b/docs/static/images/architecture/cdc-logical-replication-architecture.png differ diff --git a/docs/static/images/architecture/cdc_service_vwal_interaction.png b/docs/static/images/architecture/cdc_service_vwal_interaction.png new file mode 100644 index 000000000000..c0a8005b4537 Binary files /dev/null and b/docs/static/images/architecture/cdc_service_vwal_interaction.png differ diff --git a/docs/static/images/architecture/vwal_walsender_interaction.png b/docs/static/images/architecture/vwal_walsender_interaction.png new file mode 100644 index 000000000000..043386e9fd8f Binary files /dev/null and b/docs/static/images/architecture/vwal_walsender_interaction.png differ diff --git a/docs/static/images/yp/encryption-in-transit/add-cert.png b/docs/static/images/yp/encryption-in-transit/add-cert.png index 25dbaf2e456f..68690f671a60 100644 Binary files a/docs/static/images/yp/encryption-in-transit/add-cert.png and b/docs/static/images/yp/encryption-in-transit/add-cert.png differ diff --git a/docs/static/images/yp/encryption-in-transit/add-hashicorp-cert.png b/docs/static/images/yp/encryption-in-transit/add-hashicorp-cert.png new file mode 100644 index 000000000000..2b5294620c53 Binary files /dev/null and b/docs/static/images/yp/encryption-in-transit/add-hashicorp-cert.png differ diff --git a/docs/static/images/yp/encryption-in-transit/add-k8s-cert.png b/docs/static/images/yp/encryption-in-transit/add-k8s-cert.png new file mode 100644 index 000000000000..bc7f81d73489 Binary files /dev/null and b/docs/static/images/yp/encryption-in-transit/add-k8s-cert.png differ diff --git a/docs/static/images/yp/encryption-in-transit/add-self-cert.png b/docs/static/images/yp/encryption-in-transit/add-self-cert.png new file mode 100644 index 000000000000..d27bccf47f74 Binary files /dev/null and b/docs/static/images/yp/encryption-in-transit/add-self-cert.png differ diff --git a/docs/static/images/yp/encryption-in-transit/rotate-cert.png b/docs/static/images/yp/encryption-in-transit/rotate-cert.png new file mode 100644 index 000000000000..fa75b6ccfefe Binary files /dev/null and b/docs/static/images/yp/encryption-in-transit/rotate-cert.png differ diff --git a/docs/static/images/yp/oidc-auth-220.png b/docs/static/images/yp/oidc-auth-220.png index 50055d8e7122..543ac175fa8f 100644 Binary files a/docs/static/images/yp/oidc-auth-220.png and b/docs/static/images/yp/oidc-auth-220.png differ diff --git a/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgReplicationSlot.java b/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgReplicationSlot.java index 2c73dd859e78..4fe714230623 100644 --- a/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgReplicationSlot.java +++ b/java/yb-pgsql/src/test/java/org/yb/pgsql/TestPgReplicationSlot.java @@ -919,9 +919,12 @@ void replicationConnectionConsumptionAllDataTypes(String pluginName) throws Exce + "col_tsrange TSRANGE, " + "col_tstzrange TSTZRANGE, " + "col_daterange DATERANGE, " - + "col_discount coupon_discount_type)"; + + "col_hstore HSTORE, " + + "col_discount coupon_discount_type, " + +" col_discount_array coupon_discount_type[])"; try (Statement stmt = connection.createStatement()) { + stmt.execute("CREATE EXTENSION IF NOT EXISTS hstore;"); stmt.execute("CREATE TYPE coupon_discount_type AS ENUM ('FIXED', 'PERCENTAGE');"); stmt.execute(create_stmt); if (pluginName.equals(PG_OUTPUT_PLUGIN_NAME)) { @@ -947,7 +950,8 @@ void replicationConnectionConsumptionAllDataTypes(String pluginName) throws Exce + "'550e8400-e29b-41d4-a716-446655440000', B'101010', '2024-02-01 12:34:56+00:00', " + "'[1,10)', '[100,1000)', '[2024-01-01, 2024-12-31)', " + "'[2024-01-01 00:00:00+00:00, 2024-12-31 15:59:59+00:00)', " - + "'[2024-01-01, 2024-12-31)', 'FIXED');"); + + "'[2024-01-01, 2024-12-31)','key1 => value1, key2 => value2'::hstore, 'FIXED', " + + "array['FIXED', 'PERCENTAGE']::coupon_discount_type[]);"); } PGReplicationStream stream = replConnection.replicationStream() @@ -960,12 +964,14 @@ void replicationConnectionConsumptionAllDataTypes(String pluginName) throws Exce List result = new ArrayList(); // 1 Relation, begin, type, insert and commit record. - result.addAll(receiveMessage(stream, 5)); + result.addAll(receiveMessage(stream, 7)); List expectedResult = new ArrayList() { { add(PgOutputBeginMessage.CreateForComparison(LogSequenceNumber.valueOf("0/4"), 2)); + add(PgOutputTypeMessage.CreateForComparison("public", "hstore")); add(PgOutputTypeMessage.CreateForComparison("public", "coupon_discount_type")); + add(PgOutputTypeMessage.CreateForComparison("public", "_coupon_discount_type")); if (pluginName.equals(YB_OUTPUT_PLUGIN_NAME)) { add(PgOutputRelationMessage.CreateForComparison("public", "test_table", 'c', Arrays.asList(PgOutputRelationMessageColumn.CreateForComparison("a", 23), @@ -1003,8 +1009,12 @@ void replicationConnectionConsumptionAllDataTypes(String pluginName) throws Exce PgOutputRelationMessageColumn.CreateForComparison("col_tsrange", 3908), PgOutputRelationMessageColumn.CreateForComparison("col_tstzrange", 3910), PgOutputRelationMessageColumn.CreateForComparison("col_daterange", 3912), + // The Oids for columns below are not fixed. Changing the order of creation of + // objects (extensions, tables etc.) in the test will these Oids. + PgOutputRelationMessageColumn.CreateForComparison("col_hstore", 16385), PgOutputRelationMessageColumn.CreateForComparison( - "col_discount", /* IGNORED */ 0, /* compareDataType */ false)))); + "col_discount", 16518, /* compareDataType */ false), + PgOutputRelationMessageColumn.CreateForComparison("col_discount_array",16517)))); } else { // The replica identity for test_table in case of pgoutput is DEFAULT. add(PgOutputRelationMessage.CreateForComparison("public", "test_table", 'd', @@ -1043,10 +1053,14 @@ void replicationConnectionConsumptionAllDataTypes(String pluginName) throws Exce PgOutputRelationMessageColumn.CreateForComparison("col_tsrange", 3908), PgOutputRelationMessageColumn.CreateForComparison("col_tstzrange", 3910), PgOutputRelationMessageColumn.CreateForComparison("col_daterange", 3912), + // The Oids for columns below are not fixed. Changing the order of creation of + // objects (extensions, tables etc.) in the test will these Oids. + PgOutputRelationMessageColumn.CreateForComparison("col_hstore", 16385), PgOutputRelationMessageColumn.CreateForComparison( - "col_discount", /* IGNORED */ 0, /* compareDataType */ false)))); + "col_discount", 16518, /* compareDataType */ false), + PgOutputRelationMessageColumn.CreateForComparison("col_discount_array",16517)))); } - add(PgOutputInsertMessage.CreateForComparison(new PgOutputMessageTuple((short) 36, + add(PgOutputInsertMessage.CreateForComparison(new PgOutputMessageTuple((short) 38, Arrays.asList(new PgOutputMessageTupleColumnValue("1"), new PgOutputMessageTupleColumnValue("110110"), new PgOutputMessageTupleColumnValue("t"), @@ -1086,7 +1100,9 @@ void replicationConnectionConsumptionAllDataTypes(String pluginName) throws Exce convertTimestampToSystemTimezone("2024-01-01T00:00:00.00Z"), convertTimestampToSystemTimezone("2024-12-31T15:59:59.00Z"))), new PgOutputMessageTupleColumnValue("[2024-01-01,2024-12-31)"), - new PgOutputMessageTupleColumnValue("FIXED"))))); + new PgOutputMessageTupleColumnValue("\"key1\"=>\"value1\", \"key2\"=>\"value2\""), + new PgOutputMessageTupleColumnValue("FIXED"), + new PgOutputMessageTupleColumnValue("{FIXED,PERCENTAGE}"))))); add(PgOutputCommitMessage.CreateForComparison( LogSequenceNumber.valueOf("0/4"), LogSequenceNumber.valueOf("0/5"))); } diff --git a/managed/devops/opscli/ybops/cloud/azure/method.py b/managed/devops/opscli/ybops/cloud/azure/method.py index a9790030f875..63133753756d 100644 --- a/managed/devops/opscli/ybops/cloud/azure/method.py +++ b/managed/devops/opscli/ybops/cloud/azure/method.py @@ -51,7 +51,8 @@ def __init__(self, base_command): def add_extra_args(self): super(AzureCreateInstancesMethod, self).add_extra_args() self.parser.add_argument("--volume_type", - choices=["premium_lrs", "standardssd_lrs", "ultrassd_lrs"], + choices=["premium_lrs", "standardssd_lrs", "ultrassd_lrs", + "premiumv2_lrs"], default="premium_lrs", help="Volume type for Azure instances.") self.parser.add_argument("--security_group_id", default=None, help="Azure comma delimited security group IDs.") diff --git a/managed/devops/opscli/ybops/cloud/azure/utils.py b/managed/devops/opscli/ybops/cloud/azure/utils.py index 77db8b6351f3..91a5add216f1 100644 --- a/managed/devops/opscli/ybops/cloud/azure/utils.py +++ b/managed/devops/opscli/ybops/cloud/azure/utils.py @@ -50,8 +50,10 @@ SUBNET_ID_FORMAT_STRING = NETWORK_PROVIDER_BASE_PATH + "/virtualNetworks/{}/subnets/{}" NSG_ID_FORMAT_STRING = NETWORK_PROVIDER_BASE_PATH + "/networkSecurityGroups/{}" ULTRASSD_LRS = "ultrassd_lrs" +PREMIUMV2_LRS = "premiumv2_lrs" VNET_ID_FORMAT_STRING = NETWORK_PROVIDER_BASE_PATH + "/virtualNetworks/{}" AZURE_SKU_FORMAT = {"premium_lrs": "Premium_LRS", + "premiumv2_lrs": "PremiumV2_LRS", "standardssd_lrs": "StandardSSD_LRS", ULTRASSD_LRS: "UltraSSD_LRS"} YUGABYTE_VNET_PREFIX = "yugabyte-vnet-{}" @@ -429,7 +431,7 @@ def append_disk(self, vm, vm_name, disk_name, size, lun, zone, vol_type, region, if tags: disk_params["tags"] = tags - if vol_type == ULTRASSD_LRS: + if vol_type == ULTRASSD_LRS or vol_type == PREMIUMV2_LRS: if disk_iops is not None: disk_params['disk_iops_read_write'] = disk_iops if disk_throughput is not None: diff --git a/managed/devops/python3_requirements.txt b/managed/devops/python3_requirements.txt index 6f59672e95c5..14dca34e292f 100644 --- a/managed/devops/python3_requirements.txt +++ b/managed/devops/python3_requirements.txt @@ -22,7 +22,7 @@ ansible==2.9.27 ansible-vault==2.1.0 azure-common==1.1.28 azure-identity==1.6.1 -azure-mgmt-compute==23.1.0 +azure-mgmt-compute==27.1.0 azure-mgmt-privatedns==1.0.0 azure-mgmt-network==19.3.0 azure-mgmt-resource==19.0.0 diff --git a/managed/devops/python3_requirements_frozen.txt b/managed/devops/python3_requirements_frozen.txt index 84d5c65b73b4..36b6ac8acc2f 100644 --- a/managed/devops/python3_requirements_frozen.txt +++ b/managed/devops/python3_requirements_frozen.txt @@ -2,9 +2,9 @@ adal==1.2.7 ansible==2.9.27 ansible-vault==2.1.0 azure-common==1.1.28 -azure-core==1.30.1 +azure-core==1.30.2 azure-identity==1.6.1 -azure-mgmt-compute==23.1.0 +azure-mgmt-compute==27.1.0 azure-mgmt-core==1.4.0 azure-mgmt-network==19.3.0 azure-mgmt-privatedns==1.0.0 @@ -14,18 +14,18 @@ boto==2.49.0 boto3==1.34.23 botocore==1.34.23 cachetools==4.2.4 -certifi==2024.2.2 +certifi==2024.6.2 cffi==1.16.0 chardet==4.0.0 click==8.1.7 -cryptography==42.0.7 +cryptography==42.0.8 deepdiff==5.5.0 distro==1.5.0 fabric==2.2.1 geomet==0.2.1.post1 google-api-core==1.22.0 google-api-python-client==1.10.0 -googleapis-common-protos==1.63.0 +googleapis-common-protos==1.63.2 google-auth==1.19.2 google-auth-httplib2==0.2.0 grpcio==1.57.0 @@ -38,7 +38,7 @@ Jinja2==3.0.3 jmespath==1.0.1 MarkupSafe==2.0.1 mitogen==0.2.9 -msal==1.28.0 +msal==1.29.0 msal-extensions==0.3.1 msrest==0.7.1 msrestazure==0.6.4.post1 @@ -46,7 +46,7 @@ oauth2client==3.0.0 oauthlib==3.2.2 ordered-set==4.0.2 paramiko==3.4.0 -portalocker==2.8.2 +portalocker==2.10.0 protobuf==4.23.4 psycopg2==2.9.9 pyasn1==0.6.0 @@ -62,10 +62,10 @@ PyYAML==6.0.1 requests==2.25.1 requests-oauthlib==1.3.0 rsa==4.9 -s3transfer==0.10.1 +s3transfer==0.10.2 scp==0.14.5 six==1.16.0 -typing_extensions==4.11.0 +typing_extensions==4.12.2 uritemplate==3.0.1 -urllib3==1.26.18 +urllib3==1.26.19 yb-cassandra-driver==3.25.0 diff --git a/managed/node-agent/cli/node/configure.go b/managed/node-agent/cli/node/configure.go index fc49b5dca826..98b99ced8060 100644 --- a/managed/node-agent/cli/node/configure.go +++ b/managed/node-agent/cli/node/configure.go @@ -470,7 +470,9 @@ func configureEnabledEgress(ctx context.Context, cmd *cobra.Command) { util.ConsoleLogger().Fatalf(ctx, "Unable to register node agent - %s", err.Error()) } } else { - util.ConsoleLogger().Fatalf(ctx, "Unable to check for existing node agent - %s", err.Error()) + // Node agent already exists on YBA. But, it cannot be verified with the local key. + util.ConsoleLogger().Fatalf(ctx, "Node agent already exists on YBA but local "+ + "credentials may be invalid. It may need to be unregistered first - %s", err.Error()) } util.ConsoleLogger().Info(ctx, "Node Agent Configuration Successful") } diff --git a/managed/node-agent/resources/node-agent-provision.yaml b/managed/node-agent/resources/node-agent-provision.yaml index 9c0b67b584e4..c8c93a921607 100644 --- a/managed/node-agent/resources/node-agent-provision.yaml +++ b/managed/node-agent/resources/node-agent-provision.yaml @@ -12,7 +12,6 @@ ynp: use_system_level_systemd: false ip_address: "127.0.0.1" tmp_directory: /tmp - mount_points: /data # Comma separated values. yba: url: @@ -32,3 +31,5 @@ yba: cores: cores memory_size: size volume_size: size + mount_points: + - /mnt/d1 diff --git a/managed/node-agent/resources/ynp/commands/provision_command.py b/managed/node-agent/resources/ynp/commands/provision_command.py index 3c8ae873e4a8..8cd23192950f 100644 --- a/managed/node-agent/resources/ynp/commands/provision_command.py +++ b/managed/node-agent/resources/ynp/commands/provision_command.py @@ -123,7 +123,6 @@ def _generate_template(self): context = self.config[key] context["templatedir"] = os.path.join(os.path.dirname(module[1]), "templates") - logger.info(context) module_instance = module[0]() rendered_template = module_instance.render_templates(context) if rendered_template is not None: diff --git a/managed/node-agent/resources/ynp/configs/config.j2 b/managed/node-agent/resources/ynp/configs/config.j2 index b93148165671..b70ec256f02e 100644 --- a/managed/node-agent/resources/ynp/configs/config.j2 +++ b/managed/node-agent/resources/ynp/configs/config.j2 @@ -32,7 +32,7 @@ nproc_limit = 12000 vm_swappiness = 0 kernel_core_pattern = {{ ynp.yb_home_dir }}/cores/core_%%p_%%t_%%E vm_max_map_count = 262144 -mount_points = {{ ynp.mount_points }} +mount_points = {{ yba.instance_type.mount_points | join(' ') }} [ConfigureOs.limits] core = unlimited @@ -71,5 +71,4 @@ ports = 7000 7100 9000 9100 18018 22 5433 9042 9070 9300 12000 13000 yb_user = yugabyte yb_home_dir = {{ ynp.yb_home_dir }} tmp_directory = {{ ynp.tmp_directory }} -mount_points = {{ ynp.mount_points }} diff --git a/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/precheck.j2 b/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/precheck.j2 index 128e5d9c618e..9268a5888e91 100644 --- a/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/precheck.j2 +++ b/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/precheck.j2 @@ -87,11 +87,13 @@ else add_result "kernel.core_pattern" "FAIL" "kernel.core_pattern is set to $kernel_core_pattern_value (expected: {{ kernel_core_pattern }})" fi -mount_point_array={{ mount_points }} +threshold=49 #Gigabytes +# Convert the space-separated string to an array in bash +IFS=' ' read -r -a mount_points_array <<< {{ mount_points }} # Verify each mount point for mount_point in "${mount_point_array[@]}"; do if [ -d "$mount_point" ]; then - if [ -w "$mount_point" ] && [ "$(stat -c %A "$mount_point" | cut -c 10)" == "w" ]; then + if [ -w "$mount_point" ] && [ $(( $(stat -c %a "$mount_point") % 10 & 2 )) -ne 0 ]; then result="PASS" message="Directory $mount_point exists and is world-writable." echo "[PASS] $message" @@ -107,6 +109,19 @@ for mount_point in "${mount_point_array[@]}"; do echo "[FAIL] $message" any_fail=1 fi - add_result "$mount_point Check" "$result" "$message" + + # Get the available disk space in gigabytes. + free_space_gb=$(df -BG --output=avail "$MOUNT_POINT" | tail -n 1 | tr -d 'G ') + if [ "$free_space_gb" -gt "$threshold" ]; then + result="PASS" + message="Sufficient disk space available: ${AVAILABLE}G" + echo "[PASS] $message" + else + result="FAIL" + message="Insufficient disk space: ${free_space_gb}G available, ${threshold}G required" + echo "[FAIL] $message" + any_fail=1 + fi + add_result "$mount_point Free space check" "$result" "$message" done diff --git a/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/run.j2 b/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/run.j2 index 50e94dbeb573..7cb60c58272c 100644 --- a/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/run.j2 +++ b/managed/node-agent/resources/ynp/modules/provision/configure_os/templates/run.j2 @@ -68,66 +68,4 @@ sysctl -w vm.max_map_count={{ vm_max_map_count }} echo "Kernel settings configured." - -echo "Mounting volumes." -# Excluding the known mounts that we have to avoid. -{% set excluded_mounts = ["/", "boot", "efi"] %} - -# Generate a unique backup filename for /etc/fstab -backup_file="/etc/fstab.bak.$(date +%s)" -cp /etc/fstab $backup_file -echo "Backup of /etc/fstab created at $backup_file." - -# Get list of available volumes excluding the root and specified excluded mounts -volumes=$(lsblk -lnpo NAME,MOUNTPOINT | awk '$2 == "" {print $1}' | grep -vE 'NAME|MOUNTPOINT') - -# Prepare and mount each volume -index=1 -for volume in $volumes; do - if [[ $volume =~ /dev/nvme[0-9]n[0-9]p[0-9]+ ]]; then - echo "Skipping volume $volume" - continue - fi - mount_point="/mnt/d${index}" - - mkdir -p ${mount_point} - - if mount | grep -qE "^${volume}|^${volume}[0-9]"; then - echo "${volume} or one of its partitions is currently mounted. Skipping..." - continue - fi - - # Check if filesystem already exists - if ! blkid ${volume} | grep -q "TYPE=\"xfs\""; then - mkfs -t xfs ${volume} - echo "Filesystem created on ${volume}." - else - echo "Filesystem already exists on ${volume}." - fi - - # Add entry to /etc/fstab if it doesn't already exist - if ! grep -q "^${volume}" /etc/fstab; then - echo "${volume} ${mount_point} xfs noatime 0 0" | tee -a /etc/fstab - else - echo "Entry for ${volume} already exists in /etc/fstab." - fi - - # Mount the volume if it's not already mounted - if ! mountpoint -q ${mount_point}; then - mount ${mount_point} || fail "Failed to mount ${volume} at ${mount_point}." - echo "Mounted ${volume} at ${mount_point}." - else - echo "${mount_point} is already mounted." - fi - - # Set ownership and permissions - chown yugabyte:yugabyte ${mount_point} || fail "Failed to set ownership for ${mount_point}." - chmod 755 ${mount_point} || fail "Failed to set permissions for ${mount_point}." - echo "Set ownership and permissions for ${mount_point}." - - index=$((index + 1)) -done - -echo "Volume mounting process completed." - echo "OS Configuration applied successfully." diff --git a/managed/node-agent/resources/ynp/modules/provision/node_agent/node_agent.py b/managed/node-agent/resources/ynp/modules/provision/node_agent/node_agent.py index 0ef27cd2b52c..4d78a807397c 100644 --- a/managed/node-agent/resources/ynp/modules/provision/node_agent/node_agent.py +++ b/managed/node-agent/resources/ynp/modules/provision/node_agent/node_agent.py @@ -104,7 +104,7 @@ def _generate_instance_type_payload(self, context): 'volumeDetailsList': [] } } - mount_points = context.get('instance_type_mount_points').split(',') + mount_points = context.get('instance_type_mount_points').strip("[]").replace("'", "").split(", ") for mp in mount_points: volume_detail = { 'volumeSizeGB': context.get('instance_type_volume_size'), @@ -167,7 +167,7 @@ def _create_instance_if_not_exists(self, context, provider): logging.error(f"Request error: {req_err}") except ValueError as json_err: logging.error(f"Error parsing JSON response: {json_err}") - + def render_templates(self, context): node_agent_enabled = False yba_url = context.get('url') @@ -219,7 +219,7 @@ def render_templates(self, context): except requests.exceptions.RequestException as req_err: logging.error(f"Request error: {req_err}") - + add_node_payload = self._generate_add_node_payload(context) add_node_payload_file = os.path.join(context.get('tmp_directory'), 'add_node_to_provider.json') with open(add_node_payload_file, 'w') as f: diff --git a/managed/node-agent/util/certs_util.go b/managed/node-agent/util/certs_util.go index e5ebcc75984c..ce89dd7b15e4 100644 --- a/managed/node-agent/util/certs_util.go +++ b/managed/node-agent/util/certs_util.go @@ -156,7 +156,7 @@ func ServerKeyPath(config *Config) string { // The JWT is signed using the key in the certs directory. func GenerateJWT(ctx context.Context, config *Config) (string, error) { keyFilepath := ServerKeyPath(config) - privateKey, err := ioutil.ReadFile(keyFilepath) + privateKey, err := os.ReadFile(keyFilepath) if err != nil { FileLogger().Errorf(ctx, "Error while reading the private key: %s", err.Error()) return "", err diff --git a/managed/src/main/java/api/v2/controllers/AuthenticationApiControllerImp.java b/managed/src/main/java/api/v2/controllers/AuthenticationApiControllerImp.java new file mode 100644 index 000000000000..8256fb358648 --- /dev/null +++ b/managed/src/main/java/api/v2/controllers/AuthenticationApiControllerImp.java @@ -0,0 +1,31 @@ +// Copyright (c) Yugabyte, Inc. + +package api.v2.controllers; + +import api.v2.handlers.AuthenticationHandler; +import api.v2.models.GroupMappingSpec; +import com.google.inject.Inject; +import java.util.List; +import java.util.UUID; +import play.mvc.Http; + +public class AuthenticationApiControllerImp extends AuthenticationApiControllerImpInterface { + + @Inject AuthenticationHandler authHandler; + + @Override + public List listMappings(Http.Request request, UUID cUUID) throws Exception { + return authHandler.listMappings(cUUID); + } + + @Override + public void updateGroupMappings( + Http.Request request, UUID cUUID, List groupMappingSpec) throws Exception { + authHandler.updateGroupMappings(request, cUUID, groupMappingSpec); + } + + @Override + public void deleteGroupMappings(Http.Request request, UUID cUUID, UUID gUUID) throws Exception { + authHandler.deleteGroupMappings(request, cUUID, gUUID); + } +} diff --git a/managed/src/main/java/api/v2/handlers/AuthenticationHandler.java b/managed/src/main/java/api/v2/handlers/AuthenticationHandler.java new file mode 100644 index 000000000000..59a42ec234ba --- /dev/null +++ b/managed/src/main/java/api/v2/handlers/AuthenticationHandler.java @@ -0,0 +1,181 @@ +// Copyright (c) Yugabyte, Inc. + +package api.v2.handlers; + +import static play.mvc.Http.Status.BAD_REQUEST; +import static play.mvc.Http.Status.NOT_FOUND; + +import api.v2.mappers.RoleResourceDefinitionMapper; +import api.v2.models.GroupMappingSpec; +import api.v2.models.GroupMappingSpec.TypeEnum; +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.common.config.GlobalConfKeys; +import com.yugabyte.yw.common.config.RuntimeConfGetter; +import com.yugabyte.yw.common.rbac.RoleBindingUtil; +import com.yugabyte.yw.common.rbac.RoleResourceDefinition; +import com.yugabyte.yw.controllers.TokenAuthenticator; +import com.yugabyte.yw.models.GroupMappingInfo; +import com.yugabyte.yw.models.GroupMappingInfo.GroupType; +import com.yugabyte.yw.models.rbac.Role; +import com.yugabyte.yw.models.rbac.RoleBinding; +import com.yugabyte.yw.models.rbac.RoleBinding.RoleBindingType; +import io.ebean.annotation.Transactional; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import play.mvc.Http; + +@Slf4j +@Singleton +public class AuthenticationHandler { + + @Inject RuntimeConfGetter confGetter; + @Inject TokenAuthenticator tokenAuthenticator; + @Inject RoleBindingUtil roleBindingUtil; + + public List listMappings(UUID cUUID) throws Exception { + + checkRuntimeConfig(); + + List groupInfoList = + GroupMappingInfo.find.query().where().eq("customer_uuid", cUUID).findList(); + List specList = new ArrayList(); + for (GroupMappingInfo info : groupInfoList) { + GroupMappingSpec spec = + new GroupMappingSpec() + .groupIdentifier(info.getIdentifier()) + .type(TypeEnum.valueOf(info.getType().toString())) + .uuid(info.getGroupUUID()); + + List roleResourceDefinitions = new ArrayList<>(); + if (confGetter.getGlobalConf(GlobalConfKeys.useNewRbacAuthz)) { + // fetch all role rolebindings for the current group + List roleBindingList = RoleBinding.getAll(info.getGroupUUID()); + for (RoleBinding rb : roleBindingList) { + RoleResourceDefinition roleResourceDefinition = + new RoleResourceDefinition(rb.getRole().getRoleUUID(), rb.getResourceGroup()); + roleResourceDefinitions.add(roleResourceDefinition); + } + } else { + // No role bindings present if RBAC is off. + RoleResourceDefinition rrd = new RoleResourceDefinition(); + rrd.setRoleUUID(info.getRoleUUID()); + roleResourceDefinitions.add(rrd); + } + spec.setRoleResourceDefinitions( + RoleResourceDefinitionMapper.INSTANCE.toV2RoleResourceDefinitionList( + roleResourceDefinitions)); + specList.add(spec); + } + return specList; + } + + @Transactional + public void updateGroupMappings( + Http.Request request, UUID cUUID, List groupMappingSpec) { + boolean isSuperAdmin = tokenAuthenticator.superAdminAuthentication(request); + if (!isSuperAdmin) { + throw new PlatformServiceException(BAD_REQUEST, "Only SuperAdmin can create group mappings!"); + } + + checkRuntimeConfig(); + + for (GroupMappingSpec mapping : groupMappingSpec) { + GroupMappingInfo mappingInfo = + GroupMappingInfo.find + .query() + .where() + .eq("customer_uuid", cUUID) + .ieq("identifier", mapping.getGroupIdentifier()) + .findOne(); + + if (mappingInfo == null) { + // new entry for new group + log.info("Adding new group mapping entry for group: " + mapping.getGroupIdentifier()); + mappingInfo = + GroupMappingInfo.create( + cUUID, + mapping.getGroupIdentifier(), + GroupType.valueOf(mapping.getType().toString())); + } else { + // clear role bindings for existing group + clearRoleBindings(mappingInfo); + } + + List roleResourceDefinitions = + RoleResourceDefinitionMapper.INSTANCE.toV1RoleResourceDefinitionList( + mapping.getRoleResourceDefinitions()); + + roleBindingUtil.validateRoles(cUUID, roleResourceDefinitions); + roleBindingUtil.validateResourceGroups(cUUID, roleResourceDefinitions); + + if (confGetter.getGlobalConf(GlobalConfKeys.useNewRbacAuthz)) { + // Add role bindings if rbac is on. + for (RoleResourceDefinition rrd : roleResourceDefinitions) { + Role rbacRole = Role.getOrBadRequest(cUUID, rrd.getRoleUUID()); + RoleBinding.create(mappingInfo, RoleBindingType.Custom, rbacRole, rrd.getResourceGroup()); + } + // This role will be ignored when rbac is on. + mappingInfo.setRoleUUID(Role.get(cUUID, "ConnectOnly").getRoleUUID()); + } else { + validate(roleResourceDefinitions); + mappingInfo.setRoleUUID(roleResourceDefinitions.get(0).getRoleUUID()); + } + mappingInfo.save(); + } + } + + @Transactional + public void deleteGroupMappings(Http.Request request, UUID cUUID, UUID gUUID) { + boolean isSuperAdmin = tokenAuthenticator.superAdminAuthentication(request); + if (!isSuperAdmin) { + throw new PlatformServiceException(BAD_REQUEST, "Only SuperAdmin can delete group mappings!"); + } + + checkRuntimeConfig(); + + GroupMappingInfo entity = + GroupMappingInfo.find + .query() + .where() + .eq("customer_uuid", cUUID) + .eq("uuid", gUUID) + .findOne(); + if (entity == null) { + throw new PlatformServiceException(NOT_FOUND, "No group mapping found with uuid: " + gUUID); + } + + // Delete all role bindings + clearRoleBindings(entity); + log.info("Deleting Group Mapping with name: " + entity.getIdentifier()); + entity.delete(); + } + + private void clearRoleBindings(GroupMappingInfo mappingInfo) { + log.info("Clearing role bindings for group: " + mappingInfo.getIdentifier()); + List list = RoleBinding.getAll(mappingInfo.getGroupUUID()); + list.forEach(rb -> rb.delete()); + } + + private void checkRuntimeConfig() { + if (!confGetter.getGlobalConf(GlobalConfKeys.groupMappingRbac)) { + throw new PlatformServiceException( + BAD_REQUEST, "yb.security.group_mapping_rbac_support runtime config is disabled!"); + } + } + + /** + * Validation to make sure only a single system role is present when RBAC is off. + * + * @param cUUID + * @param rrdList + */ + private void validate(List rrdList) { + if (rrdList.size() != 1) { + throw new PlatformServiceException(BAD_REQUEST, "Need to specify a single system role!"); + } + } +} diff --git a/managed/src/main/java/api/v2/mappers/ResourceGroupMapper.java b/managed/src/main/java/api/v2/mappers/ResourceGroupMapper.java new file mode 100644 index 000000000000..232801fba83c --- /dev/null +++ b/managed/src/main/java/api/v2/mappers/ResourceGroupMapper.java @@ -0,0 +1,27 @@ +// Copyright (c) Yugabyte, Inc. + +package api.v2.mappers; + +import api.v2.models.ResourceDefinitionSpec; +import api.v2.models.ResourceGroupSpec; +import com.yugabyte.yw.models.rbac.ResourceGroup; +import com.yugabyte.yw.models.rbac.ResourceGroup.ResourceDefinition; +import org.mapstruct.InheritInverseConfiguration; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.factory.Mappers; + +@Mapper(config = CentralConfig.class) +public interface ResourceGroupMapper { + ResourceGroupMapper INSTANCE = Mappers.getMapper(ResourceGroupMapper.class); + + ResourceGroupSpec toV2ResourceGroup(ResourceGroup v1ResourceGroup); + + ResourceGroup toV1ResourceGroup(ResourceGroupSpec v2ResourceGroup); + + @Mapping(target = "resourceUuidSet", source = "resourceUUIDSet") + ResourceDefinitionSpec toV2ResourceDefinitionSpec(ResourceDefinition v1ResourceDefinition); + + @InheritInverseConfiguration + ResourceDefinition toV1ResourceDefinitionSpec(ResourceDefinitionSpec v2ResourceDefinition); +} diff --git a/managed/src/main/java/api/v2/mappers/RoleResourceDefinitionMapper.java b/managed/src/main/java/api/v2/mappers/RoleResourceDefinitionMapper.java new file mode 100644 index 000000000000..132c0b41ab9a --- /dev/null +++ b/managed/src/main/java/api/v2/mappers/RoleResourceDefinitionMapper.java @@ -0,0 +1,32 @@ +// Copyright (c) Yugabyte, Inc. + +package api.v2.mappers; + +import api.v2.models.RoleResourceDefinitionSpec; +import com.yugabyte.yw.common.rbac.RoleResourceDefinition; +import java.util.List; +import org.mapstruct.InheritInverseConfiguration; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.factory.Mappers; + +@Mapper( + config = CentralConfig.class, + uses = {ResourceGroupMapper.class}) +public interface RoleResourceDefinitionMapper { + RoleResourceDefinitionMapper INSTANCE = Mappers.getMapper(RoleResourceDefinitionMapper.class); + + @Mapping(target = "roleUuid", source = "roleUUID") + RoleResourceDefinitionSpec toV2RoleResourceDefinition( + RoleResourceDefinition v1RoleResourceDefinition); + + List toV2RoleResourceDefinitionList( + List v1RoleResourceDefinition); + + @InheritInverseConfiguration + RoleResourceDefinition toV1RoleResourceDefinition( + RoleResourceDefinitionSpec v2RoleResourceDefinitionSpec); + + List toV1RoleResourceDefinitionList( + List v2RoleResourceDefinition); +} diff --git a/managed/src/main/java/api/v2/mappers/UserIntentMapper.java b/managed/src/main/java/api/v2/mappers/UserIntentMapper.java index 973c06dfd6e7..899731a45063 100644 --- a/managed/src/main/java/api/v2/mappers/UserIntentMapper.java +++ b/managed/src/main/java/api/v2/mappers/UserIntentMapper.java @@ -127,6 +127,7 @@ default ClusterNodeSpec userIntentToClusterNodeSpec( @ValueMapping(target = "PERSISTENT", source = "Persistent"), @ValueMapping(target = "STANDARDSSD_LRS", source = "StandardSSD_LRS"), @ValueMapping(target = "PREMIUM_LRS", source = "Premium_LRS"), + @ValueMapping(target = "PREMIUMV2_LRS", source = "PremiumV2_LRS"), @ValueMapping(target = "ULTRASSD_LRS", source = "UltraSSD_LRS"), @ValueMapping(target = "LOCAL", source = "Local"), }) diff --git a/managed/src/main/java/com/yugabyte/yw/cloud/PublicCloudConstants.java b/managed/src/main/java/com/yugabyte/yw/cloud/PublicCloudConstants.java index 2ba355ed0f08..fb2380212320 100644 --- a/managed/src/main/java/com/yugabyte/yw/cloud/PublicCloudConstants.java +++ b/managed/src/main/java/com/yugabyte/yw/cloud/PublicCloudConstants.java @@ -89,6 +89,7 @@ public enum StorageType { Persistent(Common.CloudType.gcp), StandardSSD_LRS(Common.CloudType.azu), Premium_LRS(Common.CloudType.azu), + PremiumV2_LRS(Common.CloudType.azu, new Pair<>(3000, 80_000), new Pair<>(1, 1200)), UltraSSD_LRS(Common.CloudType.azu, new Pair<>(100, 160_000), new Pair<>(1, 3814)), Local(Common.CloudType.local); diff --git a/managed/src/main/java/com/yugabyte/yw/common/AppInit.java b/managed/src/main/java/com/yugabyte/yw/common/AppInit.java index 203ced7cd8da..02440186b6c1 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/AppInit.java +++ b/managed/src/main/java/com/yugabyte/yw/common/AppInit.java @@ -44,8 +44,11 @@ import com.yugabyte.yw.models.ExtraMigration; import com.yugabyte.yw.models.HighAvailabilityConfig; import com.yugabyte.yw.models.MetricConfig; +import com.yugabyte.yw.models.Principal; +import com.yugabyte.yw.models.Users; import com.yugabyte.yw.scheduler.JobScheduler; import com.yugabyte.yw.scheduler.Scheduler; +import db.migration.default_.common.R__Sync_System_Roles; import io.ebean.DB; import io.prometheus.client.CollectorRegistry; import io.prometheus.client.Gauge; @@ -133,6 +136,16 @@ public AppInit( Customer customer = Customer.getAll().get(0); alertDestinationService.createDefaultDestination(customer.getUuid()); alertConfigurationService.createDefaultConfigs(customer); + // Create system roles for the newly created customer. + R__Sync_System_Roles.syncSystemRoles(); + // Principal entry for newly created users. + for (Users user : Users.find.all()) { + Principal principal = Principal.get(user.getUuid()); + if (principal == null) { + log.info("Adding Principal entry for user with email: " + user.getEmail()); + new Principal(user).save(); + } + } } String storagePath = AppConfigHelper.getStoragePath(); diff --git a/managed/src/main/java/com/yugabyte/yw/common/PlacementInfoUtil.java b/managed/src/main/java/com/yugabyte/yw/common/PlacementInfoUtil.java index b3f7721351b2..899586f2e9b5 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/PlacementInfoUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/PlacementInfoUtil.java @@ -820,7 +820,6 @@ private static boolean isDedicatedModeChanged(Cluster cluster, Set static void setPerAZRF(PlacementInfo placementInfo, int rf, UUID defaultRegionUUID) { LOG.info("Setting per AZ replication factor. Default region {}", defaultRegionUUID); List sortedAZs = getAZsSortedByNumNodes(placementInfo); - int numNodesInUniverse = sortedAZs.stream().map(az -> az.numNodesInAZ).reduce(0, Integer::sum); // Reset per-AZ RF to 0 placementInfo.azStream().forEach(az -> az.replicationFactor = 0); @@ -882,16 +881,21 @@ static void setPerAZRF(PlacementInfo placementInfo, int rf, UUID defaultRegionUU LOG.debug("Special case when RF=3 and number of zones= 2, using 1-1 distribution"); return; } + + sortedAZPlacements.removeIf( + az -> az.getSecond().replicationFactor >= az.getSecond().numNodesInAZ); // Set per-AZ RF according to node distribution across AZs. // We already have one replica in each region. Now placing other. int i = 0; - while ((placedReplicas < rf) && (i < numNodesInUniverse)) { - Pair az = sortedAZPlacements.get(i % sortedAZs.size()); - if (az.getSecond().replicationFactor < az.getSecond().numNodesInAZ) { - az.getSecond().replicationFactor++; - placedReplicas++; + while ((placedReplicas < rf) && sortedAZPlacements.size() > 0) { + Pair az = sortedAZPlacements.get(i % sortedAZPlacements.size()); + az.getSecond().replicationFactor++; + placedReplicas++; + if (az.getSecond().replicationFactor == az.getSecond().numNodesInAZ) { + sortedAZPlacements.remove(az); + } else { + i++; } - i++; } if (placedReplicas < rf) { diff --git a/managed/src/main/java/com/yugabyte/yw/common/config/GlobalConfKeys.java b/managed/src/main/java/com/yugabyte/yw/common/config/GlobalConfKeys.java index ded1bbf669be..3acebce99e15 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/config/GlobalConfKeys.java +++ b/managed/src/main/java/com/yugabyte/yw/common/config/GlobalConfKeys.java @@ -1414,4 +1414,13 @@ public class GlobalConfKeys extends RuntimeConfigKeysModule { + " throwing UNAVAILABLE", ConfDataType.IntegerType, ImmutableList.of(ConfKeyTags.BETA)); + // Also need rbac to be on for this key to have effect. + public static final ConfKeyInfo groupMappingRbac = + new ConfKeyInfo<>( + "yb.security.group_mapping_rbac_support", + ScopeType.GLOBAL, + "Enable RBAC for Groups", + "Map LDAP/OIDC groups to custom roles defined by RBAC.", + ConfDataType.BooleanType, + ImmutableList.of(ConfKeyTags.INTERNAL)); } diff --git a/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleBindingUtil.java b/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleBindingUtil.java index 0498ff622a93..0e00185c81bd 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleBindingUtil.java +++ b/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleBindingUtil.java @@ -100,7 +100,8 @@ public void validateRoles( if (Users.Role.SuperAdmin.name().equals(role.getName())) { String errMsg = String.format( - "Cannot assign SuperAdmin role to a user in the roleResourceDefinition: %s.", + "Cannot assign SuperAdmin role to a user or group in the roleResourceDefinition:" + + " %s.", roleResourceDefinition.toString()); log.error(errMsg); throw new PlatformServiceException(BAD_REQUEST, errMsg); diff --git a/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleResourceDefinition.java b/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleResourceDefinition.java index f470e957d3b9..50a451891bfd 100644 --- a/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleResourceDefinition.java +++ b/managed/src/main/java/com/yugabyte/yw/common/rbac/RoleResourceDefinition.java @@ -14,7 +14,7 @@ import lombok.Setter; import lombok.ToString; -@ApiModel(description = "Role and resource group definition.") +@ApiModel(description = "Defines the association of Role to Resource Groups.") @Getter @Setter @AllArgsConstructor diff --git a/managed/src/main/java/com/yugabyte/yw/forms/ResizeNodeParams.java b/managed/src/main/java/com/yugabyte/yw/forms/ResizeNodeParams.java index d6030edec6aa..829ef4aee4ef 100644 --- a/managed/src/main/java/com/yugabyte/yw/forms/ResizeNodeParams.java +++ b/managed/src/main/java/com/yugabyte/yw/forms/ResizeNodeParams.java @@ -268,8 +268,10 @@ private static String getResizeIsPossibleError( return error.get(); } if (nodeDiskChanged) { - if (curDeviceInfo.storageType == PublicCloudConstants.StorageType.UltraSSD_LRS) { - return "UltraSSD doesn't support resizing without downtime"; + if (curDeviceInfo.storageType == PublicCloudConstants.StorageType.UltraSSD_LRS + || curDeviceInfo.storageType == PublicCloudConstants.StorageType.PremiumV2_LRS) { + return String.format( + "%s doesn't support resizing without downtime", curDeviceInfo.storageType); } if (currentUserIntent.providerType == Common.CloudType.azu && curDeviceInfo.volumeSize <= AZU_DISK_LIMIT_NO_DOWNTIME diff --git a/managed/src/main/java/com/yugabyte/yw/models/Audit.java b/managed/src/main/java/com/yugabyte/yw/models/Audit.java index fd05abef2411..989965376de3 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/Audit.java +++ b/managed/src/main/java/com/yugabyte/yw/models/Audit.java @@ -181,7 +181,10 @@ public enum TargetType { RoleBinding, @EnumValue("OIDC Group Mapping") - OIDCGroupMapping + OIDCGroupMapping, + + @EnumValue("Group Mapping") + GroupMapping } public enum ActionType { diff --git a/managed/src/main/java/com/yugabyte/yw/models/GroupMappingInfo.java b/managed/src/main/java/com/yugabyte/yw/models/GroupMappingInfo.java new file mode 100644 index 000000000000..0d113b5f064e --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/GroupMappingInfo.java @@ -0,0 +1,108 @@ +// Copyright (c) Yugabyte, Inc. + +package com.yugabyte.yw.models; + +import static play.mvc.Http.Status.BAD_REQUEST; + +import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.models.rbac.Role; +import io.ebean.Finder; +import io.ebean.Model; +import io.ebean.annotation.EnumValue; +import io.ebean.annotation.Transactional; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import java.util.UUID; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.extern.slf4j.Slf4j; + +@Entity +@Table(name = "groups_mapping_info") +@Data +@EqualsAndHashCode(callSuper = false) +@Slf4j +public class GroupMappingInfo extends Model { + @Id + @Column(name = "uuid", nullable = false, unique = true) + private UUID groupUUID = UUID.randomUUID(); + + @Column(name = "customer_uuid", nullable = false) + private UUID customerUUID; + + // This will only be used when RBAC is off. + @Column(name = "role_uuid", nullable = false) + private UUID roleUUID; + + @Column(name = "identifier", nullable = false, unique = true) + private String identifier = null; + + @Column(name = "type", nullable = false) + private GroupType type; + + public enum GroupType { + @EnumValue("LDAP") + LDAP, + + @EnumValue("OIDC") + OIDC; + } + + public static GroupMappingInfo create(UUID customerUUID, String identifier, GroupType type) { + // Assign ConnectOnly if no role is assigned + UUID roleUUID = Role.get(customerUUID, "ConnectOnly").getRoleUUID(); + return create(customerUUID, roleUUID, identifier, type); + } + + @Transactional + public static GroupMappingInfo create( + UUID customerUUID, UUID roleUUID, String identifier, GroupType type) { + GroupMappingInfo entity = new GroupMappingInfo(); + entity.customerUUID = customerUUID; + entity.identifier = identifier.toLowerCase(); + entity.roleUUID = roleUUID; + entity.type = type; + entity.save(); + return entity; + } + + /** Wrapper around save to make sure principal entity is created. */ + @Transactional + @Override + public void save() { + super.save(); + Principal principal = Principal.get(this.groupUUID); + if (principal == null) { + log.info("Adding Principal entry for Group: " + this.identifier); + new Principal(this).save(); + } + } + + /** Wrapper around delete to make sure principal entity is deleted. */ + @Transactional + @Override + public boolean delete() { + log.info("Deleting Principal entry for Group: " + this.identifier); + Principal principal = Principal.getOrBadRequest(this.groupUUID); + principal.delete(); + return super.delete(); + } + + public static GroupMappingInfo get(UUID groupUUID) { + GroupMappingInfo info = find.query().where().eq("uuid", groupUUID).findOne(); + return info; + } + + public static GroupMappingInfo getOrBadRequest(UUID groupUuid) { + GroupMappingInfo info = get(groupUuid); + if (info == null) { + throw new PlatformServiceException(BAD_REQUEST, "Invalid Group UUID:" + groupUuid); + } + return info; + } + + public static final Finder find = + new Finder(GroupMappingInfo.class) {}; +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/Principal.java b/managed/src/main/java/com/yugabyte/yw/models/Principal.java new file mode 100644 index 000000000000..cab9fcbb3671 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/Principal.java @@ -0,0 +1,88 @@ +// Copyright (c) Yugabyte, Inc. + +package com.yugabyte.yw.models; + +import static play.mvc.Http.Status.BAD_REQUEST; + +import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.models.GroupMappingInfo.GroupType; +import io.ebean.Finder; +import io.ebean.Model; +import io.ebean.annotation.EnumValue; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import java.util.UUID; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@Entity +@Table(name = "principal") +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = false) +public class Principal extends Model { + // A Principal is an entity which is allowed to have role bindings. + // Currently it can either be a group or an user. + + @Id + @Column(name = "uuid", nullable = false, unique = true) + private UUID uuid; + + // At most one of these can be valid. + @Column(name = "user_uuid") + private UUID userUUID; + + @Column(name = "group_uuid") + private UUID groupUUID; + + @Column(name = "type", nullable = false) + private PrincipalType type; + + public enum PrincipalType { + @EnumValue("USER") + USER, + + @EnumValue("LDAP_GROUP") + LDAP_GROUP, + + @EnumValue("OIDC_GROUP") + OIDC_GROUP + } + + public Principal(Users user) { + uuid = user.getUuid(); + userUUID = uuid; + groupUUID = null; + type = PrincipalType.USER; + } + + public Principal(GroupMappingInfo info) { + uuid = info.getGroupUUID(); + userUUID = null; + groupUUID = uuid; + if (info.getType().equals(GroupType.LDAP)) { + type = PrincipalType.LDAP_GROUP; + } else { + type = PrincipalType.OIDC_GROUP; + } + } + + public static Principal get(UUID principalUUID) { + Principal principal = find.query().where().eq("uuid", principalUUID).findOne(); + return principal; + } + + public static Principal getOrBadRequest(UUID principalUuid) { + Principal principal = get(principalUuid); + if (principal == null) { + throw new PlatformServiceException(BAD_REQUEST, "Invalid Principal UUID:" + principalUuid); + } + return principal; + } + + public static final Finder find = + new Finder(Principal.class) {}; +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/TaskInfo.java b/managed/src/main/java/com/yugabyte/yw/models/TaskInfo.java index f1fa83c35c8b..55c58ab495cf 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/TaskInfo.java +++ b/managed/src/main/java/com/yugabyte/yw/models/TaskInfo.java @@ -398,8 +398,8 @@ public static List findDuplicateDeleteBackupTasks(UUID customerUUID, U .eq("task_type", TaskType.DeleteBackup) .ne("task_state", State.Failure) .ne("task_state", State.Aborted) - .eq("details->>'customerUUID'", customerUUID.toString()) - .eq("details->>'backupUUID'", backupUUID.toString()) + .eq("task_params->>'customerUUID'", customerUUID.toString()) + .eq("task_params->>'backupUUID'", backupUUID.toString()) .findList(); } @@ -409,8 +409,8 @@ public static List findIncompleteDeleteBackupTasks(UUID customerUUID, .where() .in("task_type", TaskType.DeleteBackup, TaskType.DeleteBackupYb) .in("task_state", INCOMPLETE_STATES) - .eq("details->>'customerUUID'", customerUUID.toString()) - .eq("details->>'backupUUID'", backupUUID.toString()) + .eq("task_params->>'customerUUID'", customerUUID.toString()) + .eq("task_params->>'backupUUID'", backupUUID.toString()) .findList(); } } diff --git a/managed/src/main/java/com/yugabyte/yw/models/Users.java b/managed/src/main/java/com/yugabyte/yw/models/Users.java index 57f7ac12fbaa..4bf17e05e963 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/Users.java +++ b/managed/src/main/java/com/yugabyte/yw/models/Users.java @@ -18,6 +18,7 @@ import io.ebean.Model; import io.ebean.annotation.Encrypted; import io.ebean.annotation.EnumValue; +import io.ebean.annotation.Transactional; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiModelProperty.AccessMode; @@ -319,6 +320,28 @@ public static void deleteUser(String email) { return; } + /** Wrapper around save to make sure principal entity is created. */ + @Transactional + @Override + public void save() { + super.save(); + Principal principal = Principal.get(this.uuid); + if (principal == null) { + log.info("Adding Principal entry for user with email: " + this.email); + new Principal(this).save(); + } + } + + /** Wrapper around delete to make sure principal entity is deleted. */ + @Transactional + @Override + public boolean delete() { + log.info("Deleting Principal entry for user with email: " + this.email); + Principal principal = Principal.getOrBadRequest(this.uuid); + principal.delete(); + return super.delete(); + } + /** * Validate if the email and password combination is valid, we use this to authenticate the Users. * diff --git a/managed/src/main/java/com/yugabyte/yw/models/configs/validators/GCPProviderValidator.java b/managed/src/main/java/com/yugabyte/yw/models/configs/validators/GCPProviderValidator.java index 033ebbe1fe60..dd4da17b8652 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/configs/validators/GCPProviderValidator.java +++ b/managed/src/main/java/com/yugabyte/yw/models/configs/validators/GCPProviderValidator.java @@ -1,5 +1,7 @@ package com.yugabyte.yw.models.configs.validators; +import static play.mvc.Http.Status.BAD_REQUEST; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.api.services.compute.model.Firewall; @@ -39,6 +41,7 @@ public class GCPProviderValidator extends ProviderFieldsValidator { private final GCPCloudImpl gcpCloudImpl; private final RuntimeConfGetter runtimeConfGetter; + private final String INSTANCE_TEMPLATE_REGEX = "[a-z]([-a-z0-9]*[a-z0-9])?"; @Inject public GCPProviderValidator( @@ -229,6 +232,12 @@ private void validateInstanceTempl( .get("jsonPath") .asText(); try { + if (!instanceTemplate.matches(INSTANCE_TEMPLATE_REGEX)) { + throw new PlatformServiceException( + BAD_REQUEST, + "Instance template must start with a lowercase character and can only contain" + + " alphanumeric characters and '-'"); + } if (!apiClient.checkInstanceTempelate(instanceTemplate)) { String errorMsg = String.format( diff --git a/managed/src/main/java/com/yugabyte/yw/models/migrations/V360.java b/managed/src/main/java/com/yugabyte/yw/models/migrations/V360.java new file mode 100644 index 000000000000..a641fbfc80d0 --- /dev/null +++ b/managed/src/main/java/com/yugabyte/yw/models/migrations/V360.java @@ -0,0 +1,84 @@ +// Copyright (c) Yugabyte, Inc. + +package com.yugabyte.yw.models.migrations; + +import io.ebean.Finder; +import io.ebean.Model; +import io.ebean.annotation.EnumValue; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import java.util.List; +import java.util.UUID; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** Snapshot View of ORM entities at the time migration V360 was added. */ +public class V360 { + + @Entity + @Getter + @Setter + public static class Users extends Model { + + public static final Finder find = new Finder(Users.class) {}; + + @Id private UUID uuid; + + public static List getAll() { + return find.query().where().findList(); + } + } + + @Entity + @Table(name = "principal") + @Data + @NoArgsConstructor + @EqualsAndHashCode(callSuper = false) + public static class Principal extends Model { + + @Id + @Column(name = "uuid", nullable = false, unique = true) + private UUID uuid; + + // At most one of these can be valid. + @Column(name = "user_uuid") + private UUID userUUID; + + @Column(name = "group_uuid") + private UUID groupUUID; + + @Column(name = "type", nullable = false) + private PrincipalType type; + + public enum PrincipalType { + @EnumValue("USER") + USER, + + @EnumValue("LDAP_GROUP") + LDAP_GROUP, + + @EnumValue("OIDC_GROUP") + OIDC_GROUP + } + + public Principal(Users user) { + uuid = user.getUuid(); + userUUID = uuid; + groupUUID = null; + type = PrincipalType.USER; + } + + public static Principal get(UUID principalUUID) { + Principal principal = find.query().where().eq("uuid", principalUUID).findOne(); + return principal; + } + + public static final Finder find = + new Finder(Principal.class) {}; + } +} diff --git a/managed/src/main/java/com/yugabyte/yw/models/rbac/RoleBinding.java b/managed/src/main/java/com/yugabyte/yw/models/rbac/RoleBinding.java index 9ec20cff4a8e..17aaab3dd0e0 100644 --- a/managed/src/main/java/com/yugabyte/yw/models/rbac/RoleBinding.java +++ b/managed/src/main/java/com/yugabyte/yw/models/rbac/RoleBinding.java @@ -8,6 +8,9 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.yugabyte.yw.common.PlatformServiceException; +import com.yugabyte.yw.models.GroupMappingInfo; +import com.yugabyte.yw.models.Principal; +import com.yugabyte.yw.models.Principal.PrincipalType; import com.yugabyte.yw.models.Users; import io.ebean.Finder; import io.ebean.Model; @@ -23,6 +26,7 @@ import jakarta.persistence.Id; import jakarta.persistence.JoinColumn; import jakarta.persistence.ManyToOne; +import jakarta.persistence.Transient; import java.util.Date; import java.util.List; import java.util.UUID; @@ -50,10 +54,17 @@ public class RoleBinding extends Model { private UUID uuid; @ManyToOne + @JoinColumn(name = "principal_uuid", referencedColumnName = "uuid") + private Principal principal; + @ApiModelProperty(value = "User") - @JoinColumn(name = "user_uuid", referencedColumnName = "uuid") + @Transient private Users user; + @ApiModelProperty(value = "GroupInfo") + @Transient + private GroupMappingInfo groupInfo; + /** * This shows whether the role binding is system generated or user generated. System generated * role bindings are usually for the LDAP group users. Custom role bindings are when the user sets @@ -99,19 +110,56 @@ public enum RoleBindingType { public static RoleBinding create( Users user, RoleBindingType roleBindingType, Role role, ResourceGroup resourceGroup) { + + Principal principal = Principal.find.byId(user.getUuid()); + RoleBinding roleBinding = + new RoleBinding( + UUID.randomUUID(), + principal, + user, + null, + roleBindingType, + role, + new Date(), + new Date(), + resourceGroup); + roleBinding.save(); + return roleBinding; + } + + public static RoleBinding create( + GroupMappingInfo info, + RoleBindingType roleBindingType, + Role role, + ResourceGroup resourceGroup) { + + Principal principal = Principal.find.byId(info.getGroupUUID()); RoleBinding roleBinding = new RoleBinding( - UUID.randomUUID(), user, roleBindingType, role, new Date(), new Date(), resourceGroup); + UUID.randomUUID(), + principal, + null, + info, + roleBindingType, + role, + new Date(), + new Date(), + resourceGroup); roleBinding.save(); return roleBinding; } public static RoleBinding get(UUID roleBindingUUID) { - return find.query().where().eq("uuid", roleBindingUUID).findOne(); + RoleBinding rb = find.query().where().eq("uuid", roleBindingUUID).findOne(); + populatePrincipalInfo(rb); + return rb; } public static List fetchRoleBindingsForUser(UUID userUUID) { - return find.query().where().eq("user_uuid", userUUID).findList(); + Users user = Users.getOrBadRequest(userUUID); + List list = find.query().where().eq("principal_uuid", userUUID).findList(); + list.forEach(rb -> rb.setUser(user)); + return list; } public static RoleBinding getOrBadRequest(UUID roleBindingUUID) { @@ -124,19 +172,32 @@ public static RoleBinding getOrBadRequest(UUID roleBindingUUID) { } public static List getAll() { - return find.query().findList(); + List list = find.query().findList(); + list.forEach(rb -> populatePrincipalInfo(rb)); + return list; } - public static List getAll(UUID userUUID) { - return find.query().where().eq("user_uuid", userUUID).findList(); + public static List getAll(UUID principalUUID) { + Principal principal = Principal.getOrBadRequest(principalUUID); + List list = find.query().where().eq("principal_uuid", principalUUID).findList(); + if (principal.getType().equals(PrincipalType.USER)) { + Users user = Users.getOrBadRequest(principalUUID); + list.forEach(rb -> rb.setUser(user)); + } else { + GroupMappingInfo info = GroupMappingInfo.getOrBadRequest(principalUUID); + list.forEach(rb -> rb.setGroupInfo(info)); + } + return list; } public static List getAllWithRole(UUID roleUUID) { - return find.query().where().eq("role_uuid", roleUUID).findList(); + List rbList = find.query().where().eq("role_uuid", roleUUID).findList(); + rbList.forEach(rb -> populatePrincipalInfo(rb)); + return rbList; } public static boolean checkUserHasRole(UUID userUUID, UUID roleUUID) { - return find.query().where().eq("user_uuid", userUUID).eq("role_uuid", roleUUID).exists(); + return find.query().where().eq("principal_uuid", userUUID).eq("role_uuid", roleUUID).exists(); } public void edit(Role role, ResourceGroup resourceGroup) { @@ -144,4 +205,21 @@ public void edit(Role role, ResourceGroup resourceGroup) { this.resourceGroup = resourceGroup; this.update(); } + + /** + * If the role binding belongs to a user, we populate the user field else the groupInfo field. + * + * @param rb + */ + private static void populatePrincipalInfo(RoleBinding rb) { + if (rb == null) { + return; + } + Principal principal = Principal.getOrBadRequest(rb.getPrincipal().getUuid()); + if (principal.getType().equals(PrincipalType.USER)) { + rb.setUser(Users.getOrBadRequest(principal.getUserUUID())); + } else { + rb.setGroupInfo(GroupMappingInfo.getOrBadRequest(principal.getGroupUUID())); + } + } } diff --git a/managed/src/main/java/db/migration/default_/postgres/V361__Populate_Princpals.java b/managed/src/main/java/db/migration/default_/postgres/V361__Populate_Princpals.java new file mode 100644 index 000000000000..3b352a6aeaed --- /dev/null +++ b/managed/src/main/java/db/migration/default_/postgres/V361__Populate_Princpals.java @@ -0,0 +1,25 @@ +// Copyright (c) Yugabyte, Inc. + +package db.migration.default_.postgres; + +import com.yugabyte.yw.models.migrations.V360.Principal; +import com.yugabyte.yw.models.migrations.V360.Users; +import java.sql.SQLException; +import lombok.extern.slf4j.Slf4j; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; + +@Slf4j +public class V361__Populate_Princpals extends BaseJavaMigration { + + @Override + public void migrate(Context context) throws SQLException { + for (Users user : Users.find.all()) { + Principal principal = Principal.get(user.getUuid()); + if (principal == null) { + log.info("Adding Principal entry for user with UUID: " + user.getUuid()); + new Principal(user).save(); + } + } + } +} diff --git a/managed/src/main/resources/application.test.conf b/managed/src/main/resources/application.test.conf index e77b383bbd58..314f0a2f8d6b 100644 --- a/managed/src/main/resources/application.test.conf +++ b/managed/src/main/resources/application.test.conf @@ -36,6 +36,7 @@ yb { is_platform_downgrade_allowed=${?YB_IS_PLATFORM_DOWNGRADE_ALLOWED} security.ssh2_enabled = false security.oidc_enable_auto_create_users = true + security.group_mapping_rbac_support = true universe.user_tags.is_enforced = false universe.user_tags.enforced_tags = ["yb_task:test","yb_owner:*","yb_dept:eng","yb_dept:qa"] diff --git a/managed/src/main/resources/db/migration/default_/common/V360__Create_Group_table.sql b/managed/src/main/resources/db/migration/default_/common/V360__Create_Group_table.sql new file mode 100644 index 000000000000..41389a7508fe --- /dev/null +++ b/managed/src/main/resources/db/migration/default_/common/V360__Create_Group_table.sql @@ -0,0 +1,33 @@ +-- Copyright (c) YugaByte, Inc. + +CREATE TABLE IF not EXISTS groups_mapping_info ( + uuid UUID NOT NULL, + identifier VARCHAR(255) NOT NULL, + role_uuid UUID NOT NULL, + customer_uuid UUID NOT NULL, + type VARCHAR(5) NOT NULL, + + CONSTRAINT fk_groups_info_customer_uuid foreign key (customer_uuid) references customer (uuid) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT fk_groups_info_role_uuid foreign key (role_uuid) references role (role_uuid), + CONSTRAINT ck_groups_info_type check(type in ('LDAP', 'OIDC')), + CONSTRAINT pk_group_mapping_info PRIMARY KEY (uuid), + CONSTRAINT uq_groups_info_identifier UNIQUE (identifier) +); + +CREATE TABLE IF NOT EXISTS principal ( + -- uuid will be same as user/group uuid + uuid UUID NOT NULL, + user_uuid UUID, + group_uuid UUID, + type VARCHAR(10) NOT NULL, + + CONSTRAINT pk_principal PRIMARY KEY (uuid), + CONSTRAINT fk_principal_user foreign key (user_uuid) references users (uuid) ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT fk_principal_group foreign key (group_uuid) references groups_mapping_info (uuid) ON UPDATE CASCADE ON DELETE CASCADE, + + CONSTRAINT ck_type check(type in ('USER', 'LDAP_GROUP', 'OIDC_GROUP')), + -- CHECK to make sure only one of them is non null + CHECK ( + (user_uuid IS NOT NULL)::INTEGER + + (group_uuid IS NOT NULL)::INTEGER = 1) +); \ No newline at end of file diff --git a/managed/src/main/resources/db/migration/default_/common/V362__Alter_role_binding.sql b/managed/src/main/resources/db/migration/default_/common/V362__Alter_role_binding.sql new file mode 100644 index 000000000000..f53512987899 --- /dev/null +++ b/managed/src/main/resources/db/migration/default_/common/V362__Alter_role_binding.sql @@ -0,0 +1,10 @@ +-- Copyright (c) YugaByte, Inc. + +ALTER TABLE role_binding +DROP CONSTRAINT fk_role_binding_user_uuid; + +ALTER TABLE role_binding +RENAME COLUMN user_uuid TO principal_uuid; + +ALTER TABLE role_binding +ADD CONSTRAINT fk_role_binding_user_uuid FOREIGN KEY (principal_uuid) REFERENCES principal (uuid); \ No newline at end of file diff --git a/managed/src/main/resources/openapi/components/requestBodies/GroupMappingReq.yaml b/managed/src/main/resources/openapi/components/requestBodies/GroupMappingReq.yaml new file mode 100644 index 000000000000..ca5457d1b6b7 --- /dev/null +++ b/managed/src/main/resources/openapi/components/requestBodies/GroupMappingReq.yaml @@ -0,0 +1,7 @@ +required: true +content: + application/json: + schema: + type: array + items: + $ref: "../schemas/GroupMappingSpec.yaml" diff --git a/managed/src/main/resources/openapi/components/responses/GroupMappingResp.yaml b/managed/src/main/resources/openapi/components/responses/GroupMappingResp.yaml new file mode 100644 index 000000000000..8e2df6b83a91 --- /dev/null +++ b/managed/src/main/resources/openapi/components/responses/GroupMappingResp.yaml @@ -0,0 +1,7 @@ +description: OK +content: + application/json: + schema: + type: array + items: + $ref: "../schemas/GroupMappingSpec.yaml" diff --git a/managed/src/main/resources/openapi/components/schemas/ClusterStorageSpec.yaml b/managed/src/main/resources/openapi/components/schemas/ClusterStorageSpec.yaml index 03e5fd1fa8ce..cfdb645a9fa5 100644 --- a/managed/src/main/resources/openapi/components/schemas/ClusterStorageSpec.yaml +++ b/managed/src/main/resources/openapi/components/schemas/ClusterStorageSpec.yaml @@ -20,7 +20,7 @@ properties: description: 'Name of the storage class, if this is a kubernetes cluster' type: string storage_type: - description: 'Storage type used for this instance, if this is a aws (IO1, GP2, GP3), gcp (Scratch, Persistent) or azu (StandardSSD_LRS, Premium_LRS, UltraSSD_LRS) cluster.' + description: 'Storage type used for this instance, if this is a aws (IO1, GP2, GP3), gcp (Scratch, Persistent) or azu (StandardSSD_LRS, Premium_LRS, PremiumV2_LRS, UltraSSD_LRS) cluster.' type: string enum: - IO1 @@ -29,6 +29,7 @@ properties: - Scratch - Persistent - StandardSSD_LRS + - PremiumV2_LRS - Premium_LRS - UltraSSD_LRS - Local diff --git a/managed/src/main/resources/openapi/components/schemas/GroupMappingSpec.yaml b/managed/src/main/resources/openapi/components/schemas/GroupMappingSpec.yaml new file mode 100644 index 000000000000..b0d882a1a737 --- /dev/null +++ b/managed/src/main/resources/openapi/components/schemas/GroupMappingSpec.yaml @@ -0,0 +1,29 @@ +title: GroupMappingSpec +description: | + GroupMappingSpec + + Group mapping properties. This is used to map LDAP and OIDC group to YBA roles. +type: object +required: + - group_identifier + - type + - role_resource_definitions +properties: + group_identifier: + description: Group name incase of OIDC. Group DN incase of LDAP. + type: string + uuid: + description: System generated UUID for this group mapping. + type: string + format: uuid + readOnly: true + type: + description: The type of group. Can be either LDAP/OIDC. + type: string + enum: + - LDAP + - OIDC + role_resource_definitions: + type: array + items: + $ref: "./RoleResourceDefinitionSpec.yaml" diff --git a/managed/src/main/resources/openapi/components/schemas/ResourceDefinitionSpec.yaml b/managed/src/main/resources/openapi/components/schemas/ResourceDefinitionSpec.yaml new file mode 100644 index 000000000000..e1f44ac64a49 --- /dev/null +++ b/managed/src/main/resources/openapi/components/schemas/ResourceDefinitionSpec.yaml @@ -0,0 +1,25 @@ +title: ResourceDefinitionSpec +description: Resource definition containing the resource type and resource set. +type: object +required: + - resource_type + - allow_all +properties: + resource_type: + description: Resource Type + type: string + enum: + - UNIVERSE + - ROLE + - USER + - OTHER + allow_all: + description: Select all resources (including future resources) + type: boolean + resource_uuid_set: + description: Set of resource UUIDs + type: array + items: + type: string + format: uuid + default: [] diff --git a/managed/src/main/resources/openapi/components/schemas/ResourceGroupSpec.yaml b/managed/src/main/resources/openapi/components/schemas/ResourceGroupSpec.yaml new file mode 100644 index 000000000000..998811ebbdda --- /dev/null +++ b/managed/src/main/resources/openapi/components/schemas/ResourceGroupSpec.yaml @@ -0,0 +1,10 @@ +title: ResourceGroupSpec +description: Resource group definition for the role. Only applicable for custom roles. +type: object +required: + - resource_definition_set +properties: + resource_definition_set: + type: array + items: + $ref: "./ResourceDefinitionSpec.yaml" diff --git a/managed/src/main/resources/openapi/components/schemas/RoleResourceDefinitionSpec.yaml b/managed/src/main/resources/openapi/components/schemas/RoleResourceDefinitionSpec.yaml new file mode 100644 index 000000000000..64fbbe05f1ba --- /dev/null +++ b/managed/src/main/resources/openapi/components/schemas/RoleResourceDefinitionSpec.yaml @@ -0,0 +1,12 @@ +title: RoleResourceDefinitionSpec. +description: Defines the association of Role to Resource Groups. Part of GroupMappingSpec. +type: object +required: + - role_uuid +properties: + role_uuid: + description: UUID of the role to attach resource group to. + type: string + format: uuid + resource_group: + $ref: "./ResourceGroupSpec.yaml" diff --git a/managed/src/main/resources/openapi/openapi_split.yaml b/managed/src/main/resources/openapi/openapi_split.yaml index d6b70d9db26c..1536c40ce308 100644 --- a/managed/src/main/resources/openapi/openapi_split.yaml +++ b/managed/src/main/resources/openapi/openapi_split.yaml @@ -22,6 +22,8 @@ servers: host_port: default: 'localhost:9000' tags: + - name: Authentication + description: Authentication operations on YBA - name: Universe description: CRUD operations for a Universe paths: diff --git a/managed/src/main/resources/openapi/paths/_index.yaml b/managed/src/main/resources/openapi/paths/_index.yaml index 9693e0056b78..0dcf4f7351c3 100644 --- a/managed/src/main/resources/openapi/paths/_index.yaml +++ b/managed/src/main/resources/openapi/paths/_index.yaml @@ -1,3 +1,120 @@ +'/customers/{cUUID}/auth/group-mappings': + parameters: + - name: cUUID + in: path + description: Customer UUID + schema: + type: string + format: uuid + explode: false + style: simple + required: true + get: + operationId: listMappings + summary: List Group Mappings + description: Get list of all OIDC and LDAP Group Mappings. + tags: + - Authentication + responses: + '200': + $ref: "../components/responses/GroupMappingResp.yaml" + '400': + description: Invalid input + '500': + description: Server error + security: + - apiKeyAuth: [] + x-yba-api-audit: + noAudit: true + x-yba-api-authz: + - requiredPermission: + resourceType: other + action: read + resourceLocation: + path: customers + sourceType: endpoint + x-yba-api-since: 2024.2.0.0 + x-yba-api-visibility: preview + put: + operationId: updateGroupMappings + summary: Create Group Mappings + description: Map LDAP and OIDC groups to YBA roles + tags: + - Authentication + requestBody: + $ref: "../components/requestBodies/GroupMappingReq.yaml" + responses: + '200': + description: Operation Successfull + '400': + description: Invalid input + '500': + description: Server error + security: + - apiKeyAuth: [] + x-yba-api-audit: + auditTargetType: GroupMapping + auditTargetId: cUUID.toString() + auditActionType: Set + x-yba-api-authz: + - requiredPermission: + resourceType: other + action: super_admin_actions + resourceLocation: + path: customers + sourceType: endpoint + x-yba-api-since: 2024.2.0.0 + x-yba-api-visibility: preview +'/customers/{cUUID}/auth/group-mappings/{gUUID}': + parameters: + - name: cUUID + in: path + description: Customer UUID + schema: + type: string + format: uuid + explode: false + style: simple + required: true + - name: gUUID + in: path + description: Group UUID + schema: + type: string + format: uuid + explode: false + style: simple + required: true + delete: + operationId: deleteGroupMappings + summary: Delete Group Mappings + description: Delete LDAP and OIDC group mapping + tags: + - Authentication + responses: + '200': + description: OK + '400': + description: Invalid input + '404': + description: Not found + '500': + description: Server error + security: + - apiKeyAuth: [] + x-yba-api-audit: + auditTargetType: GroupMapping + auditTargetId: gUUID.toString() + auditActionType: Delete + x-yba-api-authz: + - requiredPermission: + resourceType: other + action: super_admin_actions + resourceLocation: + path: customers + sourceType: endpoint + x-yba-api-since: 2024.2.0.0 + x-yba-api-visibility: preview '/customers/{cUUID}/universes/{uniUUID}': parameters: - name: cUUID diff --git a/managed/src/main/resources/openapi/paths/authentication.yaml b/managed/src/main/resources/openapi/paths/authentication.yaml new file mode 100644 index 000000000000..3130b89d6b63 --- /dev/null +++ b/managed/src/main/resources/openapi/paths/authentication.yaml @@ -0,0 +1,117 @@ +'/customers/{cUUID}/auth/group-mappings': + parameters: + - name: cUUID + in: path + description: Customer UUID + schema: + type: string + format: uuid + explode: false + style: simple + required: true + get: + operationId: listMappings + summary: List Group Mappings + description: Get list of all OIDC and LDAP Group Mappings. + tags: + - Authentication + responses: + '200': + $ref: "../components/responses/GroupMappingResp.yaml" + '400': + description: Invalid input + '500': + description: Server error + security: + - apiKeyAuth: [] + x-yba-api-audit: + noAudit: true + x-yba-api-authz: + - requiredPermission: + resourceType: other + action: read + resourceLocation: + path: customers + sourceType: endpoint + x-yba-api-since: 2024.2.0.0 + x-yba-api-visibility: preview + put: + operationId: updateGroupMappings + summary: Create Group Mappings + description: Map LDAP and OIDC groups to YBA roles + tags: + - Authentication + requestBody: + $ref: "../components/requestBodies/GroupMappingReq.yaml" + responses: + '200': + description: Operation Successfull + '400': + description: Invalid input + '500': + description: Server error + security: + - apiKeyAuth: [] + x-yba-api-audit: + auditTargetType: GroupMapping + auditTargetId: cUUID.toString() + auditActionType: Set + x-yba-api-authz: + - requiredPermission: + resourceType: other + action: super_admin_actions + resourceLocation: + path: customers + sourceType: endpoint + x-yba-api-since: 2024.2.0.0 + x-yba-api-visibility: preview +'/customers/{cUUID}/auth/group-mappings/{gUUID}': + parameters: + - name: cUUID + in: path + description: Customer UUID + schema: + type: string + format: uuid + explode: false + style: simple + required: true + - name: gUUID + in: path + description: Group UUID + schema: + type: string + format: uuid + explode: false + style: simple + required: true + delete: + operationId: deleteGroupMappings + summary: Delete Group Mappings + description: Delete LDAP and OIDC group mapping + tags: + - Authentication + responses: + '200': + description: OK + '400': + description: Invalid input + '404': + description: Not found + '500': + description: Server error + security: + - apiKeyAuth: [] + x-yba-api-audit: + auditTargetType: GroupMapping + auditTargetId: gUUID.toString() + auditActionType: Delete + x-yba-api-authz: + - requiredPermission: + resourceType: other + action: super_admin_actions + resourceLocation: + path: customers + sourceType: endpoint + x-yba-api-since: 2024.2.0.0 + x-yba-api-visibility: preview diff --git a/managed/src/main/resources/reference.conf b/managed/src/main/resources/reference.conf index 0925ffa6e66b..6fc48bf09ff0 100644 --- a/managed/src/main/resources/reference.conf +++ b/managed/src/main/resources/reference.conf @@ -890,6 +890,7 @@ yb { oidc_enable_auto_create_users = false oidc_default_role = "ReadOnly" ssh2_enabled = false + group_mapping_rbac_support = false ldap { use_ldap = "false" ldap_url = "" diff --git a/managed/src/main/resources/swagger-strict.json b/managed/src/main/resources/swagger-strict.json index 0ae8360d4a09..16253e0bd06f 100644 --- a/managed/src/main/resources/swagger-strict.json +++ b/managed/src/main/resources/swagger-strict.json @@ -1384,7 +1384,7 @@ }, "target" : { "description" : "Target", - "enum" : [ "Session", "CloudProvider", "Region", "AvailabilityZone", "CustomerConfig", "KMSConfig", "Customer", "Release", "Certificate", "CustomCACertificate", "Alert", "AlertTemplateSettings", "AlertTemplateVariables", "AlertChannel", "AlertChannelTemplates", "AlertDestination", "MaintenanceWindow", "AccessKey", "Universe", "XClusterConfig", "DrConfig", "Table", "Backup", "CustomerTask", "NodeInstance", "PlatformInstance", "Schedule", "User", "LoggingConfig", "RuntimeConfigKey", "HAConfig", "HABackup", "ScheduledScript", "SupportBundle", "TelemetryProvider", "TroubleshootingPlatform", "GFlags", "Hook", "HookScope", "NodeAgent", "CustomerLicense", "PerformanceRecommendation", "PerformanceAdvisorSettings", "PerformanceAdvisorRun", "Role", "RoleBinding", "OIDCGroupMapping" ], + "enum" : [ "Session", "CloudProvider", "Region", "AvailabilityZone", "CustomerConfig", "KMSConfig", "Customer", "Release", "Certificate", "CustomCACertificate", "Alert", "AlertTemplateSettings", "AlertTemplateVariables", "AlertChannel", "AlertChannelTemplates", "AlertDestination", "MaintenanceWindow", "AccessKey", "Universe", "XClusterConfig", "DrConfig", "Table", "Backup", "CustomerTask", "NodeInstance", "PlatformInstance", "Schedule", "User", "LoggingConfig", "RuntimeConfigKey", "HAConfig", "HABackup", "ScheduledScript", "SupportBundle", "TelemetryProvider", "TroubleshootingPlatform", "GFlags", "Hook", "HookScope", "NodeAgent", "CustomerLicense", "PerformanceRecommendation", "PerformanceAdvisorSettings", "PerformanceAdvisorRun", "Role", "RoleBinding", "OIDCGroupMapping", "GroupMapping" ], "example" : "User", "readOnly" : true, "type" : "string" @@ -4766,7 +4766,7 @@ }, "storageType" : { "description" : "Storage type used for this instance", - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "throughput" : { @@ -5814,6 +5814,31 @@ "required" : [ "clusters", "creatingUser", "kubernetesUpgradeSupported", "masterGFlags", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "tserverGFlags", "upgradeOption" ], "type" : "object" }, + "GroupMappingInfo" : { + "properties" : { + "customerUUID" : { + "format" : "uuid", + "type" : "string" + }, + "groupUUID" : { + "format" : "uuid", + "type" : "string" + }, + "identifier" : { + "type" : "string" + }, + "roleUUID" : { + "format" : "uuid", + "type" : "string" + }, + "type" : { + "enum" : [ "LDAP", "OIDC" ], + "type" : "string" + } + }, + "required" : [ "customerUUID", "groupUUID", "identifier", "roleUUID", "type" ], + "type" : "object" + }, "HTTP Auth information" : { "discriminator" : "type", "properties" : { @@ -9109,6 +9134,28 @@ }, "type" : "object" }, + "Principal" : { + "properties" : { + "groupUUID" : { + "format" : "uuid", + "type" : "string" + }, + "type" : { + "enum" : [ "USER", "LDAP_GROUP", "OIDC_GROUP" ], + "type" : "string" + }, + "userUUID" : { + "format" : "uuid", + "type" : "string" + }, + "uuid" : { + "format" : "uuid", + "type" : "string" + } + }, + "required" : [ "groupUUID", "type", "userUUID", "uuid" ], + "type" : "object" + }, "Provider" : { "properties" : { "active" : { @@ -10988,6 +11035,13 @@ "format" : "date-time", "type" : "string" }, + "groupInfo" : { + "$ref" : "#/definitions/GroupMappingInfo", + "description" : "GroupInfo" + }, + "principal" : { + "$ref" : "#/definitions/Principal" + }, "resourceGroup" : { "$ref" : "#/definitions/ResourceGroup", "description" : "Details of the resource group" @@ -11018,6 +11072,7 @@ "type" : "string" } }, + "required" : [ "principal" ], "type" : "object" }, "RoleBindingFormData" : { @@ -11058,7 +11113,7 @@ "type" : "object" }, "RoleResourceDefinition" : { - "description" : "Role and resource group definition.", + "description" : "Defines the association of Role to Resource Groups.", "properties" : { "resourceGroup" : { "$ref" : "#/definitions/ResourceGroup", @@ -28192,7 +28247,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -28204,7 +28259,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -28228,7 +28283,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -28240,7 +28295,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -28264,7 +28319,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -28276,7 +28331,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" diff --git a/managed/src/main/resources/swagger.json b/managed/src/main/resources/swagger.json index 27fe7261a8a5..96b94d4b3ab7 100644 --- a/managed/src/main/resources/swagger.json +++ b/managed/src/main/resources/swagger.json @@ -1396,7 +1396,7 @@ }, "target" : { "description" : "Target", - "enum" : [ "Session", "CloudProvider", "Region", "AvailabilityZone", "CustomerConfig", "KMSConfig", "Customer", "Release", "Certificate", "CustomCACertificate", "Alert", "AlertTemplateSettings", "AlertTemplateVariables", "AlertChannel", "AlertChannelTemplates", "AlertDestination", "MaintenanceWindow", "AccessKey", "Universe", "XClusterConfig", "DrConfig", "Table", "Backup", "CustomerTask", "NodeInstance", "PlatformInstance", "Schedule", "User", "LoggingConfig", "RuntimeConfigKey", "HAConfig", "HABackup", "ScheduledScript", "SupportBundle", "TelemetryProvider", "TroubleshootingPlatform", "GFlags", "Hook", "HookScope", "NodeAgent", "CustomerLicense", "PerformanceRecommendation", "PerformanceAdvisorSettings", "PerformanceAdvisorRun", "Role", "RoleBinding", "OIDCGroupMapping" ], + "enum" : [ "Session", "CloudProvider", "Region", "AvailabilityZone", "CustomerConfig", "KMSConfig", "Customer", "Release", "Certificate", "CustomCACertificate", "Alert", "AlertTemplateSettings", "AlertTemplateVariables", "AlertChannel", "AlertChannelTemplates", "AlertDestination", "MaintenanceWindow", "AccessKey", "Universe", "XClusterConfig", "DrConfig", "Table", "Backup", "CustomerTask", "NodeInstance", "PlatformInstance", "Schedule", "User", "LoggingConfig", "RuntimeConfigKey", "HAConfig", "HABackup", "ScheduledScript", "SupportBundle", "TelemetryProvider", "TroubleshootingPlatform", "GFlags", "Hook", "HookScope", "NodeAgent", "CustomerLicense", "PerformanceRecommendation", "PerformanceAdvisorSettings", "PerformanceAdvisorRun", "Role", "RoleBinding", "OIDCGroupMapping", "GroupMapping" ], "example" : "User", "readOnly" : true, "type" : "string" @@ -4801,7 +4801,7 @@ }, "storageType" : { "description" : "Storage type used for this instance", - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "throughput" : { @@ -5853,6 +5853,31 @@ "required" : [ "clusters", "creatingUser", "kubernetesUpgradeSupported", "masterGFlags", "platformUrl", "platformVersion", "sleepAfterMasterRestartMillis", "sleepAfterTServerRestartMillis", "tserverGFlags", "upgradeOption" ], "type" : "object" }, + "GroupMappingInfo" : { + "properties" : { + "customerUUID" : { + "format" : "uuid", + "type" : "string" + }, + "groupUUID" : { + "format" : "uuid", + "type" : "string" + }, + "identifier" : { + "type" : "string" + }, + "roleUUID" : { + "format" : "uuid", + "type" : "string" + }, + "type" : { + "enum" : [ "LDAP", "OIDC" ], + "type" : "string" + } + }, + "required" : [ "customerUUID", "groupUUID", "identifier", "roleUUID", "type" ], + "type" : "object" + }, "HTTP Auth information" : { "discriminator" : "type", "properties" : { @@ -9160,6 +9185,28 @@ }, "type" : "object" }, + "Principal" : { + "properties" : { + "groupUUID" : { + "format" : "uuid", + "type" : "string" + }, + "type" : { + "enum" : [ "USER", "LDAP_GROUP", "OIDC_GROUP" ], + "type" : "string" + }, + "userUUID" : { + "format" : "uuid", + "type" : "string" + }, + "uuid" : { + "format" : "uuid", + "type" : "string" + } + }, + "required" : [ "groupUUID", "type", "userUUID", "uuid" ], + "type" : "object" + }, "Provider" : { "properties" : { "active" : { @@ -11096,6 +11143,13 @@ "format" : "date-time", "type" : "string" }, + "groupInfo" : { + "$ref" : "#/definitions/GroupMappingInfo", + "description" : "GroupInfo" + }, + "principal" : { + "$ref" : "#/definitions/Principal" + }, "resourceGroup" : { "$ref" : "#/definitions/ResourceGroup", "description" : "Details of the resource group" @@ -11126,6 +11180,7 @@ "type" : "string" } }, + "required" : [ "principal" ], "type" : "object" }, "RoleBindingFormData" : { @@ -11166,7 +11221,7 @@ "type" : "object" }, "RoleResourceDefinition" : { - "description" : "Role and resource group definition.", + "description" : "Defines the association of Role to Resource Groups.", "properties" : { "resourceGroup" : { "$ref" : "#/definitions/ResourceGroup", @@ -29654,7 +29709,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -29666,7 +29721,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -29690,7 +29745,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -29702,7 +29757,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -29726,7 +29781,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" @@ -29738,7 +29793,7 @@ "description" : "successful operation", "schema" : { "items" : { - "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "UltraSSD_LRS", "Local" ], + "enum" : [ "IO1", "GP2", "GP3", "Scratch", "Persistent", "StandardSSD_LRS", "Premium_LRS", "PremiumV2_LRS", "UltraSSD_LRS", "Local" ], "type" : "string" }, "type" : "array" diff --git a/managed/src/test/java/com/yugabyte/yw/api/v2/AuthenticationGroupMappingApisTest.java b/managed/src/test/java/com/yugabyte/yw/api/v2/AuthenticationGroupMappingApisTest.java new file mode 100644 index 000000000000..c58579b69983 --- /dev/null +++ b/managed/src/test/java/com/yugabyte/yw/api/v2/AuthenticationGroupMappingApisTest.java @@ -0,0 +1,211 @@ +package com.yugabyte.yw.api.v2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static play.mvc.Http.Status.OK; + +import com.yugabyte.yba.v2.client.ApiClient; +import com.yugabyte.yba.v2.client.ApiException; +import com.yugabyte.yba.v2.client.ApiResponse; +import com.yugabyte.yba.v2.client.Configuration; +import com.yugabyte.yba.v2.client.api.AuthenticationApi; +import com.yugabyte.yba.v2.client.models.GroupMappingSpec; +import com.yugabyte.yba.v2.client.models.GroupMappingSpec.TypeEnum; +import com.yugabyte.yba.v2.client.models.ResourceDefinitionSpec; +import com.yugabyte.yba.v2.client.models.ResourceDefinitionSpec.ResourceTypeEnum; +import com.yugabyte.yba.v2.client.models.ResourceGroupSpec; +import com.yugabyte.yba.v2.client.models.RoleResourceDefinitionSpec; +import com.yugabyte.yw.common.FakeDBApplication; +import com.yugabyte.yw.common.ModelFactory; +import com.yugabyte.yw.common.rbac.Permission; +import com.yugabyte.yw.common.rbac.PermissionInfo.Action; +import com.yugabyte.yw.common.rbac.PermissionInfo.ResourceType; +import com.yugabyte.yw.models.Customer; +import com.yugabyte.yw.models.GroupMappingInfo; +import com.yugabyte.yw.models.GroupMappingInfo.GroupType; +import com.yugabyte.yw.models.RuntimeConfigEntry; +import com.yugabyte.yw.models.Users; +import com.yugabyte.yw.models.rbac.Role; +import com.yugabyte.yw.models.rbac.Role.RoleType; +import com.yugabyte.yw.models.rbac.RoleBinding; +import db.migration.default_.common.R__Sync_System_Roles; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.UUID; +import junitparams.JUnitParamsRunner; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(JUnitParamsRunner.class) +public class AuthenticationGroupMappingApisTest extends FakeDBApplication { + private Customer customer; + private String authToken; + + private Users user; + private ApiClient v2ApiClient; + + // Define test permissions to use later. + public Permission permission1 = new Permission(ResourceType.UNIVERSE, Action.CREATE); + public Permission permission2 = new Permission(ResourceType.UNIVERSE, Action.READ); + + @Before + public void setUp() { + customer = ModelFactory.testCustomer(); + user = ModelFactory.testSuperAdminUserNewRbac(customer); + authToken = user.createAuthToken(); + v2ApiClient = Configuration.getDefaultApiClient(); + String basePath = String.format("http://localhost:%d/api/v2", port); + v2ApiClient = v2ApiClient.setBasePath(basePath).addDefaultHeader("X-AUTH-TOKEN", authToken); + Configuration.setDefaultApiClient(v2ApiClient); + R__Sync_System_Roles.syncSystemRoles(); + } + + @Test + public void testListMappingRbacOff() throws Exception { + UUID cUUID = customer.getUuid(); + GroupMappingInfo info0 = GroupMappingInfo.create(cUUID, "test-group-1", GroupType.OIDC); + GroupMappingInfo info1 = GroupMappingInfo.create(cUUID, "test-group-2", GroupType.LDAP); + info1.setRoleUUID(Role.get(cUUID, "Admin").getRoleUUID()); + info1.save(); + AuthenticationApi api = new AuthenticationApi(); + ApiResponse> result = api.listMappingsWithHttpInfo(cUUID); + assertEquals(OK, result.getStatusCode()); + List res = result.getData(); + assertEquals(2, res.size()); + + assertEquals(1, res.get(0).getRoleResourceDefinitions().size()); + assertEquals(info0.getIdentifier(), res.get(0).getGroupIdentifier()); + assertEquals(info0.getRoleUUID(), res.get(0).getRoleResourceDefinitions().get(0).getRoleUuid()); + assertEquals( + Role.get(cUUID, "ConnectOnly").getRoleUUID(), + res.get(0).getRoleResourceDefinitions().get(0).getRoleUuid()); + + assertEquals(1, res.get(1).getRoleResourceDefinitions().size()); + assertEquals(info1.getIdentifier(), res.get(1).getGroupIdentifier()); + assertEquals(info1.getRoleUUID(), res.get(1).getRoleResourceDefinitions().get(0).getRoleUuid()); + assertEquals( + Role.get(cUUID, "Admin").getRoleUUID(), + res.get(1).getRoleResourceDefinitions().get(0).getRoleUuid()); + } + + @Test + public void testCRUDMappingsRbacOn() throws Exception { + RuntimeConfigEntry.upsertGlobal("yb.rbac.use_new_authz", "true"); + AuthenticationApi api = new AuthenticationApi(); + UUID cUUID = customer.getUuid(); + // create custom role + Role customRole = + Role.create( + cUUID, + "testCustomRole", + "testDescription", + RoleType.Custom, + new HashSet<>(Arrays.asList(permission1, permission2))); + + List mappingSpecList = new ArrayList<>(); + GroupMappingSpec ldapSpec = new GroupMappingSpec(); + GroupMappingSpec oidcSpec = new GroupMappingSpec(); + + ldapSpec.setGroupIdentifier("test-group-ldap"); + ldapSpec.setType(TypeEnum.LDAP); + RoleResourceDefinitionSpec rrdSystemRole = new RoleResourceDefinitionSpec(); + rrdSystemRole.setRoleUuid(Role.get(cUUID, "Admin").getRoleUUID()); + ldapSpec.setRoleResourceDefinitions(Arrays.asList(rrdSystemRole)); + + oidcSpec.setGroupIdentifier("test-group-oidc"); + oidcSpec.setType(TypeEnum.OIDC); + RoleResourceDefinitionSpec rrdCustomRole = new RoleResourceDefinitionSpec(); + rrdCustomRole.setRoleUuid(customRole.getRoleUUID()); + ResourceDefinitionSpec rd1 = + new ResourceDefinitionSpec().allowAll(true).resourceType(ResourceTypeEnum.UNIVERSE); + ResourceDefinitionSpec rd2 = + new ResourceDefinitionSpec() + .allowAll(false) + .resourceType(ResourceTypeEnum.OTHER) + .resourceUuidSet(Arrays.asList(cUUID)); + rrdCustomRole.setResourceGroup( + new ResourceGroupSpec().resourceDefinitionSet(Arrays.asList(rd1, rd2))); + oidcSpec.setRoleResourceDefinitions(Arrays.asList(rrdSystemRole, rrdCustomRole)); + + mappingSpecList.add(ldapSpec); + mappingSpecList.add(oidcSpec); + + ApiResponse resUpdate = api.updateGroupMappingsWithHttpInfo(cUUID, mappingSpecList); + assertEquals(OK, resUpdate.getStatusCode()); + + ApiResponse> resList = api.listMappingsWithHttpInfo(cUUID); + assertEquals(OK, resList.getStatusCode()); + + List specList = resList.getData(); + assertEquals(2, specList.size()); + + GroupMappingSpec ldapGroupInfo = specList.get(0); + assertEquals("test-group-ldap", ldapGroupInfo.getGroupIdentifier()); + assertEquals(TypeEnum.LDAP, ldapGroupInfo.getType()); + assertEquals(1, ldapGroupInfo.getRoleResourceDefinitions().size()); + assertEquals( + Role.get(cUUID, "Admin").getRoleUUID(), + ldapGroupInfo.getRoleResourceDefinitions().get(0).getRoleUuid()); + + GroupMappingSpec oidcGroupInfo = specList.get(1); + assertEquals("test-group-oidc", oidcGroupInfo.getGroupIdentifier()); + assertEquals(TypeEnum.OIDC, oidcGroupInfo.getType()); + assertEquals(2, oidcGroupInfo.getRoleResourceDefinitions().size()); + assertEquals( + Role.get(cUUID, "Admin").getRoleUUID(), + oidcGroupInfo.getRoleResourceDefinitions().get(0).getRoleUuid()); + + UUID oidcGroupUUID = oidcGroupInfo.getUuid(); + + oidcSpec.setGroupIdentifier("Test-Group-OIDC"); + assertTrue(mappingSpecList.remove(ldapSpec)); + resUpdate = api.updateGroupMappingsWithHttpInfo(cUUID, mappingSpecList); + assertEquals(OK, resUpdate.getStatusCode()); + + resList = api.listMappingsWithHttpInfo(cUUID); + assertEquals(OK, resList.getStatusCode()); + + specList = resList.getData(); + assertEquals(2, specList.size()); + // Make sure group names are case insensitive and uuid remains the same after update. + assertEquals(oidcGroupUUID, specList.get(1).getUuid()); + + ApiResponse delResponse = api.deleteGroupMappingsWithHttpInfo(cUUID, oidcGroupUUID); + assertEquals(OK, delResponse.getStatusCode()); + + GroupMappingInfo info = GroupMappingInfo.get(oidcGroupUUID); + assertEquals(null, info); + assertEquals( + 0, RoleBinding.find.query().where().eq("principal_uuid", oidcGroupUUID).findList().size()); + } + + @Test + public void testBadInput() throws Exception { + RuntimeConfigEntry.upsertGlobal("yb.rbac.use_new_authz", "false"); + AuthenticationApi api = new AuthenticationApi(); + UUID cUUID = customer.getUuid(); + List mappingSpecList = new ArrayList<>(); + GroupMappingSpec ldapSpec = new GroupMappingSpec(); + + ldapSpec.setGroupIdentifier("test-group-ldap"); + ldapSpec.setType(TypeEnum.LDAP); + RoleResourceDefinitionSpec rrdSystemRole = new RoleResourceDefinitionSpec(); + rrdSystemRole.setRoleUuid(Role.get(cUUID, "Admin").getRoleUUID()); + List rrdList = new ArrayList<>(); + rrdList.add(rrdSystemRole); + rrdList.add(rrdSystemRole); + ldapSpec.setRoleResourceDefinitions(rrdList); + + mappingSpecList.add(ldapSpec); + assertThrows( + ApiException.class, () -> api.updateGroupMappingsWithHttpInfo(cUUID, mappingSpecList)); + + ldapSpec.getRoleResourceDefinitions().remove(0); + ApiResponse resUpdate = api.updateGroupMappingsWithHttpInfo(cUUID, mappingSpecList); + assertEquals(OK, resUpdate.getStatusCode()); + } +} diff --git a/managed/src/test/java/com/yugabyte/yw/common/PlacementInfoUtilTest.java b/managed/src/test/java/com/yugabyte/yw/common/PlacementInfoUtilTest.java index 9dab707eace4..0f6086007be7 100644 --- a/managed/src/test/java/com/yugabyte/yw/common/PlacementInfoUtilTest.java +++ b/managed/src/test/java/com/yugabyte/yw/common/PlacementInfoUtilTest.java @@ -4612,6 +4612,43 @@ public void testDedicatedIncreaseRF() { assertEquals(2, addedMasters.get()); } + @Test + public void testConfigureIncreaseRF6nodes() { + Customer customer = ModelFactory.testCustomer("Test Customer"); + Provider provider = ModelFactory.newProvider(customer, CloudType.aws); + + Universe existing = createFromConfig(provider, "Existing", "r1-az1-6-3"); + Region region = getOrCreate(provider, "r1"); + AvailabilityZone az1 = AvailabilityZone.getByCode(provider, "az1"); + + AvailabilityZone az2 = AvailabilityZone.createOrThrow(region, "az2", "az2", "subnet-az2"); + + UniverseDefinitionTaskParams params = new UniverseDefinitionTaskParams(); + params.setUniverseUUID(existing.getUniverseUUID()); + params.currentClusterType = ClusterType.PRIMARY; + params.clusters = existing.getUniverseDetails().clusters; + params.getPrimaryCluster().userIntent.numNodes = 7; + params.getPrimaryCluster().userIntent.replicationFactor = 7; + params.nodeDetailsSet = new HashSet<>(existing.getUniverseDetails().nodeDetailsSet); + + PlacementInfoUtil.updateUniverseDefinition( + params, customer.getId(), params.getPrimaryCluster().uuid, EDIT); + + List addedNodes = + params.nodeDetailsSet.stream() + .filter(n -> n.state == NodeState.ToBeAdded) + .collect(Collectors.toList()); + + assertEquals(1, addedNodes.size()); + + assertEquals(7, params.nodeDetailsSet.size()); + assertEquals(az2.getUuid(), addedNodes.get(0).azUuid); + assertEquals( + 1, params.getPrimaryCluster().placementInfo.findByAZUUID(az2.getUuid()).replicationFactor); + assertEquals( + 6, params.getPrimaryCluster().placementInfo.findByAZUUID(az1.getUuid()).replicationFactor); + } + private void markNodeInstancesAsOccupied(Map azUuidToNumNodes) { Map counts = new HashMap<>(azUuidToNumNodes); for (NodeInstance nodeInstance : NodeInstance.getAll()) { diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/UniverseControllerTestBase.java b/managed/src/test/java/com/yugabyte/yw/controllers/UniverseControllerTestBase.java index 15981f767cb0..4d5f009aa2d4 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/UniverseControllerTestBase.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/UniverseControllerTestBase.java @@ -308,7 +308,7 @@ protected ObjectNode createValidDeviceInfo(CloudType cloudType) { case gcp: return createDeviceInfo(StorageType.Persistent, 1, 100, null, null, null); case azu: - return createDeviceInfo(StorageType.Premium_LRS, 1, 100, null, null, null); + return createDeviceInfo(StorageType.PremiumV2_LRS, 1, 100, null, null, null); case kubernetes: return createDeviceInfo(null, 1, 100, null, null, null); default: diff --git a/managed/src/test/java/com/yugabyte/yw/controllers/UniverseCreateControllerTestBase.java b/managed/src/test/java/com/yugabyte/yw/controllers/UniverseCreateControllerTestBase.java index f3c853758152..85513eaca07e 100644 --- a/managed/src/test/java/com/yugabyte/yw/controllers/UniverseCreateControllerTestBase.java +++ b/managed/src/test/java/com/yugabyte/yw/controllers/UniverseCreateControllerTestBase.java @@ -966,16 +966,17 @@ private Object[] parametersToDeviceInfoValidation() { null, null }, + {Common.CloudType.azu, "c3.xlarge", StorageType.Premium_LRS, 1, 100, null, null, null, null}, { Common.CloudType.azu, "c3.xlarge", - PublicCloudConstants.StorageType.Premium_LRS, + PublicCloudConstants.StorageType.PremiumV2_LRS, 1, 100, null, null, null, - null + "Disk IOPS is mandatory for PremiumV2_LRS storage" }, { Common.CloudType.azu, @@ -1205,7 +1206,7 @@ private Object[] parametersToDeviceInfoValidation() { { Common.CloudType.azu, "c3.xlarge", - PublicCloudConstants.StorageType.Premium_LRS, + PublicCloudConstants.StorageType.PremiumV2_LRS, 1, null, null, diff --git a/managed/ui/src/actions/tasks.js b/managed/ui/src/actions/tasks.js index f976b473c5c1..117d786e37af 100644 --- a/managed/ui/src/actions/tasks.js +++ b/managed/ui/src/actions/tasks.js @@ -19,6 +19,8 @@ export const RETRY_TASK_RESPONSE = 'RETRY_TASK_RESPONSE'; export const ABORT_TASK = 'ABORT_TASK'; export const ABORT_TASK_RESPONSE = 'ABORT_TASK_RESPONSE'; +export const PATCH_TASKS_FOR_CUSTOMER = 'PATCH_TASKS_FOR_CUSTOMER'; + export function fetchTaskProgress(taskUUID) { const request = axios.get(`${getCustomerEndpoint()}/tasks/${taskUUID}`); return { @@ -48,6 +50,19 @@ export function fetchCustomerTasks() { }; } +/** + * used to patch a particular universe's tasks to the list of tasks + */ +export function patchTasksForCustomer(universeUUID, tasks) { + return { + type: PATCH_TASKS_FOR_CUSTOMER, + payload: { + universeUUID, + tasks + } + }; +} + export function fetchCustomerTasksSuccess(result) { return { type: FETCH_CUSTOMER_TASKS_SUCCESS, diff --git a/managed/ui/src/components/backupv2/Universe/UniverseLevelBackup.tsx b/managed/ui/src/components/backupv2/Universe/UniverseLevelBackup.tsx index 611fb9bee633..4901d555a638 100644 --- a/managed/ui/src/components/backupv2/Universe/UniverseLevelBackup.tsx +++ b/managed/ui/src/components/backupv2/Universe/UniverseLevelBackup.tsx @@ -9,12 +9,13 @@ import { FC, useState } from 'react'; import { DropdownButton, MenuItem, Tab } from 'react-bootstrap'; +import { useQuery } from 'react-query'; import { withRouter } from 'react-router'; +import { useTranslation } from 'react-i18next'; import { useSelector } from 'react-redux'; import { BackupList, Restore } from '..'; import { YBTabsPanel } from '../../panels'; import { ScheduledBackup } from '../scheduled/ScheduledBackup'; -import { BackupAndRestoreBanner } from '../restore/BackupAndRestoreBanner'; import { PointInTimeRecovery } from '../pitr/PointInTimeRecovery'; import { isYbcInstalledInUniverse, getPrimaryCluster } from '../../../utils/UniverseUtils'; import { BackupThrottleParameters } from '../components/BackupThrottleParameters'; @@ -24,13 +25,10 @@ import { RbacValidator } from '../../../redesign/features/rbac/common/RbacApiPer import { Universe } from '../../../redesign/helpers/dtos'; import { compareYBSoftwareVersions } from '../../../utils/universeUtilsTyped'; -import './UniverseLevelBackup.scss'; import { ApiPermissionMap } from '../../../redesign/features/rbac/ApiAndUserPermMapping'; -import { useQuery } from 'react-query'; import { api } from '../../../redesign/helpers/api'; import { YBErrorIndicator, YBLoading } from '../../common/indicators'; -import { useTranslation } from 'react-i18next'; - +import './UniverseLevelBackup.scss'; interface UniverseBackupProps { params: { uuid: string; @@ -89,7 +87,6 @@ const UniverseBackup: FC = ({ params: { uuid } }) => { return ( <> - {featureFlags.test.enableNewAdvancedRestoreModal ? ( showAdvancedRestore && ( = ({ return { ...b, backupUUID: b.commonBackupInfo.backupUUID }; }); + const isBackupNotSucceeded = (state: Backup_States) => [ + Backup_States.IN_PROGRESS, + Backup_States.STOPPING, + Backup_States.FAILED, + Backup_States.FAILED_TO_DELETE, + Backup_States.SKIPPED, + Backup_States.STOPPED + ].includes(state); + if (!isFilterApplied() && backups?.length === 0) { return allowTakingBackup ? ( <> @@ -657,16 +668,24 @@ export const BackupList: FC = ({ row.fullChainSizeInBytes || row.commonBackupInfo.totalBackupSizeInBytes ); }} - width="20%" + width="10%" > Size { - return ; + dataFormat={(lastBackupState, row: IBackup) => { + + return
e.stopPropagation()} className='backup-status'> + + { + isBackupNotSucceeded(row.commonBackupInfo.state) && ( + + ) + } +
; }} - width="15%" + width="25%" > Last Status diff --git a/managed/ui/src/components/backupv2/restore/BackupAndRestoreBanner.tsx b/managed/ui/src/components/backupv2/restore/BackupAndRestoreBanner.tsx deleted file mode 100644 index 299a8a72f17e..000000000000 --- a/managed/ui/src/components/backupv2/restore/BackupAndRestoreBanner.tsx +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Created on Tue Aug 16 2022 - * - * Copyright 2021 YugaByte, Inc. and Contributors - * Licensed under the Polyform Free Trial License 1.0.0 (the "License") - * You may not use this file except in compliance with the License. You may obtain a copy of the License at - * http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt - */ -import { useSelector } from 'react-redux'; -import { BACKUP_IN_PROGRESS_MSG, RESTORE_IN_PROGRESS_MSG } from '../common/BackupUtils'; - -export const BackupAndRestoreBanner = () => { - const currentUniverse = useSelector((state: any) => state.universe.currentUniverse); - - if (!currentUniverse) { - return null; - } - - const { - universeDetails: { updateInProgress, updatingTask } - } = currentUniverse.data; - - if (updateInProgress) { - if (updatingTask === 'CreateBackup') { - return BACKUP_IN_PROGRESS_MSG; - } - if (updatingTask === 'RestoreBackup') { - return RESTORE_IN_PROGRESS_MSG; - } - } - - return null; -}; diff --git a/managed/ui/src/components/tasks/TaskList/TaskListTable.js b/managed/ui/src/components/tasks/TaskList/TaskListTable.js index aaa9da0dce05..1369ba30ad14 100644 --- a/managed/ui/src/components/tasks/TaskList/TaskListTable.js +++ b/managed/ui/src/components/tasks/TaskList/TaskListTable.js @@ -4,16 +4,16 @@ import { Component } from 'react'; import PropTypes from 'prop-types'; import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table'; import { Link } from 'react-router'; -import { toast } from 'react-toastify'; import { YBPanelItem } from '../../panels'; import { timeFormatter, successStringFormatter } from '../../../utils/TableFormatters'; -import { YBConfirmModal } from '../../modals'; import { hasNecessaryPerm, RbacValidator } from '../../../redesign/features/rbac/common/RbacApiPermValidator'; import { ApiPermissionMap } from '../../../redesign/features/rbac/ApiAndUserPermMapping'; +import { TaskDetailDrawer } from '../../../redesign/features/tasks'; import { SoftwareUpgradeTaskType } from '../../universes/helpers/universeHelpers'; +import { YBConfirmModal } from '../../modals'; import './TasksList.scss'; export default class TaskListTable extends Component { @@ -25,8 +25,13 @@ export default class TaskListTable extends Component { overrideContent: PropTypes.object }; + state = { + selectedTaskUUID: undefined + } + render() { - const { taskList, title, visibleModal, hideTaskAbortModal, showTaskAbortModal } = this.props; + const { taskList, title, visibleModal, hideTaskAbortModal, showTaskAbortModal, featureFlags } = this.props; + const isNewTaskDetailsUIEnabled = featureFlags?.test?.newTaskDetailsUI || featureFlags?.released?.newTaskDetailsUI; function nameFormatter(cell, row) { return {row.title.replace(/.*:\s*/, '')}; @@ -105,9 +110,18 @@ export default class TaskListTable extends Component { return ; } }; + + const tableBodyContainer = { marginBottom: '1%', paddingBottom: '1%' }; return ( - + + { + this.setState({ + selectedTaskUUID: undefined + }); + }} /> {title}} body={ @@ -118,6 +132,9 @@ export default class TaskListTable extends Component { search multiColumnSearch searchPlaceholder="Search by Name or Type" + options={{ + onRowClick: (task) => isNewTaskDetailsUIEnabled && this.setState({ selectedTaskUUID: task.id }) + }} >