From ba567a2694c1dbe0bd86e3996047716ce3e176f8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 8 Apr 2022 11:07:52 +0200 Subject: [PATCH 01/16] added OpaConfig, removed authorization module --- deploy/crd/trinocluster.crd.yaml | 48 +-- docs/modules/ROOT/pages/usage.adoc | 23 +- ...ster-authentication-opa-authorization.yaml | 140 +++++++-- rust/crd/src/authorization.rs | 273 ------------------ rust/crd/src/lib.rs | 10 +- rust/operator-binary/src/controller.rs | 62 ++-- 6 files changed, 165 insertions(+), 391 deletions(-) delete mode 100644 rust/crd/src/authorization.rs diff --git a/deploy/crd/trinocluster.crd.yaml b/deploy/crd/trinocluster.crd.yaml index 7390710b..c6d14bc0 100644 --- a/deploy/crd/trinocluster.crd.yaml +++ b/deploy/crd/trinocluster.crd.yaml @@ -50,42 +50,6 @@ spec: required: - method type: object - authorization: - nullable: true - properties: - package: - type: string - permissions: - additionalProperties: - properties: - schemas: - nullable: true - properties: - read: - nullable: true - type: boolean - write: - nullable: true - type: boolean - type: object - tables: - additionalProperties: - properties: - read: - nullable: true - type: boolean - write: - nullable: true - type: boolean - type: object - nullable: true - type: object - type: object - type: object - required: - - package - - permissions - type: object coordinators: nullable: true properties: @@ -202,9 +166,17 @@ spec: hiveConfigMapName: nullable: true type: string - opaConfigMapName: + opa: nullable: true - type: string + properties: + configMapName: + type: string + package: + nullable: true + type: string + required: + - configMapName + type: object s3: description: Contains all the required connection information for S3. nullable: true diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 088633ed..6a021aaa 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -139,6 +139,7 @@ kubectl apply -f /etc/stackable/trino-operator/crd/trinocluster.crd.yaml ==== Insecure for testing: Create an insecure single node Trino (v362) cluster for testing. You will access the UI/CLI via http and no user / password or authorization is required. Please adapt the `s3` settings with your credentials (check `examples/simple-trino-cluster.yaml` for an example setting up Hive and Trino): + [source,yaml] ---- apiVersion: trino.stackable.tech/v1alpha1 @@ -191,31 +192,15 @@ metadata: spec: version: "0.0.362" hiveConfigMapName: simple-hive-derby - opaConfigMapName: simple-opa + opa: + configMapName: simple-opa + package: trino authentication: method: multiUser: userCredentialsSecret: namespace: default name: simple-trino-users-secret - authorization: - package: trino - permissions: - admin: - schemas: - read: true - write: true - tables: - iris_parquet: - read: true - write: true - bob: - schemas: - read: true - write: false - tables: - iris_parquet: - read: false s3: endPoint: changeme accessKey: changeme diff --git a/examples/simple-trino-cluster-authentication-opa-authorization.yaml b/examples/simple-trino-cluster-authentication-opa-authorization.yaml index daa9de63..ddb62d5a 100644 --- a/examples/simple-trino-cluster-authentication-opa-authorization.yaml +++ b/examples/simple-trino-cluster-authentication-opa-authorization.yaml @@ -51,6 +51,124 @@ stringData: # bob:bob bob: $2y$10$xVRXtYZnYuQu66SmruijPO8WHFM/UK5QPHTr.Nzf4JMcZSqt3W.2. --- +apiVersion: v1 +kind: ConfigMap +metadata: + name: simple-trino-opa-rego + labels: + opa.stackable.tech/bundle: "trino" +data: + trino.rego: | + package trino + + users = { + "admin": { + "schemas": { + "read": true, + "write": true + }, + "tables":{ + "iris_parquet": { + "read": true, + "write": true + } + } + }, + "bob": { + "schemas": { + "read": true, + "write": false + }, + "tables": { + "iris_parquet": { + "read": false, + "write": null + } + } + } + } + + default can_access_table = false + can_access_table { + user_can_read_table + } + + default can_create_table = false + can_create_table { + user_can_write_table + } + + default can_drop_table = false + can_drop_table { + user_can_write_table + } + + default can_show_tables = false + can_show_tables { + user_can_read_table + } + + default can_access_schema = false + can_access_schema { + user_can_read_schema + } + + default can_create_schema = false + can_create_schema { + user_can_write_schema + } + + default can_drop_schema = false + can_drop_schema { + user_can_write_schema + } + + default can_show_schemas = false + can_show_schemas { + user_can_read_schema + } + + default can_access_catalog = false + can_access_catalog { + is_valid_user + } + + default can_execute_query = false + can_execute_query { + is_valid_user + } + + default can_select_from_columns = false + can_select_from_columns { + is_valid_user + can_access_table + } + + default can_view_query_owned_by = false + can_view_query_owned_by { + is_valid_user + } + + user_can_read_table { + users[input.user.name].tables[input.request.table.table].read == true + } + + user_can_write_table { + users[input.user.name].tables[input.request.table.table].write == true + } + + user_can_read_schema { + users[input.user.name].schemas.read == true + } + + user_can_write_schema { + users[input.user.name].schemas.write == true + } + + is_valid_user { + _ = users[input.user.name] + } +--- apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCluster metadata: @@ -58,31 +176,15 @@ metadata: spec: version: "0.0.362" hiveConfigMapName: simple-hive-derby - opaConfigMapName: simple-opa + opa: + configMapName: simple-opa + package: trino authentication: method: multiUser: userCredentialsSecret: namespace: default name: simple-trino-users-secret - authorization: - package: trino - permissions: - admin: - schemas: - read: true - write: true - tables: - iris_parquet: - read: true - write: true - bob: - schemas: - read: true - write: false - tables: - iris_parquet: - read: false s3: endPoint: changeme accessKey: changeme diff --git a/rust/crd/src/authorization.rs b/rust/crd/src/authorization.rs deleted file mode 100644 index be9f23c8..00000000 --- a/rust/crd/src/authorization.rs +++ /dev/null @@ -1,273 +0,0 @@ -use crate::{TrinoCluster, TrinoClusterSpec}; - -use serde::{Deserialize, Serialize}; -use snafu::{ResultExt, Snafu}; -use stackable_operator::builder::{ConfigMapBuilder, ObjectMetaBuilder}; -use stackable_operator::client::Client; -use stackable_operator::k8s_openapi::api::core::v1::ConfigMap; -use stackable_operator::kube::ResourceExt; -use stackable_operator::schemars::{self, JsonSchema}; -use std::collections::BTreeMap; - -const FIELD_MANAGER_SCOPE: &str = "trinocluster"; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("failed to build rego rule config map"))] - FailedRegoRuleConfigMapBuild { - source: stackable_operator::error::Error, - }, - #[snafu(display("failed to apply rego rule config map"))] - FailedRegoRuleConfigMapApply { - source: stackable_operator::error::Error, - }, - #[snafu(display("object is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::error::Error, - }, - #[snafu(display("failed to convert permission set to JSON: {permissions:?}"))] - JsonConversion { - source: serde_json::Error, - permissions: BTreeMap, - }, -} - -type Result = std::result::Result; - -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct Authorization { - pub package: String, - pub permissions: BTreeMap, -} - -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct UserPermission { - pub schemas: Option, - pub tables: Option>, -} - -#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AccessPermission { - pub read: Option, - pub write: Option, -} - -pub async fn create_rego_rules(client: &Client, trino: &TrinoCluster) -> Result<()> { - let spec: &TrinoClusterSpec = &trino.spec; - - if let Some(authorization) = &spec.authorization { - let rego_rules = build_rego_rules(authorization)?; - create_or_update_rego_config_map(client, trino, &authorization.package, rego_rules).await?; - } - - Ok(()) -} - -async fn create_or_update_rego_config_map( - client: &Client, - trino: &TrinoCluster, - package_name: &str, - rego_rules: String, -) -> Result { - let config_map_data = [(format!("{}.rego", package_name), rego_rules)] - .into_iter() - .collect::>(); - - let config_map = ConfigMapBuilder::new() - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(trino) - .name(format!("{}-opa-rego-{}", trino.name(), package_name)) - .labels( - [( - "opa.stackable.tech/bundle".to_string(), - package_name.to_string(), - )] - .into_iter() - .collect::>(), - ) - .ownerreference_from_resource(trino, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .build(), - ) - .data(config_map_data) - .build() - .context(FailedRegoRuleConfigMapBuildSnafu)?; - - client - .apply_patch(FIELD_MANAGER_SCOPE, &config_map, &config_map) - .await - .context(FailedRegoRuleConfigMapApplySnafu) -} - -fn build_rego_rules(authorization_rules: &Authorization) -> Result { - let mut rules = String::new(); - - rules.push_str(&format!(" package {}\n\n", authorization_rules.package)); - rules.push_str(&build_user_permission_json( - &authorization_rules.permissions, - )?); - rules.push_str(&build_main_rego_rules()); - rules.push_str(&build_helper_rego_rules()); - - Ok(rules) -} - -fn build_user_permission_json( - user_permissions: &BTreeMap, -) -> Result { - let mut user_json = String::new(); - - let json = &serde_json::to_string(&user_permissions).with_context(|_| JsonConversionSnafu { - permissions: user_permissions.clone(), - })?; - - user_json.push_str(" users = "); - user_json.push_str(json); - user_json.push('\n'); - - Ok(user_json) -} - -fn build_main_rego_rules() -> String { - let main_rules = " - default can_access_table = false - can_access_table { - user_can_read_table - } - - default can_create_table = false - can_create_table { - user_can_write_table - } - - default can_drop_table = false - can_drop_table { - user_can_write_table - } - - default can_show_tables = false - can_show_tables { - user_can_read_table - } - - default can_access_schema = false - can_access_schema { - user_can_read_schema - } - - default can_create_schema = false - can_create_schema { - user_can_write_schema - } - - default can_drop_schema = false - can_drop_schema { - user_can_write_schema - } - - default can_show_schemas = false - can_show_schemas { - user_can_read_schema - } - - default can_access_catalog = false - can_access_catalog { - is_valid_user - } - - default can_execute_query = false - can_execute_query { - is_valid_user - } - - default can_select_from_columns = false - can_select_from_columns { - is_valid_user - can_access_table - } - - default can_view_query_owned_by = false - can_view_query_owned_by { - is_valid_user - } -"; - - main_rules.to_string() -} - -fn build_helper_rego_rules() -> String { - let sub_rules = " - user_can_read_table { - users[input.user.name].tables[input.request.table.table].read == true - } - - user_can_write_table { - users[input.user.name].tables[input.request.table.table].write == true - } - - user_can_read_schema { - users[input.user.name].schemas.read == true - } - - user_can_write_schema { - users[input.user.name].schemas.write == true - } - - is_valid_user { - _ = users[input.user.name] - } -"; - - sub_rules.to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - use indoc::indoc; - use rstest::rstest; - - #[rstest] - #[case::test( - indoc! {" - package: trino - permissions: - admin: - schemas: - read: true - write: true - tables: - test_table_1: - read: true - write: true - test_table_2: - read: true - bob: - schemas: - read: false - write: false - tables: - test_table_1: - read: true - "}, - )] - fn test_build_rego_rules(#[case] auth: &str) -> Result<()> { - let authorization = parse_authorization_from_yaml(auth); - let rego_rules = build_rego_rules(&authorization)?; - - assert!(rego_rules.contains("package trino")); - assert!(rego_rules.contains("user_can_read_table")); - assert!(rego_rules.contains("can_drop_schema")); - - Ok(()) - } - - fn parse_authorization_from_yaml(authorization: &str) -> Authorization { - let auth: Authorization = serde_yaml::from_str(authorization).unwrap(); - auth - } -} diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index a37ed737..450bb70d 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,13 +1,11 @@ pub mod authentication; -pub mod authorization; pub mod discovery; -use crate::authentication::Authentication; -use crate::authorization::Authorization; -use crate::discovery::TrinoPodRef; +use crate::{authentication::Authentication, discovery::TrinoPodRef}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, Snafu}; +use stackable_operator::opa::OpaConfig; use stackable_operator::{ kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt}, product_config_utils::{ConfigError, Configuration}, @@ -109,13 +107,11 @@ pub struct TrinoClusterSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub hive_config_map_name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub opa_config_map_name: Option, + pub opa: Option, /// A reference to a secret containing username/password for defined users #[serde(default, skip_serializing_if = "Option::is_none")] pub authentication: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub authorization: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] pub s3: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub coordinators: Option>, diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 83f0b4d7..82c4f3fa 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -24,6 +24,7 @@ use stackable_operator::{ }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, + opa::OpaApiVersion, product_config, product_config::{types::PropertyNameKind, ProductConfigManager}, product_config_utils::{ @@ -35,8 +36,6 @@ use stackable_operator::{ use stackable_trino_crd::{ authentication, authentication::TrinoAuthenticationConfig, - authorization, - authorization::create_rego_rules, discovery::{TrinoDiscovery, TrinoDiscoveryProtocol, TrinoPodRef}, TrinoCluster, TrinoClusterSpec, TrinoRole, ACCESS_CONTROL_PROPERTIES, APP_NAME, CONFIG_DIR_NAME, CONFIG_PROPERTIES, DATA_DIR_NAME, DISCOVERY_URI, FIELD_MANAGER_SCOPE, @@ -52,7 +51,6 @@ use std::{ time::Duration, }; use strum::{EnumDiscriminants, IntoStaticStr}; -use tracing::warn; pub struct Ctx { pub client: stackable_operator::client::Client, @@ -113,14 +111,16 @@ pub enum Error { }, #[snafu(display("failed to load Product Config"))] ProductConfigLoadFailed, - #[snafu(display("failed to write rego rules for authorization"))] - WriteRegoRuleAuthorizationFailed { source: authorization::Error }, #[snafu(display("failed to processing authentication config element from k8s"))] FailedProcessingAuthentication { source: authentication::Error }, #[snafu(display("internal operator failure"))] InternalOperatorFailure { source: stackable_trino_crd::Error }, #[snafu(display("no coordinator pods found for discovery"))] MissingCoordinatorPods, + #[snafu(display("invalid OpaConfig"))] + InvalidOpaConfig { + source: stackable_operator::error::Error, + }, } type Result = std::result::Result; @@ -142,10 +142,17 @@ pub async fn reconcile_trino(trino: Arc, ctx: Context) -> Res let authentication_config = user_authentication(&trino, client).await?; - // rego rules - create_rego_rules(client, &trino) - .await - .context(WriteRegoRuleAuthorizationFailedSnafu)?; + // Assemble the OPA connection string from the discovery and the given path if provided + let opa_connect_string = if let Some(opa_config) = &trino.spec.opa { + Some( + opa_config + .full_document_url_from_config_map(client, &*trino, None, OpaApiVersion::V1) + .await + .context(InvalidOpaConfigSnafu)?, + ) + } else { + None + }; let coordinator_role_service = build_coordinator_role_service(&trino)?; client @@ -162,8 +169,13 @@ pub async fn reconcile_trino(trino: Arc, ctx: Context) -> Res for (role_group, config) in role_config { let rolegroup = trino_role.rolegroup_ref(&trino, role_group); let rg_service = build_rolegroup_service(&trino, &rolegroup)?; - let rg_configmap = - build_rolegroup_config_map(&trino, &trino_role, &rolegroup, &config)?; + let rg_configmap = build_rolegroup_config_map( + &trino, + &trino_role, + &rolegroup, + &config, + opa_connect_string.as_deref(), + )?; let rg_catalog_configmap = build_rolegroup_catalog_config_map(&trino, &rolegroup, &config)?; let rg_stateful_set = build_rolegroup_statefulset( @@ -242,6 +254,7 @@ fn build_rolegroup_config_map( _role: &TrinoRole, rolegroup_ref: &RoleGroupRef, config: &HashMap>, + opa_connect_string: Option<&str>, ) -> Result { let mut cm_conf_data = BTreeMap::new(); @@ -323,14 +336,15 @@ fn build_rolegroup_config_map( } } - if trino.spec.opa_config_map_name.is_some() { + if let Some(opa_connect) = opa_connect_string { let mut opa_config = BTreeMap::new(); - // the "opa.policy.uri" property will be added via command script later from the env variable "OPA" opa_config.insert( "access-control.name".to_string(), Some("tech.stackable.trino.opa.OpaAuthorizer".to_string()), ); + opa_config.insert("opa.policy.uri".to_string(), Some(opa_connect.to_string())); + let config_properties = product_config::writer::to_java_properties_string(opa_config.iter()) .context(FailedToWriteJavaPropertiesSnafu)?; @@ -452,10 +466,6 @@ fn build_rolegroup_statefulset( }) .collect::>(); - if let Some(opa) = env_var_from_discovery_config_map(&trino.spec.opa_config_map_name, "OPA") { - env.push(opa); - }; - if let Some(hive) = env_var_from_discovery_config_map(&trino.spec.hive_config_map_name, "HIVE") { env.push(hive); @@ -765,24 +775,6 @@ fn container_trino_args( rw_conf = RW_CONFIG_DIR_NAME, hive_properties = HIVE_PROPERTIES )]) } - // opa required? - if trino.spec.opa_config_map_name.is_some() { - let opa_package_name = match trino.spec.authorization.as_ref() { - Some(auth) => auth.package.clone(), - None => { - warn!("No package specified in 'spec.authorization'. Defaulting to 'trino'."); - "trino".to_string() - } - }; - - args.extend(vec![ - format!( "echo Writing OPA connect string \"opa.policy.uri=${{OPA}}v1/data/{package_name}\" to {rw_conf}/{access_control}", - package_name = opa_package_name, rw_conf = RW_CONFIG_DIR_NAME, access_control = ACCESS_CONTROL_PROPERTIES - ), - format!( "echo \"opa.policy.uri=${{OPA}}v1/data/{package_name}/\" >> {rw_conf}/{access_control}", - package_name = opa_package_name, rw_conf = RW_CONFIG_DIR_NAME, access_control = ACCESS_CONTROL_PROPERTIES - )]) - } // start command args.push(format!( From 52a4bc22bc8e62967abd6f86891fd6ce3ab40642 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 8 Apr 2022 11:11:33 +0200 Subject: [PATCH 02/16] adapted docs and examples --- docs/modules/ROOT/pages/usage.adoc | 22 +------------------ ...ster-authentication-opa-authorization.yaml | 2 +- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 6a021aaa..804376fb 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -106,27 +106,7 @@ data: can_view_query_owned_by = true ---- -You can let the Trino operator write its own Rego rules by configuring the `authorization` field in the custom resource. This is a rudimentary implementation for user access. - -[source,yaml] ----- -authorization: - package: trino - permissions: - admin: - schemas: - read: true - write: true - tables: - iris_parquet: - read: true - write: true - iris_csv: - read: true - write: true ----- - -Here we define permissions for an admin user who can read and write `schemas`, as well as having full access to the `iris_parquet` and `iris_csv` table. Currently, this is more for demonstration purposes. Users should write their own rego rules for more complex OPA authorization. +Users should write their own rego rules for more complex OPA authorization. === Trino diff --git a/examples/simple-trino-cluster-authentication-opa-authorization.yaml b/examples/simple-trino-cluster-authentication-opa-authorization.yaml index ddb62d5a..c5625838 100644 --- a/examples/simple-trino-cluster-authentication-opa-authorization.yaml +++ b/examples/simple-trino-cluster-authentication-opa-authorization.yaml @@ -54,7 +54,7 @@ stringData: apiVersion: v1 kind: ConfigMap metadata: - name: simple-trino-opa-rego + name: simple-trino-opa-bundle labels: opa.stackable.tech/bundle: "trino" data: From 5852341a5d982c6a9f89eacf4c39b4ab278fec84 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 8 Apr 2022 11:38:57 +0200 Subject: [PATCH 03/16] regenerated charts --- deploy/helm/trino-operator/crds/crds.yaml | 48 +++++------------------ deploy/manifests/crds.yaml | 48 +++++------------------ 2 files changed, 20 insertions(+), 76 deletions(-) diff --git a/deploy/helm/trino-operator/crds/crds.yaml b/deploy/helm/trino-operator/crds/crds.yaml index c063af12..7171b859 100644 --- a/deploy/helm/trino-operator/crds/crds.yaml +++ b/deploy/helm/trino-operator/crds/crds.yaml @@ -52,42 +52,6 @@ spec: required: - method type: object - authorization: - nullable: true - properties: - package: - type: string - permissions: - additionalProperties: - properties: - schemas: - nullable: true - properties: - read: - nullable: true - type: boolean - write: - nullable: true - type: boolean - type: object - tables: - additionalProperties: - properties: - read: - nullable: true - type: boolean - write: - nullable: true - type: boolean - type: object - nullable: true - type: object - type: object - type: object - required: - - package - - permissions - type: object coordinators: nullable: true properties: @@ -204,9 +168,17 @@ spec: hiveConfigMapName: nullable: true type: string - opaConfigMapName: + opa: nullable: true - type: string + properties: + configMapName: + type: string + package: + nullable: true + type: string + required: + - configMapName + type: object s3: description: Contains all the required connection information for S3. nullable: true diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 3e860c67..9c1f5bb3 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -53,42 +53,6 @@ spec: required: - method type: object - authorization: - nullable: true - properties: - package: - type: string - permissions: - additionalProperties: - properties: - schemas: - nullable: true - properties: - read: - nullable: true - type: boolean - write: - nullable: true - type: boolean - type: object - tables: - additionalProperties: - properties: - read: - nullable: true - type: boolean - write: - nullable: true - type: boolean - type: object - nullable: true - type: object - type: object - type: object - required: - - package - - permissions - type: object coordinators: nullable: true properties: @@ -205,9 +169,17 @@ spec: hiveConfigMapName: nullable: true type: string - opaConfigMapName: + opa: nullable: true - type: string + properties: + configMapName: + type: string + package: + nullable: true + type: string + required: + - configMapName + type: object s3: description: Contains all the required connection information for S3. nullable: true From f8cf46d8a426f483a555412dcdb9d057b3f0d4a3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 8 Apr 2022 12:15:48 +0200 Subject: [PATCH 04/16] cleanup --- ...o-cluster-authentication-opa-authorization.yaml | 14 +------------- rust/operator-binary/src/controller.rs | 8 +++++++- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/examples/simple-trino-cluster-authentication-opa-authorization.yaml b/examples/simple-trino-cluster-authentication-opa-authorization.yaml index c5625838..5b1b96db 100644 --- a/examples/simple-trino-cluster-authentication-opa-authorization.yaml +++ b/examples/simple-trino-cluster-authentication-opa-authorization.yaml @@ -88,63 +88,51 @@ data: } } - default can_access_table = false can_access_table { user_can_read_table } - default can_create_table = false can_create_table { user_can_write_table } - default can_drop_table = false can_drop_table { user_can_write_table } - default can_show_tables = false can_show_tables { user_can_read_table } - default can_access_schema = false can_access_schema { user_can_read_schema } - default can_create_schema = false can_create_schema { user_can_write_schema } - default can_drop_schema = false can_drop_schema { user_can_write_schema } - default can_show_schemas = false can_show_schemas { user_can_read_schema } - default can_access_catalog = false can_access_catalog { is_valid_user } - default can_execute_query = false can_execute_query { is_valid_user } - default can_select_from_columns = false can_select_from_columns { - is_valid_user + is_valid_user can_access_table } - default can_view_query_owned_by = false can_view_query_owned_by { is_valid_user } diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 82c4f3fa..aabb8b7b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -343,7 +343,13 @@ fn build_rolegroup_config_map( Some("tech.stackable.trino.opa.OpaAuthorizer".to_string()), ); - opa_config.insert("opa.policy.uri".to_string(), Some(opa_connect.to_string())); + opa_config.insert( + "opa.policy.uri".to_string(), + // TODO: We have to add a slash in the end of the URL, otherwise the authorizer + // ignores / cuts off the package name and can not find the rule + // see: https://github.com/stackabletech/trino-opa-authorizer/issues/15 + Some(format!("{}/", opa_connect)), + ); let config_properties = product_config::writer::to_java_properties_string(opa_config.iter()) From 5301c957212eae5e03e42f08e9aa6e9e3b436f6a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 8 Apr 2022 12:19:10 +0200 Subject: [PATCH 05/16] adapted changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a225f9e5..2ff5c38f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,15 +17,19 @@ All notable changes to this project will be documented in this file. the custom resource for the obsolete regorule-operator. This means that the rego rule operator is not required anymore for authorization and opa-operator tag >= `0.9.0` ([#157]). +- BREAKING: `OpaConfigMapName` in CRD to `opa` using the `OpaConfig` from operator-rs ([#186]). ### Removed - `stackable-regorule-crd` dependency ([#157]). - BREAKING: `nodeEnvironment` from CRD. Will default to the `metadata.name` field (can be overriden) ([#183]). +- BREAKING: Removed `authorization` module from CRD and code and provided the opa bundle via `ConfigMap` directly + instead of generating it ([#186]). [#149]: https://github.com/stackabletech/trino-operator/pull/149 [#157]: https://github.com/stackabletech/trino-operator/pull/157 [#183]: https://github.com/stackabletech/trino-operator/pull/183 +[#186]: https://github.com/stackabletech/trino-operator/pull/186 ## [0.3.1] - 2022-02-17 From 881de0f42b2d364ad48fe46ee9829005bb8a4f0f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 8 Apr 2022 13:18:11 +0200 Subject: [PATCH 06/16] attempt to fix linters --- CHANGELOG.md | 3 +- ...ster-authentication-opa-authorization.yaml | 42 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ff5c38f..5864c59c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,8 +23,7 @@ All notable changes to this project will be documented in this file. - `stackable-regorule-crd` dependency ([#157]). - BREAKING: `nodeEnvironment` from CRD. Will default to the `metadata.name` field (can be overriden) ([#183]). -- BREAKING: Removed `authorization` module from CRD and code and provided the opa bundle via `ConfigMap` directly - instead of generating it ([#186]). +- BREAKING: Removed `authorization` module from CRD and code and provided the opa bundle via `ConfigMap` directly instead of generating it ([#186]). [#149]: https://github.com/stackabletech/trino-operator/pull/149 [#157]: https://github.com/stackabletech/trino-operator/pull/157 diff --git a/examples/simple-trino-cluster-authentication-opa-authorization.yaml b/examples/simple-trino-cluster-authentication-opa-authorization.yaml index 5b1b96db..781140f9 100644 --- a/examples/simple-trino-cluster-authentication-opa-authorization.yaml +++ b/examples/simple-trino-cluster-authentication-opa-authorization.yaml @@ -62,12 +62,12 @@ data: package trino users = { - "admin": { + "admin": { "schemas": { "read": true, "write": true }, - "tables":{ + "tables":{ "iris_parquet": { "read": true, "write": true @@ -75,8 +75,8 @@ data: } }, "bob": { - "schemas": { - "read": true, + "schemas": { + "read": true, "write": false }, "tables": { @@ -87,72 +87,72 @@ data: } } } - + can_access_table { user_can_read_table } - + can_create_table { user_can_write_table } - + can_drop_table { user_can_write_table } - + can_show_tables { user_can_read_table } - + can_access_schema { user_can_read_schema } - + can_create_schema { user_can_write_schema } - + can_drop_schema { user_can_write_schema } - + can_show_schemas { user_can_read_schema } - + can_access_catalog { is_valid_user } - + can_execute_query { is_valid_user } - + can_select_from_columns { is_valid_user can_access_table } - + can_view_query_owned_by { is_valid_user } - + user_can_read_table { users[input.user.name].tables[input.request.table.table].read == true } - + user_can_write_table { users[input.user.name].tables[input.request.table.table].write == true } - + user_can_read_schema { users[input.user.name].schemas.read == true } - + user_can_write_schema { users[input.user.name].schemas.write == true } - + is_valid_user { _ = users[input.user.name] } From 7a4032f28746cba3140fd94de8d6eedca5180b69 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 12 Apr 2022 14:10:16 +0200 Subject: [PATCH 07/16] mini docs changes --- docs/modules/ROOT/pages/usage.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 804376fb..e163b7c2 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -1,4 +1,4 @@ -= Usage +/home/felix/repos/kafka-operator= Usage Trino works together with the Apache Hive metastore and S3 bucket. @@ -65,11 +65,11 @@ htpasswd -nbBC 10 admin admin === Authorization -In order to authorize Trino via OPA, a `ConfigMap` containing Rego rules for Trino has to be applied. The following example is an all access Rego rule for testing. Do not use that in production! +In order to authorize Trino via OPA, a `ConfigMap` containing Rego rules for Trino has to be applied. The following example is an all access Rego rule for testing. Do not use it in production! [source,yaml] ---- -apiVersion: opa.stackable.tech/v1alpha1 +apiVersion: v1 kind: ConfigMap metadata: name: opa-bundle-trino From bda1ed512a8d658ac9553b48cade12a121acbd8e Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Tue, 12 Apr 2022 14:10:58 +0200 Subject: [PATCH 08/16] ~ --- docs/modules/ROOT/pages/usage.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index e163b7c2..a2a0d766 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -1,4 +1,4 @@ -/home/felix/repos/kafka-operator= Usage += Usage Trino works together with the Apache Hive metastore and S3 bucket. From be9cdacc0fad2dbd8bfbc6c2f361bc96fcbe821d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 20 Apr 2022 16:50:52 +0200 Subject: [PATCH 09/16] adapted to trino 377 and new opa authorizer --- deploy/config-spec/properties.yaml | 19 --- deploy/crd/trinocluster.crd.yaml | 12 -- .../trino-operator/configs/properties.yaml | 19 --- deploy/helm/trino-operator/crds/crds.yaml | 12 -- deploy/manifests/configmap.yaml | 2 +- deploy/manifests/crds.yaml | 12 -- docs/modules/ROOT/pages/usage.adoc | 32 ++--- .../ROOT/partials/supported-versions.adoc | 2 +- ...ster-authentication-opa-authorization.yaml | 119 ++++-------------- rust/crd/src/lib.rs | 13 +- rust/operator-binary/src/controller.rs | 20 ++- 11 files changed, 46 insertions(+), 216 deletions(-) diff --git a/deploy/config-spec/properties.yaml b/deploy/config-spec/properties.yaml index 4b1e8ef4..ee36e810 100644 --- a/deploy/config-spec/properties.yaml +++ b/deploy/config-spec/properties.yaml @@ -148,25 +148,6 @@ properties: required: true asOfVersion: "0.0.0" - - property: &queryMaxTotalMemoryPerNode - propertyNames: - - name: "query.max-total-memory-per-node" - kind: - type: "file" - file: "config.properties" - datatype: - type: "string" - unit: *unitMemory - defaultValues: - - fromVersion: "0.0.0" - value: "2GB" - roles: - - name: "coordinator" - required: true - - name: "worker" - required: true - asOfVersion: "0.0.0" - - property: &httpServerAuthenticationType propertyNames: - name: "http-server.authentication.type" diff --git a/deploy/crd/trinocluster.crd.yaml b/deploy/crd/trinocluster.crd.yaml index c6d14bc0..3e41e42b 100644 --- a/deploy/crd/trinocluster.crd.yaml +++ b/deploy/crd/trinocluster.crd.yaml @@ -70,9 +70,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -106,9 +103,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -225,9 +219,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -261,9 +252,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: diff --git a/deploy/helm/trino-operator/configs/properties.yaml b/deploy/helm/trino-operator/configs/properties.yaml index 4b1e8ef4..ee36e810 100644 --- a/deploy/helm/trino-operator/configs/properties.yaml +++ b/deploy/helm/trino-operator/configs/properties.yaml @@ -148,25 +148,6 @@ properties: required: true asOfVersion: "0.0.0" - - property: &queryMaxTotalMemoryPerNode - propertyNames: - - name: "query.max-total-memory-per-node" - kind: - type: "file" - file: "config.properties" - datatype: - type: "string" - unit: *unitMemory - defaultValues: - - fromVersion: "0.0.0" - value: "2GB" - roles: - - name: "coordinator" - required: true - - name: "worker" - required: true - asOfVersion: "0.0.0" - - property: &httpServerAuthenticationType propertyNames: - name: "http-server.authentication.type" diff --git a/deploy/helm/trino-operator/crds/crds.yaml b/deploy/helm/trino-operator/crds/crds.yaml index 7171b859..9b02afa2 100644 --- a/deploy/helm/trino-operator/crds/crds.yaml +++ b/deploy/helm/trino-operator/crds/crds.yaml @@ -72,9 +72,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -108,9 +105,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -227,9 +221,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -263,9 +254,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: diff --git a/deploy/manifests/configmap.yaml b/deploy/manifests/configmap.yaml index 3bb54fe7..c1e0ea06 100644 --- a/deploy/manifests/configmap.yaml +++ b/deploy/manifests/configmap.yaml @@ -1,7 +1,7 @@ --- apiVersion: v1 data: - properties.yaml: "version: 0.1.0\nspec:\n units:\n - unit: &unitNodeEnvironment\n name: \"node_environment\"\n regex: \"^[a-z][a-z0-9_]*[a-z0-9]$\"\n examples:\n - \"a1_2_3b\"\n - unit: &unitMemory\n name: \"memory\"\n regex: \"(^\\\\p{N}+)(?:\\\\s*)((?:b|k|m|g|t|p|kb|mb|gb|tb|pb|B|K|M|G|T|P|KB|MB|GB|TB|PB)\\\\b$)\"\n examples:\n - \"1024b\"\n - \"1024kb\"\n - \"500m\"\n - \"1g\"\n\n###################################################################################################\n# node.properties\n###################################################################################################\nproperties:\n - property: &nodeEnvironment\n propertyNames:\n - name: \"node.environment\"\n kind:\n type: \"file\"\n file: \"node.properties\"\n datatype:\n type: \"string\"\n unit: *unitNodeEnvironment\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n###################################################################################################\n# config.properties\n###################################################################################################\n - property: &coordinator\n propertyNames:\n - name: \"coordinator\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"bool\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"false\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &nodeSchedulerIncludeCoordinator\n propertyNames:\n - name: \"node-scheduler.include-coordinator\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"bool\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"false\"\n roles:\n - name: \"coordinator\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &httpServerHttpPort\n propertyNames:\n - name: \"http-server.http.port\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"integer\"\n min: \"1024\"\n max: \"65535\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"8080\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &httpServerHttpsPort\n propertyNames:\n - name: \"http-server.https.port\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"integer\"\n min: \"1024\"\n max: \"65535\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"8443\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n - property: &queryMaxMemory\n propertyNames:\n - name: \"query.max-memory\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"string\"\n unit: *unitMemory\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"50GB\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &queryMaxMemoryPerNode\n propertyNames:\n - name: \"query.max-memory-per-node\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"string\"\n unit: *unitMemory\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"1GB\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &queryMaxTotalMemoryPerNode\n propertyNames:\n - name: \"query.max-total-memory-per-node\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"string\"\n unit: *unitMemory\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"2GB\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &httpServerAuthenticationType\n propertyNames:\n - name: \"http-server.authentication.type\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"string\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n###################################################################################################\n# password-authenticator.properties\n###################################################################################################\n\n - property: &passwordAuthenticatorName\n propertyNames:\n - name: \"password-authenticator.name\"\n kind:\n type: \"file\"\n file: \"password-authenticator.properties\"\n datatype:\n type: \"string\"\n allowedValues:\n - \"file\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n - property: &passwordAuthenticatorName\n propertyNames:\n - name: \"file.password-file\"\n kind:\n type: \"file\"\n file: \"password-authenticator.properties\"\n datatype:\n type: \"string\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n\n###################################################################################################\n# jvm.config\n###################################################################################################\n\n###################################################################################################\n# log.properties\n###################################################################################################\n\n - property: &ioTrino\n propertyNames:\n - name: \"io.trino\"\n kind:\n type: \"file\"\n file: \"log.properties\"\n datatype:\n type: \"string\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"INFO\"\n allowedValues:\n - \"INFO\"\n - \"DEBUG\"\n - \"WARN\"\n - \"ERROR\" \n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n" + properties.yaml: "version: 0.1.0\nspec:\n units:\n - unit: &unitNodeEnvironment\n name: \"node_environment\"\n regex: \"^[a-z][a-z0-9_]*[a-z0-9]$\"\n examples:\n - \"a1_2_3b\"\n - unit: &unitMemory\n name: \"memory\"\n regex: \"(^\\\\p{N}+)(?:\\\\s*)((?:b|k|m|g|t|p|kb|mb|gb|tb|pb|B|K|M|G|T|P|KB|MB|GB|TB|PB)\\\\b$)\"\n examples:\n - \"1024b\"\n - \"1024kb\"\n - \"500m\"\n - \"1g\"\n\n###################################################################################################\n# node.properties\n###################################################################################################\nproperties:\n - property: &nodeEnvironment\n propertyNames:\n - name: \"node.environment\"\n kind:\n type: \"file\"\n file: \"node.properties\"\n datatype:\n type: \"string\"\n unit: *unitNodeEnvironment\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n###################################################################################################\n# config.properties\n###################################################################################################\n - property: &coordinator\n propertyNames:\n - name: \"coordinator\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"bool\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"false\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &nodeSchedulerIncludeCoordinator\n propertyNames:\n - name: \"node-scheduler.include-coordinator\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"bool\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"false\"\n roles:\n - name: \"coordinator\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &httpServerHttpPort\n propertyNames:\n - name: \"http-server.http.port\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"integer\"\n min: \"1024\"\n max: \"65535\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"8080\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &httpServerHttpsPort\n propertyNames:\n - name: \"http-server.https.port\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"integer\"\n min: \"1024\"\n max: \"65535\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"8443\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n - property: &queryMaxMemory\n propertyNames:\n - name: \"query.max-memory\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"string\"\n unit: *unitMemory\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"50GB\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &queryMaxMemoryPerNode\n propertyNames:\n - name: \"query.max-memory-per-node\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"string\"\n unit: *unitMemory\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"1GB\"\n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n\n - property: &httpServerAuthenticationType\n propertyNames:\n - name: \"http-server.authentication.type\"\n kind:\n type: \"file\"\n file: \"config.properties\"\n datatype:\n type: \"string\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n###################################################################################################\n# password-authenticator.properties\n###################################################################################################\n\n - property: &passwordAuthenticatorName\n propertyNames:\n - name: \"password-authenticator.name\"\n kind:\n type: \"file\"\n file: \"password-authenticator.properties\"\n datatype:\n type: \"string\"\n allowedValues:\n - \"file\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n - property: &passwordAuthenticatorName\n propertyNames:\n - name: \"file.password-file\"\n kind:\n type: \"file\"\n file: \"password-authenticator.properties\"\n datatype:\n type: \"string\"\n roles:\n - name: \"coordinator\"\n required: false\n asOfVersion: \"0.0.0\"\n\n\n###################################################################################################\n# jvm.config\n###################################################################################################\n\n###################################################################################################\n# log.properties\n###################################################################################################\n\n - property: &ioTrino\n propertyNames:\n - name: \"io.trino\"\n kind:\n type: \"file\"\n file: \"log.properties\"\n datatype:\n type: \"string\"\n defaultValues:\n - fromVersion: \"0.0.0\"\n value: \"INFO\"\n allowedValues:\n - \"INFO\"\n - \"DEBUG\"\n - \"WARN\"\n - \"ERROR\" \n roles:\n - name: \"coordinator\"\n required: true\n - name: \"worker\"\n required: true\n asOfVersion: \"0.0.0\"\n" kind: ConfigMap metadata: name: trino-operator-configmap diff --git a/deploy/manifests/crds.yaml b/deploy/manifests/crds.yaml index 9c1f5bb3..15fcb4e8 100644 --- a/deploy/manifests/crds.yaml +++ b/deploy/manifests/crds.yaml @@ -73,9 +73,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -109,9 +106,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -228,9 +222,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: @@ -264,9 +255,6 @@ spec: queryMaxMemoryPerNode: nullable: true type: string - queryMaxTotalMemoryPerNode: - nullable: true - type: string type: object configOverrides: additionalProperties: diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index a2a0d766..12751886 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -65,7 +65,7 @@ htpasswd -nbBC 10 admin admin === Authorization -In order to authorize Trino via OPA, a `ConfigMap` containing Rego rules for Trino has to be applied. The following example is an all access Rego rule for testing. Do not use it in production! +In order to authorize Trino via OPA, a `ConfigMap` containing Rego rules for Trino has to be applied. The following example is an all access Rego rule for testing with the user `admin`. Do not use it in production! [source,yaml] ---- @@ -79,31 +79,17 @@ data: trino.rego: | package trino - can_execute_query = true + import future.keywords.in - can_access_catalog = true + default allow = false - can_create_schema = true + allow { + is_admin + } - can_drop_schema = true - - can_access_schema = true - - can_create_table = true - - can_drop_table = true - - can_access_table = true - - can_access_column = true - - can_show_schemas = true - - can_show_tables = true - - can_select_from_columns = true - - can_view_query_owned_by = true + is_admin() { + input.context.identity.user == "admin" + } ---- Users should write their own rego rules for more complex OPA authorization. diff --git a/docs/modules/ROOT/partials/supported-versions.adoc b/docs/modules/ROOT/partials/supported-versions.adoc index 2caccc7a..266db339 100644 --- a/docs/modules/ROOT/partials/supported-versions.adoc +++ b/docs/modules/ROOT/partials/supported-versions.adoc @@ -2,4 +2,4 @@ // This is a separate file, since it is used by both the direct Trino documentation, and the overarching // Stackable Platform documentation. -- 362 +- 377 diff --git a/examples/simple-trino-cluster-authentication-opa-authorization.yaml b/examples/simple-trino-cluster-authentication-opa-authorization.yaml index 781140f9..1612a7ae 100644 --- a/examples/simple-trino-cluster-authentication-opa-authorization.yaml +++ b/examples/simple-trino-cluster-authentication-opa-authorization.yaml @@ -60,101 +60,30 @@ metadata: data: trino.rego: | package trino - - users = { - "admin": { - "schemas": { - "read": true, - "write": true - }, - "tables":{ - "iris_parquet": { - "read": true, - "write": true - } - } - }, - "bob": { - "schemas": { - "read": true, - "write": false - }, - "tables": { - "iris_parquet": { - "read": false, - "write": null - } - } - } - } - - can_access_table { - user_can_read_table - } - - can_create_table { - user_can_write_table - } - - can_drop_table { - user_can_write_table - } - - can_show_tables { - user_can_read_table - } - - can_access_schema { - user_can_read_schema - } - - can_create_schema { - user_can_write_schema - } - - can_drop_schema { - user_can_write_schema - } - - can_show_schemas { - user_can_read_schema - } - - can_access_catalog { - is_valid_user - } - - can_execute_query { - is_valid_user - } - - can_select_from_columns { - is_valid_user - can_access_table - } - - can_view_query_owned_by { - is_valid_user - } - - user_can_read_table { - users[input.user.name].tables[input.request.table.table].read == true - } - - user_can_write_table { - users[input.user.name].tables[input.request.table.table].write == true - } - - user_can_read_schema { - users[input.user.name].schemas.read == true - } - - user_can_write_schema { - users[input.user.name].schemas.write == true - } - - is_valid_user { - _ = users[input.user.name] + + import future.keywords.in + + default allow = false + + allow { + is_admin + } + + allow { + is_bob + can_be_accessed_by_bob + } + + is_admin() { + input.context.identity.user == "admin" + } + + is_bob() { + input.context.identity.user == "bob" + } + + can_be_accessed_by_bob() { + input.action.operation in ["ImpersonateUser", "FilterCatalogs", "AccessCatalog", "ExecuteQuery"] } --- apiVersion: trino.stackable.tech/v1alpha1 diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 450bb70d..83adf01d 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -43,12 +43,12 @@ pub const COORDINATOR: &str = "coordinator"; pub const HTTP_SERVER_HTTP_PORT: &str = "http-server.http.port"; pub const HTTP_SERVER_HTTPS_PORT: &str = "http-server.https.port"; pub const HTTP_SERVER_HTTPS_ENABLED: &str = "http-server.https.enabled"; +pub const HTTP_SERVER_HTTPS_KEYSTORE_KEY: &str = "http-server.https.keystore.key"; pub const HTTP_SERVER_KEYSTORE_PATH: &str = "http-server.https.keystore.path"; pub const HTTP_SERVER_AUTHENTICATION_TYPE: &str = "http-server.authentication.type"; pub const HTTP_SERVER_AUTHENTICATION_TYPE_PASSWORD: &str = "PASSWORD"; pub const QUERY_MAX_MEMORY: &str = "query.max-memory"; pub const QUERY_MAX_MEMORY_PER_NODE: &str = "query.max-memory-per-node"; -pub const QUERY_MAX_TOTAL_MEMORY_PER_NODE: &str = "query.max-total-memory-per-node"; pub const DISCOVERY_URI: &str = "discovery.uri"; // password-authenticator.properties pub const PASSWORD_AUTHENTICATOR_NAME: &str = "password-authenticator.name"; @@ -207,7 +207,6 @@ pub struct TrinoConfig { // config.properties pub query_max_memory: Option, pub query_max_memory_per_node: Option, - pub query_max_total_memory_per_node: Option, // log.properties pub log_level: Option, } @@ -269,14 +268,6 @@ impl Configuration for TrinoConfig { ); } - if let Some(query_max_total_memory_per_node) = &self.query_max_total_memory_per_node - { - result.insert( - QUERY_MAX_TOTAL_MEMORY_PER_NODE.to_string(), - Some(query_max_total_memory_per_node.to_string()), - ); - } - if resource.spec.authentication.is_some() { result.insert( HTTP_SERVER_HTTPS_ENABLED.to_string(), @@ -292,7 +283,7 @@ impl Configuration for TrinoConfig { ); result.insert( - "http-server.https.keystore.key".to_string(), + HTTP_SERVER_HTTPS_KEYSTORE_KEY.to_string(), Some("secret".to_string()), ); diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index aabb8b7b..64449126 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -146,7 +146,12 @@ pub async fn reconcile_trino(trino: Arc, ctx: Context) -> Res let opa_connect_string = if let Some(opa_config) = &trino.spec.opa { Some( opa_config - .full_document_url_from_config_map(client, &*trino, None, OpaApiVersion::V1) + .full_document_url_from_config_map( + client, + &*trino, + Some("allow"), + OpaApiVersion::V1, + ) .await .context(InvalidOpaConfigSnafu)?, ) @@ -342,14 +347,7 @@ fn build_rolegroup_config_map( "access-control.name".to_string(), Some("tech.stackable.trino.opa.OpaAuthorizer".to_string()), ); - - opa_config.insert( - "opa.policy.uri".to_string(), - // TODO: We have to add a slash in the end of the URL, otherwise the authorizer - // ignores / cuts off the package name and can not find the rule - // see: https://github.com/stackabletech/trino-opa-authorizer/issues/15 - Some(format!("{}/", opa_connect)), - ); + opa_config.insert("opa.policy.uri".to_string(), Some(opa_connect.to_string())); let config_properties = product_config::writer::to_java_properties_string(opa_config.iter()) @@ -494,8 +492,8 @@ fn build_rolegroup_statefulset( let container_trino = ContainerBuilder::new(APP_NAME) .image(format!( - "docker.stackable.tech/stackable/trino:{}-stackable0", - trino_version + "docker.stackable.tech/stackable/trino:0.5.0", + //trino_version )) .command(vec!["/bin/bash".to_string(), "-c".to_string()]) .args(container_trino_args(trino, authentication_config)) From e8dc3dd11eb35a7cd03634ab458c11205f400cbe Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 20 Apr 2022 16:52:12 +0200 Subject: [PATCH 10/16] adapted version to 377 in examples and doc --- docs/modules/ROOT/pages/usage.adoc | 4 ++-- ...simple-trino-cluster-authentication-opa-authorization.yaml | 2 +- examples/simple-trino-cluster.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 12751886..9b054cc6 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -113,7 +113,7 @@ kind: TrinoCluster metadata: name: simple-trino spec: - version: "0.0.362" + version: "0.0.377" hiveConfigMapName: simple-hive-derby s3: endPoint: changeme @@ -156,7 +156,7 @@ kind: TrinoCluster metadata: name: simple-trino spec: - version: "0.0.362" + version: "0.0.377" hiveConfigMapName: simple-hive-derby opa: configMapName: simple-opa diff --git a/examples/simple-trino-cluster-authentication-opa-authorization.yaml b/examples/simple-trino-cluster-authentication-opa-authorization.yaml index 1612a7ae..0279b18e 100644 --- a/examples/simple-trino-cluster-authentication-opa-authorization.yaml +++ b/examples/simple-trino-cluster-authentication-opa-authorization.yaml @@ -91,7 +91,7 @@ kind: TrinoCluster metadata: name: simple-trino spec: - version: "0.0.362" + version: "0.0.377" hiveConfigMapName: simple-hive-derby opa: configMapName: simple-opa diff --git a/examples/simple-trino-cluster.yaml b/examples/simple-trino-cluster.yaml index e113fa1d..7a704246 100644 --- a/examples/simple-trino-cluster.yaml +++ b/examples/simple-trino-cluster.yaml @@ -30,7 +30,7 @@ kind: TrinoCluster metadata: name: simple-trino spec: - version: "0.0.362" + version: "0.0.377" hiveConfigMapName: simple-hive-derby s3: endPoint: changeme From ec103bf29ec6c8723fedc507dbecbdea5c41357f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 20 Apr 2022 17:10:37 +0200 Subject: [PATCH 11/16] adapted to operator-rs 0.17 --- CHANGELOG.md | 6 +- Cargo.lock | 285 +++++++++++++++++++++++-- rust/crd/Cargo.toml | 2 +- rust/crd/src/lib.rs | 2 +- rust/operator-binary/Cargo.toml | 4 +- rust/operator-binary/src/controller.rs | 20 +- rust/operator-binary/src/main.rs | 10 +- 7 files changed, 288 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5864c59c..8c894107 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,23 +12,27 @@ All notable changes to this project will be documented in this file. ### Changed -- `operator-rs` `0.10.0` -> `0.15.0` ([#149], [#157], [#183]). +- `operator-rs` `0.10.0` -> `0.17.0` ([#149], [#157], [#183], [#xxx]). - BREAKING: The operator now writes a `ConfigMap` for Rego rules instead of the custom resource for the obsolete regorule-operator. This means that the rego rule operator is not required anymore for authorization and opa-operator tag >= `0.9.0` ([#157]). - BREAKING: `OpaConfigMapName` in CRD to `opa` using the `OpaConfig` from operator-rs ([#186]). +- Trino version to 377 ([#xxx]). +- Opa rego example adapted to the new `trino-opa-authorizer` ([#xxx]). ### Removed - `stackable-regorule-crd` dependency ([#157]). - BREAKING: `nodeEnvironment` from CRD. Will default to the `metadata.name` field (can be overriden) ([#183]). - BREAKING: Removed `authorization` module from CRD and code and provided the opa bundle via `ConfigMap` directly instead of generating it ([#186]). +- Possibly BREAKING: Removed `query.max-total-memory-per-node` config parameter ([#xxx]). [#149]: https://github.com/stackabletech/trino-operator/pull/149 [#157]: https://github.com/stackabletech/trino-operator/pull/157 [#183]: https://github.com/stackabletech/trino-operator/pull/183 [#186]: https://github.com/stackabletech/trino-operator/pull/186 +[#xxx]: https://github.com/stackabletech/trino-operator/pull/xxx ## [0.3.1] - 2022-02-17 diff --git a/Cargo.lock b/Cargo.lock index c717fd09..3c372c15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,17 @@ version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +[[package]] +name = "async-trait" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -103,6 +114,18 @@ dependencies = [ "git2", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.1.0" @@ -216,6 +239,26 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + [[package]] name = "darling" version = "0.13.4" @@ -628,6 +671,24 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-openssl" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ee5d7a8f718585d1c3c61dfde28ef5b0bb14734b4db13f5ada856cdc6c612b" +dependencies = [ + "http", + "hyper", + "linked_hash_set", + "once_cell", + "openssl", + "openssl-sys", + "parking_lot", + "tokio", + "tokio-openssl", + "tower-layer", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -698,6 +759,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "integer-encoding" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e85a1509a128c855368e135cffcde7eac17d8e1083f41e2b98c58bc1a5074be" + [[package]] name = "itoa" version = "1.0.1" @@ -724,6 +791,15 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "json-patch" version = "0.2.6" @@ -763,9 +839,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcc72fdf0c491160a34d4a1bfb03f96da8a5054288d61c816d514b5c2fa49ea" +checksum = "342744dfeb81fe186b84f485b33f12c6a15d3396987d933b06a566a3db52ca38" dependencies = [ "k8s-openapi", "kube-client", @@ -776,9 +852,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b01c722d55ffedec74cbc259b4508d8a59bf19540006ec87618f76ab156579" +checksum = "3f69a504997799340408635d6e351afb8aab2c34ca3165e162f41b3b34a69a79" dependencies = [ "base64", "bytes", @@ -789,6 +865,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-openssl", "hyper-timeout", "hyper-tls", "jsonpath_lib", @@ -812,9 +889,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd9e3535777edd122cc26fe3fe6357066b33eff63d8b919862edbe7a956a679" +checksum = "a4a247487699941baaf93438d65b12d4e32450bea849d619d19ed394e8a4a645" dependencies = [ "chrono", "form_urlencoded", @@ -830,9 +907,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1322e25c20dd6f18ca6baecc88bb130331e99d988df9d7a9a207f15819e05bff" +checksum = "203f7c5acf9d0dfb0b08d44ec1d66ace3d1dfe0cdd82e65e274f3f96615d666c" dependencies = [ "darling", "proc-macro2", @@ -843,9 +920,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.70.0" +version = "0.71.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816c8c086f8bbcf9a4db0b7a68db90b784ef6292a57de35c64cccb90d5edfbe5" +checksum = "02ea50e6ed56578e1d1d02548901b12fe6d3edbf110269a396955e285d487973" dependencies = [ "ahash", "backoff", @@ -907,6 +984,15 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "linked_hash_set" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47186c6da4d81ca383c7c47c1bfc80f4b95f4720514d860a5407aaf4233f9588" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lock_api" version = "0.4.7" @@ -1065,6 +1151,60 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c0b12cd9e3f9b35b52f6e0dac66866c519b26f424f4bbf96e3fe8bfbdc5229" +dependencies = [ + "async-trait", + "lazy_static", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", + "tokio", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "2.10.0" @@ -1200,8 +1340,8 @@ dependencies = [ [[package]] name = "product-config" -version = "0.3.1" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.3.1#40c93e5283beef100c9fecdb6368f1e1480db3e8" +version = "0.4.0" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.4.0#e1e5938b4f6120f85a088194e86d22433fdba731" dependencies = [ "fancy-regex", "java-properties", @@ -1439,7 +1579,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float", + "ordered-float 2.10.0", "serde", ] @@ -1553,8 +1693,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.15.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.15.0#c7c408d476c0b7ba06833c19e5c9d2aefa5875ae" +version = "0.17.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.17.0#ea47f679e2fe3f198691de7c1b742ac8ed8462c6" dependencies = [ "backoff", "chrono", @@ -1567,6 +1707,8 @@ dependencies = [ "k8s-openapi", "kube", "lazy_static", + "opentelemetry", + "opentelemetry-jaeger", "product-config", "rand", "regex", @@ -1578,6 +1720,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "tracing-opentelemetry", "tracing-subscriber", ] @@ -1715,6 +1858,28 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b82ca8f46f95b3ce96081fe3dd89160fdea970c254bb72925255d1b62aae692e" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float 1.1.1", + "threadpool", +] + [[package]] name = "time" version = "0.1.43" @@ -1791,6 +1956,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-openssl" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08f9ffb7809f1b20c1b398d92acf4cc719874b3b2b2d9ea2f09b4a80350878a" +dependencies = [ + "futures-util", + "openssl", + "openssl-sys", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.1" @@ -1909,6 +2097,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f9378e96a9361190ae297e7f3a8ff644aacd2897f244b1ff81f381669196fa6" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", +] + [[package]] name = "tracing-subscriber" version = "0.3.11" @@ -2027,6 +2228,60 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + [[package]] name = "winapi" version = "0.3.9" diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index 88ec0729..f75bac95 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/stackabletech/trino-operator" version = "0.3.2-nightly" [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.17.0" } semver = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 83adf01d..e5170beb 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -5,7 +5,7 @@ use crate::{authentication::Authentication, discovery::TrinoPodRef}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, Snafu}; -use stackable_operator::opa::OpaConfig; +use stackable_operator::commons::opa::OpaConfig; use stackable_operator::{ kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt}, product_config_utils::{ConfigError, Configuration}, diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 9ec4234c..195bf3b0 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/stackabletech/trino-operator" version = "0.3.2-nightly" [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.17.0" } stackable-trino-crd = { path = "../crd" } anyhow = "1.0" @@ -27,5 +27,5 @@ tracing = "0.1" [build-dependencies] built = { version = "0.5", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.17.0" } stackable-trino-crd = { path = "../crd" } diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d5555172..d0185282 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -3,6 +3,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ConfigMapBuilder, ContainerBuilder, ObjectMetaBuilder, PodBuilder}, client::Client, + commons::opa::OpaApiVersion, k8s_openapi::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, @@ -24,7 +25,6 @@ use stackable_operator::{ }, labels::{role_group_selector_labels, role_selector_labels}, logging::controller::ReconcilerError, - opa::OpaApiVersion, product_config, product_config::{types::PropertyNameKind, ProductConfigManager}, product_config_utils::{ @@ -779,24 +779,6 @@ fn container_trino_args( rw_conf = RW_CONFIG_DIR_NAME, hive_properties = HIVE_PROPERTIES )]) } - // opa required? - if trino.spec.opa_config_map_name.is_some() { - let opa_package_name = match trino.spec.authorization.as_ref() { - Some(auth) => auth.package.clone(), - None => { - warn!("No package specified in 'spec.authorization'. Defaulting to 'trino'."); - "trino".to_string() - } - }; - - args.extend(vec![ - format!( "echo Writing OPA connect string \"opa.policy.uri=${{OPA}}v1/data/{package_name}\" to {rw_conf}/{access_control}", - package_name = opa_package_name, rw_conf = RW_CONFIG_DIR_NAME, access_control = ACCESS_CONTROL_PROPERTIES - ), - format!( "echo \"opa.policy.uri=${{OPA}}v1/data/{package_name}/\" >> {rw_conf}/{access_control}", - package_name = opa_package_name, rw_conf = RW_CONFIG_DIR_NAME, access_control = ACCESS_CONTROL_PROPERTIES - )]) - } // start command args.push(format!( diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 4fcb9634..a1448dd4 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -3,6 +3,7 @@ mod controller; use clap::Parser; use futures::stream::StreamExt; use stackable_operator::cli::ProductOperatorRun; +use stackable_operator::logging::TracingTarget; use stackable_operator::{ cli::Command, k8s_openapi::api::{ @@ -16,7 +17,7 @@ use stackable_operator::{ }, logging::controller::report_controller_reconciled, }; -use stackable_trino_crd::TrinoCluster; +use stackable_trino_crd::{TrinoCluster, APP_NAME}; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); @@ -31,7 +32,11 @@ struct Opts { #[tokio::main] async fn main() -> anyhow::Result<()> { - stackable_operator::logging::initialize_logging("TRINO_OPERATOR_LOG"); + stackable_operator::logging::initialize_logging( + "TRINO_OPERATOR_LOG", + APP_NAME, + TracingTarget::None, + ); let opts = Opts::parse(); match opts.cmd { @@ -39,6 +44,7 @@ async fn main() -> anyhow::Result<()> { Command::Run(ProductOperatorRun { product_config, watch_namespace, + tracing_target: _, }) => { stackable_operator::utils::print_startup_string( built_info::PKG_DESCRIPTION, From 8cee996311aaaf3473dd89c93cecddf620d99d60 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 22 Apr 2022 16:40:44 +0200 Subject: [PATCH 12/16] use opa version 0.37.2 --- ...cluster-authentication-opa-authorization.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/simple-trino-cluster-authentication-opa-authorization.yaml b/examples/simple-trino-cluster-authentication-opa-authorization.yaml index 0279b18e..1d46dadf 100644 --- a/examples/simple-trino-cluster-authentication-opa-authorization.yaml +++ b/examples/simple-trino-cluster-authentication-opa-authorization.yaml @@ -30,7 +30,7 @@ kind: OpaCluster metadata: name: simple-opa spec: - version: "0.27.1" + version: "0.37.2" servers: roleGroups: default: @@ -60,28 +60,28 @@ metadata: data: trino.rego: | package trino - + import future.keywords.in - + default allow = false - + allow { is_admin } - + allow { is_bob can_be_accessed_by_bob } - + is_admin() { input.context.identity.user == "admin" } - + is_bob() { input.context.identity.user == "bob" } - + can_be_accessed_by_bob() { input.action.operation in ["ImpersonateUser", "FilterCatalogs", "AccessCatalog", "ExecuteQuery"] } From e9d5f89f6b813df4083f7319ba10e69aa0f4d40f Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 25 Apr 2022 16:53:15 +0200 Subject: [PATCH 13/16] added secret-operator requirement to docs --- docs/modules/ROOT/pages/installation.adoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index d71ca97a..4194d0e5 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -17,6 +17,12 @@ installation method. First ensure that you have installed the Stackable Operator $ helm repo add stackable https://repo.stackable.tech/repository/helm-stable/ ---- +We also need some addition components of the Stackable Data Platform. The secret-operator if you are running with authentication. +[source,bash] +---- +$ helm install secret-operator stackable/secret-operator +---- + Then install the Stackable Operator for Trino [source,bash] ---- From f87ad9fa285058e56e72d0ead62c59295969e0bc Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 25 Apr 2022 17:44:07 +0200 Subject: [PATCH 14/16] removed all old references to 362 --- docs/modules/ROOT/pages/usage.adoc | 10 +++++----- rust/operator-binary/src/controller.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/modules/ROOT/pages/usage.adoc b/docs/modules/ROOT/pages/usage.adoc index 9b054cc6..5dcb004b 100644 --- a/docs/modules/ROOT/pages/usage.adoc +++ b/docs/modules/ROOT/pages/usage.adoc @@ -104,7 +104,7 @@ kubectl apply -f /etc/stackable/trino-operator/crd/trinocluster.crd.yaml ==== Insecure for testing: -Create an insecure single node Trino (v362) cluster for testing. You will access the UI/CLI via http and no user / password or authorization is required. Please adapt the `s3` settings with your credentials (check `examples/simple-trino-cluster.yaml` for an example setting up Hive and Trino): +Create an insecure single node Trino (v377) cluster for testing. You will access the UI/CLI via http and no user / password or authorization is required. Please adapt the `s3` settings with your credentials (check `examples/simple-trino-cluster.yaml` for an example setting up Hive and Trino): [source,yaml] ---- @@ -142,12 +142,12 @@ spec: To access the CLI please execute: [source] ---- -./trino-cli-362-executable.jar --debug --server http://: --user=admin +./trino-cli-377-executable.jar --debug --server http://: --user=admin ---- ==== Secure (https) for production: -Create a secure single node Trino (v362) cluster. This will disable the UI access via http and requires username and password from the secret above. Please adapt the `s3` settings with your credentials (check `examples/simple-trino-cluster-authentication-opa-authorization.yaml` for a full example setting up Hive, OPA, Secrets and Trino): +Create a secure single node Trino (v377) cluster. This will disable the UI access via http and requires username and password from the secret above. Please adapt the `s3` settings with your credentials (check `examples/simple-trino-cluster-authentication-opa-authorization.yaml` for a full example setting up Hive, OPA, Secrets and Trino): [source,yaml] ---- @@ -191,10 +191,10 @@ spec: config: {} ---- -To access the CLI please execute: +To access the CLI please download the https://repo1.maven.org/maven2/io/trino/trino-cli/377/trino-cli-377-executable.jar[Trino CLI] and execute: [source] ---- -./trino-cli-362-executable.jar --debug --server https://: --user=admin --password --insecure +./trino-cli-377-executable.jar --debug --server https://: --user=admin --password --insecure ---- If you use self signed certificates, you also need the `--insecure` flag above which can be omitted otherwise. diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index d0185282..cb6a5d5b 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -656,7 +656,7 @@ fn build_rolegroup_service( }) } -/// Returns our semver representation for product config e.g. 0.0.362 +/// Returns our semver representation for product config e.g. 0.0.377 pub fn trino_version(trino: &TrinoCluster) -> Result<&str> { trino .spec @@ -665,7 +665,7 @@ pub fn trino_version(trino: &TrinoCluster) -> Result<&str> { .context(ObjectHasNoVersionSnafu) } -/// Returns the "real" Trino version for docker images e.g. 362 +/// Returns the "real" Trino version for docker images e.g. 377 pub fn trino_version_trim(trino: &TrinoCluster) -> Result<&str> { let spec: &TrinoClusterSpec = &trino.spec; spec.version From d396cc23cd7d34067a8859084d318ed901f4a1b6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 25 Apr 2022 17:55:22 +0200 Subject: [PATCH 15/16] Adapted changelog --- CHANGELOG.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c894107..2e9fd86a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,27 +12,27 @@ All notable changes to this project will be documented in this file. ### Changed -- `operator-rs` `0.10.0` -> `0.17.0` ([#149], [#157], [#183], [#xxx]). +- `operator-rs` `0.10.0` -> `0.17.0` ([#149], [#157], [#183], [#193]). - BREAKING: The operator now writes a `ConfigMap` for Rego rules instead of the custom resource for the obsolete regorule-operator. This means that the rego rule operator is not required anymore for authorization and opa-operator tag >= `0.9.0` ([#157]). - BREAKING: `OpaConfigMapName` in CRD to `opa` using the `OpaConfig` from operator-rs ([#186]). -- Trino version to 377 ([#xxx]). -- Opa rego example adapted to the new `trino-opa-authorizer` ([#xxx]). +- Trino version to 377 ([#193]). +- Opa rego example adapted to the new `trino-opa-authorizer` ([#193]). ### Removed - `stackable-regorule-crd` dependency ([#157]). - BREAKING: `nodeEnvironment` from CRD. Will default to the `metadata.name` field (can be overriden) ([#183]). - BREAKING: Removed `authorization` module from CRD and code and provided the opa bundle via `ConfigMap` directly instead of generating it ([#186]). -- Possibly BREAKING: Removed `query.max-total-memory-per-node` config parameter ([#xxx]). +- Possibly BREAKING: Removed `query.max-total-memory-per-node` config parameter ([#193]). [#149]: https://github.com/stackabletech/trino-operator/pull/149 [#157]: https://github.com/stackabletech/trino-operator/pull/157 [#183]: https://github.com/stackabletech/trino-operator/pull/183 [#186]: https://github.com/stackabletech/trino-operator/pull/186 -[#xxx]: https://github.com/stackabletech/trino-operator/pull/xxx +[#193]: https://github.com/stackabletech/trino-operator/pull/193 ## [0.3.1] - 2022-02-17 From 59bbedb496948061a341befb4d04c31afb108727 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 26 Apr 2022 09:11:26 +0200 Subject: [PATCH 16/16] adapted to PR review --- rust/operator-binary/src/main.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index a1448dd4..b499598d 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -3,7 +3,6 @@ mod controller; use clap::Parser; use futures::stream::StreamExt; use stackable_operator::cli::ProductOperatorRun; -use stackable_operator::logging::TracingTarget; use stackable_operator::{ cli::Command, k8s_openapi::api::{ @@ -32,20 +31,19 @@ struct Opts { #[tokio::main] async fn main() -> anyhow::Result<()> { - stackable_operator::logging::initialize_logging( - "TRINO_OPERATOR_LOG", - APP_NAME, - TracingTarget::None, - ); - let opts = Opts::parse(); match opts.cmd { Command::Crd => println!("{}", serde_yaml::to_string(&TrinoCluster::crd())?,), Command::Run(ProductOperatorRun { product_config, watch_namespace, - tracing_target: _, + tracing_target, }) => { + stackable_operator::logging::initialize_logging( + "TRINO_OPERATOR_LOG", + APP_NAME, + tracing_target, + ); stackable_operator::utils::print_startup_string( built_info::PKG_DESCRIPTION, built_info::PKG_VERSION,