Generate kube config based on plugin RBAC (#1020)

Generate custom kube config and distribute it to plugins

cmd/botkube/main.go

@@ -211,6 +211,7 @@ func run(ctx context.Context) error {
 			CommandGuard:      cmdGuard,
 			PluginManager:     pluginManager,
 			BotKubeVersion:    botkubeVersion,
+			RestCfg:           kubeConfig,
 			AuditReporter:     auditReporter,
 		},
 	)
@@ -380,7 +381,7 @@ func run(ctx context.Context) error {
 
 	actionProvider := action.NewProvider(logger.WithField(componentLogFieldKey, "Action Provider"), conf.Actions, executorFactory)
 
-	sourcePluginDispatcher := source.NewDispatcher(logger, bots, sinkNotifiers, pluginManager, actionProvider, reporter, auditReporter)
+	sourcePluginDispatcher := source.NewDispatcher(logger, bots, sinkNotifiers, pluginManager, actionProvider, reporter, auditReporter, kubeConfig)
 	scheduler := source.NewScheduler(logger, conf, sourcePluginDispatcher)
 	err = scheduler.Start(ctx)
 	if err != nil {

cmd/source/cm-watcher/main.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"os"
 
 	"github.com/MakeNowJust/heredoc"
 	"github.com/hashicorp/go-plugin"
@@ -76,13 +75,13 @@ func (CMWatcher) Stream(ctx context.Context, in source.StreamInput) (source.Stre
 		Output: make(chan []byte),
 	}
 
-	go listenEvents(ctx, cfg.ConfigMap, out.Output)
+	go listenEvents(ctx, in.Context.KubeConfig, cfg.ConfigMap, out.Output)
 
 	return out, nil
 }
 
-func listenEvents(ctx context.Context, obj Object, sink chan<- []byte) {
-	config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
+func listenEvents(ctx context.Context, kubeConfig []byte, obj Object, sink chan<- []byte) {
+	config, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig)
 	exitOnError(err)
 	clientset, err := kubernetes.NewForConfig(config)
 	exitOnError(err)

go.mod

@@ -57,6 +57,7 @@ require (
 	k8s.io/kubectl v0.25.4
 	k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2
 	sigs.k8s.io/controller-runtime v0.13.1
+	sigs.k8s.io/yaml v1.3.0
 )
 
 require (
@@ -204,7 +205,6 @@ require (
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 go 1.19

helm/botkube/e2e-test-values.yaml

@@ -1,6 +1,17 @@
 analytics:
   disable: true
 
+rbac:
+  create: true
+  rules:
+    - apiGroups: [ "*" ]
+      resources: [ "*" ]
+      verbs: [ "get", "watch", "list" ] # defaults
+    - apiGroups: [ "" ]
+      resources: [ "services" ]
+      verbs: [ "patch" ] # needed for label action
+  staticGroupName: &static-group-name "botkube-plugins-default"
+
 communications:
   'default-group':
     slack: # Configuration for the Slack app with RTM support
@@ -65,6 +76,25 @@ sources:
   'k8s-events':
     displayName: "K8s recommendations"
     'botkube/kubernetes':
+      context: &defaultPluginContext
+        defaultNamespace: "default"
+        # -- RBAC configuration for this plugin.
+        rbac:
+          # -- Static impersonation for a given username and groups.
+          group:
+            type: Static
+            # -- Prefix that will be applied to .static.value[*].
+            prefix: ""
+            static:
+              # -- Name of group.rbac.authorization.k8s.io the plugin will be bound to.
+              values: [*static-group-name] # "botkube-plugins-read-only" is the default
+          user:
+            type: Static
+            # -- Prefix that will be applied to .static.value[*].
+            prefix: ""
+            static:
+              # -- Name of user.rbac.authorization.k8s.io the plugin will be bound to.
+              value: *static-group-name
       enabled: true
       config:
         log:
@@ -88,6 +118,7 @@ sources:
   'k8s-annotated-cm-delete':
     displayName: "K8s ConfigMap delete events"
     'botkube/kubernetes':
+      context: *defaultPluginContext
       enabled: true
       config:
         log:
@@ -105,6 +136,7 @@ sources:
 
   'k8s-pod-create-events':
     'botkube/kubernetes':
+      context: *defaultPluginContext
       enabled: true
       config:
         log:
@@ -121,6 +153,7 @@ sources:
   'k8s-service-create-event-for-action-only':
     displayName: "K8s Service creation, used only by action"
     'botkube/kubernetes':
+      context: *defaultPluginContext
       enabled: true
       config:
         namespaces:
@@ -135,6 +168,7 @@ sources:
   'k8s-updates':
     displayName: "K8s ConfigMaps updates"
     'botkube/kubernetes':
+      context: *defaultPluginContext
       enabled: true
       config:
         log:
@@ -159,6 +193,7 @@ sources:
   'plugin-based':
     displayName: "K8s ConfigMaps changes"
     botkube/cm-watcher:
+      context: *defaultPluginContext
       enabled: true
       config:
         configMap:
@@ -229,6 +264,7 @@ executors:
         changeResponseToUpperCase: true
 
     botkube/helm:
+      context: *defaultPluginContext
       enabled: true
 
 plugins:
@@ -288,17 +324,6 @@ settings:
 extraAnnotations:
   botkube.io/disable: "true"
 
-rbac:
-  create: true
-  rules:
-    - apiGroups: [ "*" ]
-      resources: [ "*" ]
-      verbs: [ "get", "watch", "list" ] # defaults
-    - apiGroups: [ "" ]
-      resources: [ "services" ]
-      verbs: [ "patch" ] # needed for label action
-
-
 extraEnv:
   - name: LOG_LEVEL_SOURCE_BOTKUBE_KUBERNETES
     value: debug

helm/botkube/templates/clusterrole.yaml

@@ -21,6 +21,14 @@ rules:
   - apiGroups: [""]
     resources: ["namespaces"]
     verbs: ["get"]
+  - apiGroups:
+    - ""
+    resources:
+    - users
+    - groups
+    - serviceaccounts
+    verbs:
+    - impersonate
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

helm/botkube/values.yaml

@@ -116,6 +116,25 @@ sources:
     # -- Describes Kubernetes source configuration.
     # @default -- See the `values.yaml` file for full object.
     botkube/kubernetes:
+      context: &defaultPluginContext
+        defaultNamespace: "default"
+        # -- RBAC configuration for this plugin.
+        rbac:
+          # -- Static impersonation for a given username and groups.
+          group:
+            type: Static
+            # -- Prefix that will be applied to .static.value[*].
+            prefix: ""
+            static:
+              # -- Name of group.rbac.authorization.k8s.io the plugin will be bound to.
+              values: [*static-group-name] # "botkube-plugins-read-only" is the default
+          user:
+            type: Static
+            # -- Prefix that will be applied to .static.value[*].
+            prefix: ""
+            static:
+              # -- Name of user.rbac.authorization.k8s.io the plugin will be bound to.
+              value: *static-group-name
       enabled: true
       config:
         namespaces:
@@ -141,6 +160,7 @@ sources:
     # -- Describes Kubernetes source configuration.
     # @default -- See the `values.yaml` file for full object.
     botkube/kubernetes:
+      context: *defaultPluginContext
       enabled: true
       config:
         # -- Filter settings for various sources.
@@ -313,6 +333,7 @@ sources:
     # -- Describes Kubernetes source configuration.
     # @default -- See the `values.yaml` file for full object.
     botkube/kubernetes:
+      context: *defaultPluginContext
       enabled: true
       config:
         # -- Describes namespaces for every Kubernetes resources you want to watch or exclude.
@@ -352,6 +373,7 @@ sources:
     # -- Describes Kubernetes source configuration.
     # @default -- See the `values.yaml` file for full object.
     botkube/kubernetes:
+      context: *defaultPluginContext
       enabled: true
       config:
         # -- Describes namespaces for every Kubernetes resources you want to watch or exclude.
@@ -382,6 +404,7 @@ sources:
     # -- Describes Kubernetes source configuration.
     # @default -- See the `values.yaml` file for full object.
     botkube/kubernetes:
+      context: *defaultPluginContext
       enabled: true
       config:
         # -- Describes namespaces for every Kubernetes resources you want to watch or exclude.
@@ -476,24 +499,7 @@ executors:
         helmConfigDir: "/tmp/helm/"
         # -- Location for storing cached files. Must be under the Helm config directory.
         helmCacheDir: "/tmp/helm/.cache"
-      context: &defaultExecutorContext
-        # -- RBAC configuration for this plugin.
-        rbac:
-          # -- Static impersonation for a given username and groups.
-          group:
-            type: Static
-            # -- Prefix that will be applied to .static.value[*].
-            prefix: ""
-            static:
-              # -- Name of group.rbac.authorization.k8s.io the plugin will be bound to.
-              values: [*static-group-name] # "botkube-plugins-read-only" is the default
-          user:
-            type: Static
-            # -- Prefix that will be applied to .static.value[*].
-            prefix: ""
-            static:
-              # -- Name of user.rbac.authorization.k8s.io the plugin will be bound to.
-              value: "default"
+      context: *defaultPluginContext
 
     ## Kubectl executor configuration
     ## Plugin name syntax: <repo>/<plugin>[@<version>]. If version is not provided, the latest version from repository is used.
@@ -518,7 +524,7 @@ executors:
       #      verbs: [ "api-resources", "api-versions", "cluster-info", "describe", "explain", "get", "logs", "top" ]
       #      # Configures which K8s resource are displayed in resources dropdown.
       #      resources: [ "deployments", "pods", "namespaces", "daemonsets", "statefulsets", "storageclasses", "nodes", "configmaps", "services", "ingresses", "replicasets", "secrets", "cronjobs", "jobs" ]
-      context: *defaultExecutorContext
+      context: *defaultPluginContext
 
 # -- Custom aliases for given commands.
 # The aliases are replaced with the underlying command before executing it.

internal/executor/helm/executor.go

@@ -3,6 +3,7 @@ package helm
 import (
 	"context"
 	"fmt"
+	"os"
 
 	"github.com/MakeNowJust/heredoc"
 	"github.com/alexflint/go-arg"
@@ -104,37 +105,47 @@ func (e *Executor) Execute(ctx context.Context, in executor.ExecuteInput) (execu
 		in.Command = fmt.Sprintf("%s -n %s", in.Command, cfg.DefaultNamespace)
 	}
 
+	kubeConfigPath, deleteFn, err := pluginx.PersistKubeConfig(ctx, in.Context.KubeConfig)
+	if err != nil {
+		return executor.ExecuteOutput{}, fmt.Errorf("while writing kubeConfig file: %w", err)
+	}
+	defer func() {
+		if deleteErr := deleteFn(ctx); deleteErr != nil {
+			fmt.Fprintf(os.Stderr, "failed to delete cube config file %s: %v", kubeConfigPath, deleteErr)
+		}
+	}()
+
 	switch {
 	case helmCmd.Install != nil:
-		return e.handleHelmCommand(ctx, helmCmd.Install, cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.Install, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.UninstallCommandAliases.Get() != nil:
-		return e.handleHelmCommand(ctx, helmCmd.UninstallCommandAliases.Get(), cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.UninstallCommandAliases.Get(), cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.ListCommandAliases.Get() != nil:
-		return e.handleHelmCommand(ctx, helmCmd.ListCommandAliases.Get(), cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.ListCommandAliases.Get(), cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.Version != nil:
-		return e.handleHelmCommand(ctx, helmCmd.Version, cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.Version, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.Status != nil:
-		return e.handleHelmCommand(ctx, helmCmd.Status, cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.Status, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.Test != nil:
-		return e.handleHelmCommand(ctx, helmCmd.Test, cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.Test, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.Rollback != nil:
-		return e.handleHelmCommand(ctx, helmCmd.Rollback, cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.Rollback, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.Upgrade != nil:
-		return e.handleHelmCommand(ctx, helmCmd.Upgrade, cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.Upgrade, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.HistoryCommandAliases.Get() != nil:
-		return e.handleHelmCommand(ctx, helmCmd.HistoryCommandAliases.Get(), cfg, wasHelpRequested, in.Command)
+		return e.handleHelmCommand(ctx, helmCmd.HistoryCommandAliases.Get(), cfg, wasHelpRequested, in.Command, kubeConfigPath)
 	case helmCmd.Get != nil:
 		switch {
 		case helmCmd.Get.All != nil:
-			return e.handleHelmCommand(ctx, helmCmd.Get.All, cfg, wasHelpRequested, in.Command)
+			return e.handleHelmCommand(ctx, helmCmd.Get.All, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 		case helmCmd.Get.Hooks != nil:
-			return e.handleHelmCommand(ctx, helmCmd.Get.Hooks, cfg, wasHelpRequested, in.Command)
+			return e.handleHelmCommand(ctx, helmCmd.Get.Hooks, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 		case helmCmd.Get.Manifest != nil:
-			return e.handleHelmCommand(ctx, helmCmd.Get.Manifest, cfg, wasHelpRequested, in.Command)
+			return e.handleHelmCommand(ctx, helmCmd.Get.Manifest, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 		case helmCmd.Get.Notes != nil:
-			return e.handleHelmCommand(ctx, helmCmd.Get.Notes, cfg, wasHelpRequested, in.Command)
+			return e.handleHelmCommand(ctx, helmCmd.Get.Notes, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 		case helmCmd.Get.Values != nil:
-			return e.handleHelmCommand(ctx, helmCmd.Get.Values, cfg, wasHelpRequested, in.Command)
+			return e.handleHelmCommand(ctx, helmCmd.Get.Values, cfg, wasHelpRequested, in.Command, kubeConfigPath)
 		default:
 			return executor.ExecuteOutput{
 				Data: helmCmd.Get.Help(),
@@ -153,7 +164,7 @@ func (*Executor) Help(context.Context) (api.Message, error) {
 }
 
 // handleHelmList construct a Helm CLI command and run it.
-func (e *Executor) handleHelmCommand(ctx context.Context, cmd command, cfg Config, wasHelpRequested bool, rawCmd string) (executor.ExecuteOutput, error) {
+func (e *Executor) handleHelmCommand(ctx context.Context, cmd command, cfg Config, wasHelpRequested bool, rawCmd, kubeConfig string) (executor.ExecuteOutput, error) {
 	if wasHelpRequested {
 		return executor.ExecuteOutput{
 			Data: cmd.Help(),
@@ -169,6 +180,7 @@ func (e *Executor) handleHelmCommand(ctx context.Context, cmd command, cfg Confi
 		"HELM_DRIVER":      cfg.HelmDriver,
 		"HELM_CACHE_HOME":  cfg.HelmCacheDir,
 		"HELM_CONFIG_HOME": cfg.HelmConfigDir,
+		"KUBECONFIG":       kubeConfig,
 	}
 
 	out, err := e.executeCommandWithEnvs(ctx, rawCmd, envs)