Provide example on how to bootstrap a self-managed cluster

Signed-off-by: Andrea Mazzotti <[email protected]>
rancher · Jan 16, 2025 · df56393 · df56393
1 parent 0450b5f
commit df56393
Show file tree

Hide file tree

Showing 2 changed files with 344 additions and 0 deletions.
diff --git a/scripts/bootstrap-cluster.sh b/scripts/bootstrap-cluster.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+
+# Copyright © 2024 - 2025 SUSE LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is an example on how to provision a self-managed RKE2 Rancher cluster.
+#
+# One bootstrap cluster is provisioned using kind, and initialized with the required CAPI providers.
+# Using the bootstrap cluster, a downstream RKE2+CAPD cluster is provisioned.
+# Note that CAPD is used for demo purposes.
+# Rancher, Turtles, the required CAPI providers and all dependencies are installed on the downstream cluster.
+# Finally the downstream cluster is moved to itself, for self-management.
+# For more information see: https://cluster-api.sigs.k8s.io/clusterctl/commands/move
+
+set -e
+
+CAPI_VERSION="v1.9.3"
+CAPD_VERSION="v1.9.3"
+CAPRKE2_VERSION="v0.10.0"
+
+# Initialize a bootstrap CAPI cluster
+cat << EOF | kind create cluster --config -
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: bootstrap-capi-management
+nodes:
+- role: control-plane
+  image: kindest/node:v1.31.4
+  extraMounts:
+    - hostPath: /var/run/docker.sock
+      containerPath: /var/run/docker.sock
+EOF
+
+# Create a temporary clusterctl config
+CONFIG_DIR="/tmp/cluster-api"
+CONFIG_FILE="$CONFIG_DIR/clusterctl.yaml"
+mkdir -p $CONFIG_DIR
+cat << EOF > $CONFIG_FILE
+providers:
+- name: "rke2"
+  url: "https://github.com/rancher/cluster-api-provider-rke2/releases/$CAPRKE2_VERSION/bootstrap-components.yaml"
+  type: "BootstrapProvider"
+- name: "rke2"
+  url: "https://github.com/rancher/cluster-api-provider-rke2/releases/$CAPRKE2_VERSION/control-plane-components.yaml"
+  type: "ControlPlaneProvider"
+EOF
+
+# Level 5 is highest for debugging
+export CLUSTERCTL_LOG_LEVEL=4
+
+# Install core, docker, and RKE2 providers
+clusterctl init --config "$CONFIG_FILE" \
+                --core "cluster-api:$CAPI_VERSION" \
+                --bootstrap "rke2:$CAPRKE2_VERSION" --control-plane "rke2:$CAPRKE2_VERSION" \
+                --infrastructure "docker:$CAPD_VERSION"
+
+# Wait for all providers to be ready
+kubectl -n rke2-control-plane-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/rke2-control-plane-controller-manager
+kubectl -n rke2-bootstrap-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/rke2-bootstrap-controller-manager
+kubectl -n capd-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/capd-controller-manager
+kubectl -n capi-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/capi-controller-manager
+
+# Bootstrap a downstream RKE2 cluster
+cd "$(dirname "$0")"
+kubectl apply -f ./bootstrap-cluster.yaml
+kubectl wait --for=condition=Ready clusters/rancher --timeout=300s
+
+# Fetch the downstream kubeconfig and switch to it
+KUBECONFIG_FILE="/tmp/cluster-api/rancher"
+clusterctl get kubeconfig rancher > "$KUBECONFIG_FILE"
+chmod 600 "$KUBECONFIG_FILE"
+export KUBECONFIG="$KUBECONFIG_FILE"
+
+# Install Rancher, Turtles, and dependencies on the downstream cluster
+helm repo add jetstack https://charts.jetstack.io
+helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
+helm repo add turtles https://rancher.github.io/turtles
+helm repo update
+
+helm install cert-manager jetstack/cert-manager \
+    --namespace cert-manager --create-namespace \
+    --set crds.enabled=true \
+    --wait
+
+# Note kubectl wait --for=create requires kubectl >= v1.31.x
+kubectl -n kube-system wait --for=create daemonsets/rke2-ingress-nginx-controller
+kubectl -n kube-system wait --for=jsonpath='{.status.numberReady}'=2 daemonsets/rke2-ingress-nginx-controller --timeout=300s
+helm install rancher rancher-latest/rancher \
+    --namespace cattle-system --create-namespace \
+    --set bootstrapPassword=rancheradmin \
+    --set replicas=1 \
+    --set hostname="my.rancher.example.com" \
+    --set features=embedded-cluster-api=false \
+    --wait
+
+helm install rancher-turtles turtles/rancher-turtles \
+    -n rancher-turtles-system  --create-namespace \
+    --dependency-update \
+    --set cluster-api-operator.cluster-api.version="$CAPI_VERSION" \
+    --set cluster-api-operator.cluster-api.rke2.version="$CAPRKE2_VERSION" \
+    --wait --timeout 180s
+
+# Install CAPD provider on downstream cluster
+cat << EOF | kubectl apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: capd-system
+---
+apiVersion: turtles-capi.cattle.io/v1alpha1
+kind: CAPIProvider
+metadata:
+  name: docker
+  namespace: capd-system
+spec:
+  type: infrastructure
+  name: docker
+  version: $CAPD_VERSION
+EOF
+
+# Wait for all providers to be ready on downstream cluster
+kubectl -n rke2-control-plane-system wait --for=create deployments/rke2-control-plane-controller-manager --timeout=300s
+kubectl -n rke2-control-plane-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/rke2-control-plane-controller-manager
+kubectl -n rke2-bootstrap-system wait --for=create deployments/rke2-bootstrap-controller-manager --timeout=300s
+kubectl -n rke2-bootstrap-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/rke2-bootstrap-controller-manager
+kubectl -n capd-system wait --for=create deployments/capd-controller-manager --timeout=300s
+kubectl -n capd-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/capd-controller-manager
+kubectl -n capi-system wait --for=create deployments/capi-controller-manager --timeout=300s
+kubectl -n capi-system wait --for=jsonpath='{.status.readyReplicas}'=1 deployments/capi-controller-manager
+
+# Pivot the CAPI cluster to the newly provisioned downstream cluster
+clusterctl move --to-kubeconfig="$KUBECONFIG_FILE"
+
+# Delete the no longer necessary bootstrap cluster
+echo "Rancher cluster has been provisioned and moved. The bootstrap-capi-management cluster can now be deleted."
diff --git a/scripts/bootstrap-cluster.yaml b/scripts/bootstrap-cluster.yaml
@@ -0,0 +1,198 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster 
+metadata:
+  namespace: default
+  name: rancher 
+spec:
+  clusterNetwork:
+    pods:
+      cidrBlocks:
+      - 10.45.0.0/16
+    services:
+      cidrBlocks:
+      - 10.46.0.0/16
+    serviceDomain: cluster.local
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: RKE2ControlPlane
+    name: rancher-control-plane
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerCluster
+    name: rancher
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerCluster
+metadata:
+  name: rancher
+  namespace: default
+spec:
+  loadBalancer:
+    customHAProxyConfigTemplateRef:
+      name: rancher-lb-config
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: RKE2ControlPlane
+metadata:
+  name: rancher-control-plane
+  namespace: default
+spec: 
+  replicas: 1
+  version:  v1.31.4+rke2r1
+  registrationMethod: control-plane-endpoint
+  rolloutStrategy:
+    type: "RollingUpdate"
+    rollingUpdate:
+      maxSurge: 1
+  agentConfig: {}
+  serverConfig:
+    disableComponents:
+      kubernetesComponents:
+      - cloudController
+    kubeAPIServer:
+      extraArgs:
+      - --anonymous-auth=true
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+    kind: DockerMachineTemplate
+    name: controlplane
+  nodeDrainTimeout: 2m
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: worker-md-0
+  namespace: default
+spec:
+  clusterName: rancher
+  replicas: 1
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: rancher
+  template:
+    spec:
+      version: v1.31.4+rke2r1
+      clusterName: rancher
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: RKE2ConfigTemplate
+          name: rancher-agent
+          namespace: default
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: DockerMachineTemplate
+        name: worker
+        namespace: default
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: controlplane
+  namespace: default
+  labels:
+    clusterctl.cluster.x-k8s.io/move: "true"
+spec:
+  template:
+    spec:
+      customImage: kindest/node:v1.31.4
+      bootstrapTimeout: 15m
+      extraMounts:
+        - containerPath: /var/run/docker.sock
+          hostPath: /var/run/docker.sock
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: DockerMachineTemplate
+metadata:
+  name: worker
+  namespace: default
+  labels:
+    clusterctl.cluster.x-k8s.io/move: "true"
+spec:
+  template:
+    spec:
+      customImage: kindest/node:v1.31.4
+      bootstrapTimeout: 15m
+      extraMounts:
+        - containerPath: /var/run/docker.sock
+          hostPath: /var/run/docker.sock
+---
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: RKE2ConfigTemplate
+metadata:
+  namespace: default
+  name: rancher-agent
+  labels:
+    clusterctl.cluster.x-k8s.io/move: "true"
+spec: 
+  template:
+    spec:
+      agentConfig: {}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: rancher-lb-config
+  namespace: default
+  labels:
+    clusterctl.cluster.x-k8s.io/move: "true"
+data:
+  value: |-
+    # generated by kind
+    global
+      log /dev/log local0
+      log /dev/log local1 notice
+      daemon
+      # limit memory usage to approximately 18 MB
+      # (see https://github.com/kubernetes-sigs/kind/pull/3115)
+      maxconn 100000
+
+    resolvers docker
+      nameserver dns 127.0.0.11:53
+
+    defaults
+      log global
+      mode tcp
+      option dontlognull
+      # TODO: tune these
+      timeout connect 5000
+      timeout client 50000
+      timeout server 50000
+      # allow to boot despite dns don't resolve backends
+      default-server init-addr none
+
+    frontend stats
+      mode http
+      bind *:8404
+      stats enable
+      stats uri /stats
+      stats refresh 1s
+      stats admin if TRUE
+
+    frontend control-plane
+      bind *:{{ .FrontendControlPlanePort }}
+      {{ if .IPv6 -}}
+      bind :::{{ .FrontendControlPlanePort }};
+      {{- end }}
+      default_backend kube-apiservers
+
+    backend kube-apiservers
+      option httpchk GET /healthz
+
+      {{range $server, $backend := .BackendServers}}
+      server {{ $server }} {{ JoinHostPort $backend.Address $.BackendControlPlanePort }} check check-ssl verify none resolvers docker resolve-prefer {{ if $.IPv6 -}} ipv6 {{- else -}} ipv4 {{- end }}
+      {{- end}}
+
+    frontend rke2-join
+      bind *:9345
+      {{ if .IPv6 -}}
+      bind :::9345;
+      {{- end }}
+      default_backend rke2-servers
+
+    backend rke2-servers
+      option httpchk GET /v1-rke2/readyz
+      http-check expect status 403
+      {{range $server, $backend := .BackendServers}}
+      server {{ $server }} {{ $backend.Address }}:9345 check check-ssl verify none
+      {{- end}}