introduce the system-upgrade-controller

images/01-k3s/Dockerfile

@@ -4,7 +4,7 @@ FROM ${REPO}/k3os-base:${TAG}
 
 ARG ARCH
 ENV ARCH ${ARCH}
-ENV VERSION v1.0.0
+ENV VERSION v1.17.0+k3s.1
 ADD https://raw.githubusercontent.com/rancher/k3s/${VERSION}/install.sh /output/install.sh
 ENV INSTALL_K3S_VERSION=${VERSION} \
     INSTALL_K3S_SKIP_START=true \

images/02-rootfs/Dockerfile

@@ -31,9 +31,8 @@ RUN cd /usr/src/image && \
    ;done && \
    rmdir usr
 
-# Fix ldd and coreutils links
+# Fix coreutils links
 RUN cd /usr/src/image/bin \
- && ln -sf $(realpath $(which ldd)) ldd \
  && find -xtype l -ilname ../usr/bin/coreutils -exec ln -sf coreutils {} \;
 
 # Fix sudo

overlay/libexec/k3os/boot

@@ -130,6 +130,12 @@ setup_mounts()
     done
 }
 
+setup_manifests()
+{
+    mkdir -p /var/lib/rancher/k3s/server/manifests
+    rsync -a --exclude='*.example' /usr/share/rancher/k3s/server/manifests/ /var/lib/rancher/k3s/server/manifests/
+}
+
 do_grow_live()
 {
     parted $1 resizepart $2 yes 100%
@@ -173,4 +179,5 @@ setup_ttys
 setup_sudoers
 setup_services
 setup_config
+setup_manifests
 cleanup

overlay/share/rancher/k3s/server/manifests/system-upgrade-controller.yaml

@@ -0,0 +1,88 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: k3os-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: k3os-upgrade
+  namespace: k3os-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name:  system-upgrade
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: k3os-upgrade
+    namespace: k3os-system
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: default-controller-env
+  namespace: k3os-system
+data:
+  SYSTEM_UPGRADE_CONTROLLER_DEBUG: "true"
+  SYSTEM_UPGRADE_CONTROLLER_THREADS: "2"
+  SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: "900"
+  SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "2"
+  SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: "rancher/kubectl:v1.17.0"
+  SYSTEM_UPGRADE_JOB_PRIVILEGED: "true"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: system-upgrade-controller
+  namespace: k3os-system
+spec:
+  selector:
+    matchLabels:
+      upgrade.cattle.io/controller: system-upgrade-controller
+  template:
+    metadata:
+      labels:
+        upgrade.cattle.io/controller: system-upgrade-controller # necessary to avoid drain
+    spec:
+      serviceAccountName: k3os-upgrade
+      tolerations:
+        - key: "node.kubernetes.io/unschedulable" # necessary to avoid cordon/drain
+          operator: "Exists"
+          effect: "NoSchedule"
+      containers:
+        - name: system-upgrade-controller
+          image: rancher/system-upgrade-controller:v0.1.0
+          envFrom:
+            - configMapRef:
+                name: default-controller-env
+          env:
+            - name: SYSTEM_UPGRADE_CONTROLLER_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.labels['upgrade.cattle.io/controller']
+            - name: SYSTEM_UPGRADE_CONTROLLER_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: SYSTEM_UPGRADE_CONTROLLER_SERVICE_ACCOUNT
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.serviceAccountName
+          volumeMounts:
+            - name: etc-ssl
+              mountPath: /etc/ssl
+            - name: tmp
+              mountPath: /tmp
+      volumes:
+        - name: etc-ssl
+          hostPath:
+            path: /etc/ssl
+            type: Directory
+        - name: tmp
+          emptyDir: {}

overlay/share/rancher/k3s/server/manifests/system-upgrade-plans/k3os-latest.yaml

@@ -0,0 +1,53 @@
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  # This `name` should be short but descriptive.
+  name: k3os-latest
+  # The same `namespace` as is used for the system-upgrade-controller Deployment.
+  namespace: k3os-system
+spec:
+  # The maximum number of concurrent nodes to apply this update on.
+  concurrency: 1
+  # The value for `channel` is assumed to be a URL that returns HTTP 302 with the last path element of the value
+  # returned in the Location header assumed to be an image tag.
+  # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/upgrade/plan/plan.go#L177
+  channel: https://github.com/rancher/k3os/releases/latest
+  # Providing a value for `version` will prevent polling/resolution of the `channel` if specified.
+  # version: v0.9.0-dev
+  nodeSelector:
+    matchExpressions:
+      # This limits application of this upgrade only to nodes that have opted in by applying this label.
+      # Additionally, a value of `disabled` for this label on a node will cause the controller to skip over the node.
+      # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/upgrade/plan/plan.go#L216
+      # NOTICE THAT THE NAME PORTION OF THIS LABEL MATCHES THE PLAN NAME. This is related to the fact that the
+      # system-upgrade-controller will tag the node with this very label having the value of the applied version.
+      # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/upgrade/plan/plan.go#L112-L115
+      - {key: plan.upgrade.cattle.io/k3os-latest, operator: Exists}
+      # This label is set by k3OS, therefore a node without it should not apply this upgrade.
+      - {key: k3os.io/mode, operator: Exists}
+      # Additionally, do not attempt to upgrade nodes booted from "live" CDROM.
+      - {key: k3os.io/mode, operator: NotIn, values: ["live"]}
+  # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/apis/upgrade.cattle.io/v1/types.go#L58
+  drain:
+    # deleteLocalData: true
+    # ignoreDaemonSets: true
+    force: true
+  # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/apis/upgrade.cattle.io/v1/types.go#L51
+  upgrade:
+    # The tag portion of the image will be overridden with the value from `.status.latestVersion` a.k.a. the resolved version.
+    # SEE https://github.com/rancher/system-upgrade-controller/blob/v0.1.0/pkg/apis/upgrade.cattle.io/v1/types.go#L47
+    image: rancher/k3os
+    command: [k3os, --debug]
+    # It is safe to specify `--kernel` on overlay installations as the destination path will not exist and so the
+    # upgrade of the kernel component will be skipped (with a warning in the log).
+    args:
+      - upgrade
+      - --kernel
+      - --rootfs
+      - --remount
+      - --sync
+      - --reboot
+      - --lock-file=/host/run/k3os/upgrade.lock
+      - --source=/k3os/system
+      - --destination=/host/k3os/system

package/Dockerfile

@@ -1,3 +1,5 @@
 FROM scratch
-ENV PATH /k3os/system/k3os/current:/k3os/system/k3s/current
 COPY build/k3os/system/ /k3os/system/
+ENV PATH /k3os/system/k3os/current:/k3os/system/k3s/current:${PATH}
+ENTRYPOINT ["k3os"]
+CMD ["help"]

pkg/cc/funcs.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rancher/k3os/pkg/module"
 	"github.com/rancher/k3os/pkg/ssh"
 	"github.com/rancher/k3os/pkg/sysctl"
+	"github.com/rancher/k3os/pkg/version"
 	"github.com/rancher/k3os/pkg/writefile"
 	"github.com/sirupsen/logrus"
 )
@@ -140,10 +141,11 @@ func ApplyK3S(cfg *config.CloudConfig, restart, install bool) error {
 	if mode != "" {
 		labels = append(labels, fmt.Sprintf("k3os.io/mode=%s", mode))
 	}
+	labels = append(labels, fmt.Sprintf("k3os.io/version=%s", version.Version))
 	sort.Strings(labels)
 
 	for _, l := range labels {
-		args = append(args, "--kubelet-arg", "node-labels="+l)
+		args = append(args, "--node-label", l)
 	}
 
 	for _, taint := range cfg.K3OS.Taints {

pkg/mode/mode.go

@@ -3,11 +3,14 @@ package mode
 import (
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
+
+	"github.com/rancher/k3os/pkg/system"
 )
 
-func Get() (string, error) {
-	bytes, err := ioutil.ReadFile("/run/k3os/mode")
+func Get(prefix ...string) (string, error) {
+	bytes, err := ioutil.ReadFile(filepath.Join(filepath.Join(prefix...), system.StatePath("mode")))
 	if os.IsNotExist(err) {
 		return "", nil
 	} else if err != nil {