Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Baseboardmanagement controller implementation #3

Merged
merged 16 commits into from
May 17, 2022
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ doc/
.idea
.vscode
coverage.txt
cover.out

# Terraform
.terraform
Expand Down
4 changes: 1 addition & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@ COPY go.sum go.sum
RUN go mod download

# Copy the go source
COPY main.go main.go
COPY api/ api/
COPY controllers/ controllers/
COPY ./ ./

# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
Expand Down
4 changes: 2 additions & 2 deletions api/v1alpha1/baseboardmanagement_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ type BaseboardManagementSpec struct {
Connection Connection `json:"connection"`

// Power is the desired power state of the BaseboardManagement.
// +kubebuilder:validation:Enum=On;Off
// +kubebuilder:validation:Enum=on;off
Power PowerState `json:"power"`
}

Expand All @@ -75,7 +75,7 @@ type Connection struct {
// BaseboardManagementStatus defines the observed state of BaseboardManagement
type BaseboardManagementStatus struct {
// Power is the current power state of the BaseboardManagement.
// +kubebuilder:validation:Enum=On;Off
// +kubebuilder:validation:Enum=on;off
// +optional
Power PowerState `json:"powerState,omitempty"`

Expand Down
8 changes: 4 additions & 4 deletions config/crd/bases/bmc.tinkerbell.org_baseboardmanagements.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,8 +73,8 @@ spec:
power:
description: Power is the desired power state of the BaseboardManagement.
enum:
- "On"
- "Off"
- "on"
- "off"
type: string
required:
- connection
Expand Down Expand Up @@ -102,8 +102,8 @@ spec:
powerState:
description: Power is the current power state of the BaseboardManagement.
enum:
- "On"
- "Off"
- "on"
- "off"
type: string
type: object
type: object
Expand Down
2 changes: 2 additions & 0 deletions config/rbac/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ resources:
- role_binding.yaml
- leader_election_role.yaml
- leader_election_role_binding.yaml
- secrets_viewer_role.yaml
- secrets_viewer_role_binding.yaml
# Comment the following 4 lines if you want to disable
# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
# which protects your /metrics endpoint.
Expand Down
14 changes: 14 additions & 0 deletions config/rbac/secrets_viewer_role.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# permissions to do view Secrets.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: secrets-viewer-role
rules:
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
12 changes: 12 additions & 0 deletions config/rbac/secrets_viewer_role_binding.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: secrets-viewer-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: secrets-viewer-role
subjects:
- kind: ServiceAccount
name: controller-manager
namespace: system
208 changes: 194 additions & 14 deletions controllers/baseboardmanagement_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,42 +18,222 @@ package controllers

import (
"context"
"fmt"
"strings"

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"

bmcv1alpha1 "github.com/tinkerbell/rufio/api/v1alpha1"
"github.com/tinkerbell/rufio/pkg/bmcclient"
)

// BaseboardManagementReconciler reconciles a BaseboardManagement object
type BaseboardManagementReconciler struct {
client.Client
Scheme *runtime.Scheme
client client.Client
scheme *runtime.Scheme
pokearu marked this conversation as resolved.
Show resolved Hide resolved
bmcClientFactory bmcclient.BMCClientFactory
logger logr.Logger
}

// NewBaseboardManagementReconciler returns a new BaseboardManagementReconciler
func NewBaseboardManagementReconciler(client client.Client, scheme *runtime.Scheme, bmcClientFactory bmcclient.BMCClientFactory, logger logr.Logger) *BaseboardManagementReconciler {
return &BaseboardManagementReconciler{
client: client,
scheme: scheme,
bmcClientFactory: bmcClientFactory,
logger: logger,
}
}

// bmFieldReconciler defines a function to reconcile BaseboardManagement spec field
type bmFieldReconciler func(context.Context, *bmcv1alpha1.BaseboardManagement, bmcclient.BMCClient) error
pokearu marked this conversation as resolved.
Show resolved Hide resolved

//+kubebuilder:rbac:groups=bmc.tinkerbell.org,resources=baseboardmanagements,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=bmc.tinkerbell.org,resources=baseboardmanagements/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=bmc.tinkerbell.org,resources=baseboardmanagements/finalizers,verbs=update

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the BaseboardManagement object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
// Reconcile ensures the state of a BaseboardManagement.
// Gets the BaseboardManagement object and uses the SecretReference to initialize a BMC Client.
// Ensures the BMC power is set to the desired state.
// Updates the status and conditions accordingly.
func (r *BaseboardManagementReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)
logger := r.logger.WithValues("BaseboardManagement", req.NamespacedName)
logger.Info("Reconciling BaseboardManagement", "name", req.NamespacedName)

// Fetch the BaseboardManagement object
baseboardManagement := &bmcv1alpha1.BaseboardManagement{}
err := r.client.Get(ctx, req.NamespacedName, baseboardManagement)
if err != nil {
if apierrors.IsNotFound(err) {
return ctrl.Result{}, nil
}

logger.Error(err, "Failed to get BaseboardManagement", "name", req.NamespacedName)
pokearu marked this conversation as resolved.
Show resolved Hide resolved
return ctrl.Result{}, fmt.Errorf("failed to get BaseboardManagement: %v", err)
}

// Deletion is a noop.
if !baseboardManagement.DeletionTimestamp.IsZero() {
return ctrl.Result{}, nil
}

return r.reconcile(ctx, baseboardManagement)
}

func (r *BaseboardManagementReconciler) reconcile(ctx context.Context, bm *bmcv1alpha1.BaseboardManagement) (ctrl.Result, error) {
logger := r.logger.WithValues("BaseboardManagement", bm.Name, "Namespace", bm.Namespace)
pokearu marked this conversation as resolved.
Show resolved Hide resolved

// Fetching username, password from SecretReference
// Requeue if error fetching secret
username, password, err := r.resolveAuthSecretRef(ctx, bm.Spec.Connection.AuthSecretRef)
if err != nil {
return ctrl.Result{Requeue: true}, fmt.Errorf("resolving authentication from SecretReference: %v", err)
}

// TODO(user): your logic here
// TODO (pokearu): Remove port hardcoding
// Initializing BMC Client
bmcClient := r.bmcClientFactory(bm.Spec.Connection.Host, "623", username, password)
err = bmcClient.OpenConnection(ctx)
if err != nil {
logger.Error(err, "BMC connection failed", "host", bm.Spec.Connection.Host)
result, setConditionErr := r.setCondition(ctx, bm, bmcv1alpha1.ConnectionError, err.Error())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is nice, do you also want to update the status for other error cases below?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I was thinking on this. We could add condition types, like PowerStatusFailed PowerSetFailed?

Copy link
Member

@chrisdoherty4 chrisdoherty4 May 9, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we sure we have conditions right? A condition is typically defining a particular condition about the current state or a record of having passed through a particular state at some point, for example Connected, as opposed to a piece of data. That doesn't mean we can't add supplementary data like the error message but the condition itself needs to make sense and I'm not sure ConnectionError does given it is describing data that could just live at the top level of Status.

I feel like the following makes more sense

const (
   Connected = "Connected"
)

type BaseboardManagementCondition struct {
	// Type of the BaseboardManagement condition.
	Type BaseboardManagementConditionType `json:"type"`

    // Could be True, False or Unknown (empty string).
	Status BaseBoardManagementConditionStatus `json:"status"`

	// Message represents human readable message indicating details about last transition.
	// +optional
	Message string `json:"message,omitempty"`
}

In the case of Connected a status of False indicates a problem and an error message is likely available where as Unknown indicates we haven't even attempted to connect.

The above is more akin to traditional usage of conditions that feature in the core API types like Pod lifecycle conditions.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With respect to communicating a power status action failed that's really an event. We've tried to reconcile status by retrieving the current power state (or setting it) and it was unsuccessful so we generate an event for an operator to inspect. Events show up in the kubectl describe output.

I mention that because we shouldn't get carried away with conditions. Conditions are meant to be useful for machines, events should be useful for humans.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@pokearu Wants to address the conditions discussion in a separate PR so resolving for now.

if setConditionErr != nil {
return result, utilerrors.NewAggregate([]error{fmt.Errorf("failed to set conditions: %v", setConditionErr), err})
}
return result, err
}

// Close BMC connection after reconcilation
defer func() {
err = bmcClient.CloseConnection(ctx)
if err != nil {
pokearu marked this conversation as resolved.
Show resolved Hide resolved
logger.Error(err, "BMC close connection failed", "host", bm.Spec.Connection.Host)
}
}()

// fieldReconcilers defines BaseboardManagement spec field reconciler functions
fieldReconcilers := []bmFieldReconciler{
r.reconcilePower,
}
for _, reconiler := range fieldReconcilers {
err := reconiler(ctx, bm, bmcClient)
if err != nil {
pokearu marked this conversation as resolved.
Show resolved Hide resolved
logger.Error(err, "Failed to reconcile BaseboardManagement", "host", bm.Spec.Connection.Host)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you still want to update the status on failure. You want to converge on desired state, not make it all-or-nothing

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed, I modified the reconciler to update status when reconciling fields 👍

return ctrl.Result{}, err
}
}

// Patch the status after each reconciliation
return r.reconcileStatus(ctx, bm)
}

func (r *BaseboardManagementReconciler) reconcilePower(ctx context.Context, bm *bmcv1alpha1.BaseboardManagement, bmcClient bmcclient.BMCClient) error {
powerStatus, err := bmcClient.GetPowerStatus(ctx)
if err != nil {
return err
}

// If BaseboardManagement has desired power state then return
if bm.Spec.Power == bmcv1alpha1.PowerState(strings.ToLower(powerStatus)) {
return nil
}

// Setting baseboard management to desired power state
err = bmcClient.SetPowerState(ctx, string(bm.Spec.Power))
if err != nil {
return err
}

return nil
pokearu marked this conversation as resolved.
Show resolved Hide resolved
}

// setCondition updates the status.Condition if the condition type is present.
// Appends if new condition is found.
// Patches the BaseboardManagement status.
func (r *BaseboardManagementReconciler) setCondition(ctx context.Context, bm *bmcv1alpha1.BaseboardManagement, cType bmcv1alpha1.BaseboardManagementConditionType, message string) (ctrl.Result, error) {
patch := client.MergeFrom(bm.DeepCopy())

currentConditions := bm.Status.Conditions
for i := range currentConditions {
// If condition exists, update the message if different
if currentConditions[i].Type == cType {
if currentConditions[i].Message != message {
bm.Status.Conditions[i].Message = message
return r.patchStatus(ctx, bm, patch)
}
return ctrl.Result{}, nil
}
}

// Append new condition to Conditions
condition := bmcv1alpha1.BaseboardManagementCondition{
Type: cType,
Message: message,
}
bm.Status.Conditions = append(bm.Status.Conditions, condition)

return r.patchStatus(ctx, bm, patch)
}

// reconcileStatus updates the Power and Conditions and patches BaseboardManagement status.
func (r *BaseboardManagementReconciler) reconcileStatus(ctx context.Context, bm *bmcv1alpha1.BaseboardManagement) (ctrl.Result, error) {
patch := client.MergeFrom(bm.DeepCopy())

// Update the power status of the BaseboardManagement
bm.Status.Power = bm.Spec.Power
// Clear conditions
bm.Status.Conditions = []bmcv1alpha1.BaseboardManagementCondition{}

return r.patchStatus(ctx, bm, patch)
}

// patchStatus patches the specifies patch on the BaseboardManagement.
func (r *BaseboardManagementReconciler) patchStatus(ctx context.Context, bm *bmcv1alpha1.BaseboardManagement, patch client.Patch) (ctrl.Result, error) {
logger := r.logger.WithValues("BaseboardManagement", bm.Name, "Namespace", bm.Namespace)

err := r.client.Status().Patch(ctx, bm, patch)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@chrisdoherty4 @jacobweinstock there is no idempotency/locking in the whole reconcile, do we want/need that?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope. When you're only considering the object under reconciliation there's no races as the manager ensures you're the only reconciler reconciling that resource.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is, of course, the possibility that an object is patched by something else but I don't think we have those semantics here.

if err != nil {
logger.Error(err, "Failed to patch BaseboardManagement status")
return ctrl.Result{}, fmt.Errorf("failed to patch BaseboardManagement status: %v", err)
}

return ctrl.Result{}, nil
}

// resolveAuthSecretRef Gets the Secret from the SecretReference.
// Returns the username and password encoded in the Secret.
func (r *BaseboardManagementReconciler) resolveAuthSecretRef(ctx context.Context, secretRef corev1.SecretReference) (string, string, error) {
secret := &corev1.Secret{}
key := types.NamespacedName{Namespace: secretRef.Namespace, Name: secretRef.Name}

if err := r.client.Get(ctx, key, secret); err != nil {
if apierrors.IsNotFound(err) {
return "", "", fmt.Errorf("secret %s not found: %v", key, err)
}

return "", "", fmt.Errorf("failed to retrieve secret %s : %v", secretRef, err)
}

username, ok := secret.Data["username"]
if !ok {
return "", "", fmt.Errorf("required secret key username not present")
pokearu marked this conversation as resolved.
Show resolved Hide resolved
}

password, ok := secret.Data["password"]
if !ok {
return "", "", fmt.Errorf("required secret key password not present")
}

return string(username), string(password), nil
chrisdoherty4 marked this conversation as resolved.
Show resolved Hide resolved
}

// SetupWithManager sets up the controller with the Manager.
func (r *BaseboardManagementReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
Expand Down
16 changes: 14 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@ module github.com/tinkerbell/rufio
go 1.18

require (
github.com/bmc-toolbox/bmclib v0.4.15
github.com/go-logr/logr v1.2.0
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.17.0
github.com/spf13/pflag v1.0.5
golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff
k8s.io/api v0.23.0
k8s.io/apimachinery v0.23.0
Expand All @@ -21,22 +24,30 @@ require (
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.0.2 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-logr/logr v1.2.0 // indirect
github.com/gebn/bmc v0.0.0-20200904230046-a5643220ab2a // indirect
github.com/go-logr/zapr v1.2.0 // indirect
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.5 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/gopacket v1.1.18 // indirect
github.com/google/uuid v1.1.2 // indirect
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jacobweinstock/registrar v0.4.2 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
Expand All @@ -46,7 +57,7 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.28.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stmcginnis/gofish v0.12.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.19.1 // indirect
Expand All @@ -62,6 +73,7 @@ require (
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/go-playground/validator.v9 v9.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
Expand Down
Loading