Skip to content

Commit

Permalink
wasi/k8s: add host module for doing k8 lookups
Browse files Browse the repository at this point in the history
  • Loading branch information
davidmdm committed Feb 14, 2025
1 parent c4cb4a6 commit 5fff1b2
Show file tree
Hide file tree
Showing 12 changed files with 295 additions and 16 deletions.
4 changes: 2 additions & 2 deletions Dockerfile.atc
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ WORKDIR /app

COPY go.mod go.sum ./

RUN go mod download
RUN GOTOOLCHAIN=auto go mod download

COPY ./cmd/atc ./cmd/atc
COPY ./internal ./internal
COPY ./pkg ./pkg

RUN go build -o /bin/atc ./cmd/atc
RUN GOTOOLCHAIN=auto go build -o /bin/atc ./cmd/atc

FROM alpine

Expand Down
12 changes: 6 additions & 6 deletions cmd/atc/internal/testing/Dockerfile.wasmcache
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@ WORKDIR /app

COPY go.mod go.sum ./

RUN go mod download
RUN GOTOOLCHAIN=auto go mod download

COPY . .

RUN \
GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/flight.v1.wasm ./cmd/atc/internal/testing/apis/backend/v1/flight && \
GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/flight.v2.wasm ./cmd/atc/internal/testing/apis/backend/v2/flight && \
GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/flight.dev.wasm ./cmd/atc/internal/testing/apis/backend/v2/dev && \
GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/converter.wasm ./cmd/atc/internal/testing/apis/backend/converter && \
go build -o ./bin/server ./cmd/atc/internal/testing/wasmcache
GOTOOLCHAIN=auto GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/flight.v1.wasm ./cmd/atc/internal/testing/apis/backend/v1/flight && \
GOTOOLCHAIN=auto GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/flight.v2.wasm ./cmd/atc/internal/testing/apis/backend/v2/flight && \
GOTOOLCHAIN=auto GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/flight.dev.wasm ./cmd/atc/internal/testing/apis/backend/v2/dev && \
GOTOOLCHAIN=auto GOOS=wasip1 GOARCH=wasm go build -o ./cmd/atc/internal/testing/wasmcache/wasm/converter.wasm ./cmd/atc/internal/testing/apis/backend/converter && \
GOTOOLCHAIN=auto go build -o ./bin/server ./cmd/atc/internal/testing/wasmcache

FROM alpine

Expand Down
8 changes: 4 additions & 4 deletions cmd/yokecd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"

"github.com/yokecd/yoke/internal"
"github.com/yokecd/yoke/internal/k8s"
"github.com/yokecd/yoke/pkg/yoke"
)

Expand Down Expand Up @@ -49,14 +49,14 @@ func run(ctx context.Context, cfg Config) (err error) {
return fmt.Errorf("failed to get in cluster config: %w", err)
}

clientset, err := kubernetes.NewForConfig(rest)
client, err := k8s.NewClient(rest)
if err != nil {
return fmt.Errorf("failed to instantiate kubernetes clientset: %w", err)
}

secrets := make(map[string]string, len(cfg.Flight.Refs))
for name, ref := range cfg.Flight.Refs {
secret, err := clientset.CoreV1().Secrets(cmp.Or(ref.Namespace, cfg.Namespace)).Get(ctx, ref.Secret, v1.GetOptions{})
secret, err := client.Clientset.CoreV1().Secrets(cmp.Or(ref.Namespace, cfg.Namespace)).Get(ctx, ref.Secret, v1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to get secret reference %q: %w", ref.Secret, err)
}
Expand Down Expand Up @@ -99,7 +99,7 @@ func run(ctx context.Context, cfg Config) (err error) {
return nil, fmt.Errorf("failed to get wasm path: %w", err)
}

data, _, err := yoke.EvalFlight(ctx, cfg.Application.Name, yoke.FlightParams{
data, _, err := yoke.EvalFlight(ctx, client, cfg.Application.Name, yoke.FlightParams{
Path: wasmPath,
Input: strings.NewReader(cfg.Flight.Input),
Args: cfg.Flight.Args,
Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
module github.com/yokecd/yoke

go 1.23.0
// TODO: use go1.24.0 once it is released. Blocker for releasing this feature.
// It is needed for the go:wasmexport directive.
go 1.24rc2

require (
github.com/alecthomas/chroma/v2 v2.15.0
Expand Down
3 changes: 2 additions & 1 deletion internal/atc/atc.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,8 @@ func (atc atc) Reconcile(ctx context.Context, event ctrl.Event) (result ctrl.Res
return fmt.Errorf("failed to load wasm: %w", err)
}
mod, err := wasi.Compile(ctx, wasi.CompileParams{
Wasm: data,
Wasm: data,
Client: ctrl.Client(ctx),
})
if err != nil {
return fmt.Errorf("failed to compile wasm: %w", err)
Expand Down
78 changes: 78 additions & 0 deletions internal/wasi/wasi.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,29 @@ package wasi

import (
"bytes"
"cmp"
"context"
"crypto/rand"
"fmt"
"io"
"reflect"

"github.com/tetratelabs/wazero"
"github.com/tetratelabs/wazero/api"
"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"

"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"

"github.com/davidmdm/x/xerr"

kerrors "k8s.io/apimachinery/pkg/api/errors"

"github.com/yokecd/yoke/internal"
"github.com/yokecd/yoke/internal/k8s"
"github.com/yokecd/yoke/internal/wasm"
)

type ExecParams struct {
Expand All @@ -24,6 +35,7 @@ type ExecParams struct {
Args []string
Env map[string]string
CacheDir string
Client *k8s.Client
}

func Execute(ctx context.Context, params ExecParams) (output []byte, err error) {
Expand All @@ -36,6 +48,7 @@ func Execute(ctx context.Context, params ExecParams) (output []byte, err error)
mod, err := Compile(ctx, CompileParams{
Wasm: params.Wasm,
CacheDir: params.CacheDir,
Client: params.Client,
})
if err != nil {
return nil, nil, fmt.Errorf("failed to compile module: %w", err)
Expand Down Expand Up @@ -89,6 +102,7 @@ func Execute(ctx context.Context, params ExecParams) (output []byte, err error)
type CompileParams struct {
Wasm []byte
CacheDir string
Client *k8s.Client
}

type Module struct {
Expand Down Expand Up @@ -143,6 +157,70 @@ func Compile(ctx context.Context, params CompileParams) (Module, error) {

runtime := wazero.NewRuntimeWithConfig(ctx, cfg)

hostModule := runtime.NewHostModuleBuilder("host")

for name, fn := range map[string]any{
"k8s_lookup": func(ctx context.Context, module api.Module, stateRef wasm.Ptr, name, namespace, kind, apiVersion wasm.String) wasm.Buffer {
gv, err := schema.ParseGroupVersion(apiVersion.Load(module))
if err != nil {
return wasm.Error(ctx, module, stateRef, wasm.StateError, err.Error())
}

mapping, err := params.Client.Mapper.RESTMapping(schema.GroupKind{Group: gv.Group, Kind: kind.Load(module)}, gv.Version)
if err != nil {
return wasm.Error(ctx, module, stateRef, wasm.StateError, err.Error())
}

intf := func() dynamic.ResourceInterface {
intf := params.Client.Dynamic.Resource(mapping.Resource)
if mapping.Scope == meta.RESTScopeNamespace {
return intf.Namespace(cmp.Or(namespace.Load(module), "default"))
}
return intf
}()

resource, err := intf.Get(ctx, name.Load(module), metav1.GetOptions{})
if err != nil {
errState := func() wasm.State {
switch {
case kerrors.IsNotFound(err):
return wasm.StateNotFound
case kerrors.IsForbidden(err):
return wasm.StateForbidden
case kerrors.IsUnauthorized(err):
return wasm.StateUnauthenticated
default:
return wasm.StateError
}
}()
return wasm.Error(ctx, module, stateRef, errState, err.Error())
}

data, err := resource.MarshalJSON()
if err != nil {
return wasm.Error(ctx, module, stateRef, wasm.StateError, err.Error())
}

results, err := module.ExportedFunction("malloc").Call(ctx, uint64(len(data)))
if err != nil {
// if we cannot malloc, let's crash with gumption.
panic(err)
}

buffer := wasm.Buffer(results[0])

module.Memory().Write(buffer.Address(), data)

return buffer
},
} {
hostModule = hostModule.NewFunctionBuilder().WithFunc(fn).Export(name)
}

if _, err := hostModule.Instantiate(ctx); err != nil {
return Module{}, fmt.Errorf("failed to instantiate host module: %w", err)
}

wasi_snapshot_preview1.MustInstantiate(ctx, runtime)

mod, err := runtime.CompileModule(ctx, params.Wasm)
Expand Down
94 changes: 94 additions & 0 deletions internal/wasm/wasm.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package wasm

import (
"cmp"
"context"
"unsafe"

"github.com/tetratelabs/wazero/api"
)

type (
String uint64
Buffer uint64
Ptr uint32
)

// State is used to convey the state of a host module function call. Given that a host function
// will generally do something the wasm module cannot do, it will likely do some sort of IO.
// This means that the call can either succeed or fail with some error. This allows us to interpret
// the returned memory buffer as either containing a value or an error.
//
// State is a uint32 allowing us to define well-known generic errors that packages can use to express semantic meaning.
// It is not exhaustive. As new use cases are added, we can add new semantic errors.
//
// Currently the only host function we expose is k8s.Lookup, this means means the host function can set any of the below states
// and the k8s package can use them to return meaningful error types to the user that they can in turn act upon.
type State uint32

const (
StateOK State = iota
StateError
StateNotFound
StateUnauthenticated
StateForbidden
)

func PtrTo[T any](value *T) Ptr {
return Ptr(uintptr(unsafe.Pointer(value)))
}

func Malloc(ctx context.Context, module api.Module, data []byte) Buffer {
results, err := module.ExportedFunction("malloc").Call(ctx, uint64(len(data)))
if err != nil {
panic(err)
}
buffer := Buffer(results[0])
module.Memory().Write(buffer.Address(), data)
return buffer
}

func Error(ctx context.Context, module api.Module, ptr Ptr, state State, err string) Buffer {
mem := module.Memory()
mem.WriteUint32Le(uint32(ptr), uint32(cmp.Or(state, StateError)))
return Malloc(ctx, module, []byte(err))
}

func (value String) Load(module api.Module) string {
return string(value.LoadBytes(module))
}

func (value String) LoadBytes(module api.Module) []byte {
data, ok := module.Memory().Read(uint32(value>>32), uint32(value))
if !ok {
panic("memory read out of bounds")
}
return data
}

func FromString(value string) String {
position := uint32(uintptr(unsafe.Pointer(unsafe.StringData(value))))
bytes := uint32(len(value))
return String(uint64(position)<<32 | uint64(bytes))
}

func FromSlice(value []byte) Buffer {
ptr := uint64(uintptr(unsafe.Pointer(&value[0])))
return Buffer(ptr<<32 | uint64(len(value)))
}

func (buffer Buffer) Address() uint32 {
return uint32(buffer >> 32)
}

func (buffer Buffer) Length() uint32 {
return uint32((buffer << 32) >> 32)
}

func (buffer Buffer) Slice() []byte {
return unsafe.Slice((*byte)(unsafe.Pointer(uintptr(buffer.Address()))), buffer.Length())
}

func (buffer Buffer) String() string {
return unsafe.String((*byte)(unsafe.Pointer(uintptr(buffer.Address()))), buffer.Length())
}
42 changes: 42 additions & 0 deletions pkg/flight/wasi/k8s/errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package k8s

import "errors"

type ErrorNotFound string

func (err ErrorNotFound) Error() string { return string(err) }

func (ErrorNotFound) Is(target error) bool {
_, ok := target.(ErrorNotFound)
return ok
}

func IsErrNotFound(err error) bool {
return errors.Is(err, ErrorNotFound(""))
}

type ErrorUnauthenticated string

func (err ErrorUnauthenticated) Error() string { return string(err) }

func (ErrorUnauthenticated) Is(target error) bool {
_, ok := target.(ErrorUnauthenticated)
return ok
}

func IsErrUnauthenticated(err error) bool {
return errors.Is(err, ErrorUnauthenticated(""))
}

type ErrorForbidden string

func (err ErrorForbidden) Error() string { return string(err) }

func (ErrorForbidden) Is(target error) bool {
_, ok := target.(ErrorForbidden)
return ok
}

func IsErrForbidden(err error) bool {
return errors.Is(err, ErrorForbidden(""))
}
Loading

0 comments on commit 5fff1b2

Please sign in to comment.