Skip to content

Commit

Permalink
Merge pull request #70 from Itxaka/osbuilder_go
Browse files Browse the repository at this point in the history
  • Loading branch information
Itxaka authored Jul 20, 2023
2 parents f2294c2 + de9b7bb commit ce8fe1f
Show file tree
Hide file tree
Showing 26 changed files with 3,547 additions and 73 deletions.
24 changes: 24 additions & 0 deletions .github/workflows/enki.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
name: 'run enki unit tests'

on:
pull_request:

concurrency:
group: enki-${{ github.ref || github.head_ref }}
cancel-in-progress: true

env:
FORCE_COLOR: 1

jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: earthly/[email protected]
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Build
run: cd tools-image/enki && earthly -P +test
100 changes: 38 additions & 62 deletions tools-image/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,84 +3,59 @@ ARG LEAP_VERSION=15.4
ARG LUET_VERSION=0.33.0
FROM quay.io/luet/base:$LUET_VERSION AS luet

FROM opensuse/leap:$LEAP_VERSION as luet-install
FROM golang:1.20 as enki
ENV CGO_ENABLED=0
COPY ./enki /src/enki
WORKDIR /src/enki
RUN go mod download
# Set arg/env after go mod download, otherwise we invalidate the cached layers due to the commit changing easily
ARG ENKI_VERSION=0.0.1
ARG ENKI_COMMIT=""
ENV ENKI_VERSION=${ENKI_VERSION}
ENV ENKI_COMMIT=${ENKI_COMMIT}
RUN go build \
-ldflags "-w -s \
-X github.com/kairos-io/enki/internal/version.version=$ENKI_VERSION \
-X github.com/kairos-io/enki/internal/version.gitCommit=$ENKI_COMMIT" \
-o /usr/bin/enki

FROM opensuse/leap:$LEAP_VERSION as default
COPY --from=luet /usr/bin/luet /usr/bin/luet
ENV LUET_NOLOCK=true
ENV TMPDIR=/tmp
COPY luet.yaml /etc/luet/luet.yaml
RUN luet install -y system/elemental-cli
RUN luet install -y livecd/grub2 --system-target /grub2
RUN luet install -y livecd/grub2-efi-image --system-target /efi

# remove luet tmp files. Side effect of setting the system-target is that it treats it as a root fs
RUN rm -Rf /grub2/var
RUN rm -Rf /efi/var
RUN zypper ref && zypper dup -y
## ISO+ Arm image + Netboot + cloud images Build depedencies
RUN zypper ref && zypper in -y bc qemu-tools jq cdrtools docker git curl gptfdisk kpartx sudo xfsprogs parted util-linux-systemd e2fsprogs curl util-linux udev rsync grub2 dosfstools grub2-x86_64-efi squashfs mtools xorriso lvm2 zstd

## amd64 Live CD artifacts
FROM quay.io/kairos/packages:grub2-livecd-0.0.6 AS grub2
FROM quay.io/kairos/packages:grub2-efi-image-livecd-0.0.6 AS efi
## Live CD artifacts
RUN luet install -y livecd/grub2 --system-target /grub2
RUN luet install -y livecd/grub2-efi-image --system-target /efi

## RPI64

## Firmware is in the amd64 repo (noarch)
FROM quay.io/kairos/packages:u-boot-rpi64-firmware-2021.01-5.1 AS rpi-u-boot
FROM quay.io/kairos/packages:raspberrypi-firmware-firmware-2021.03.10-2.1 AS rpi-firmware
FROM quay.io/kairos/packages:raspberrypi-firmware-config-firmware-2021.03.10-2.1 AS rpi-firmware-config
FROM quay.io/kairos/packages:raspberrypi-firmware-dt-firmware-2021.03.15-2.1 AS rpi-firmware-dt
RUN luet install -y firmware/u-boot-rpi64 firmware/raspberrypi-firmware firmware/raspberrypi-firmware-config firmware/raspberrypi-firmware-dt --system-target /rpi/

## PineBook64 Pro
FROM quay.io/kairos/packages:u-boot-rockchip-arm-vendor-blob-0.1 AS pinebook-u-boot

## Generic ARM artifacts
FROM quay.io/kairos/packages-arm64:grub-efi-static-0.2 AS grub-efi
FROM quay.io/kairos/packages-arm64:grub-config-static-0.3 AS grub-config
FROM quay.io/kairos/packages-arm64:grub-artifacts-static-0.2 AS grub-artifacts
RUN luet install -y arm-vendor-blob/u-boot-rockchip --system-target /pinebookpro/u-boot

## RAW images
FROM quay.io/kairos/packages:grub-efi-static-0.1 AS grub-raw-efi
FROM quay.io/kairos/packages:grub-config-static-0.1 AS grub-raw-config
FROM quay.io/kairos/packages:grub-artifacts-static-0.1 AS grub-raw-artifacts

FROM opensuse/leap:$LEAP_VERSION
COPY --from=luet-install /usr/bin/elemental /usr/bin/elemental
COPY --from=luet /usr/bin/luet /usr/bin/luet

# ISO files
COPY --from=luet-install /grub2 /grub2
COPY --from=luet-install /efi /efi

# RAW images
COPY --from=grub-raw-efi / /raw/grub
COPY --from=grub-raw-config / /raw/grubconfig
COPY --from=grub-raw-artifacts / /raw/grubartifacts

# RPI64
COPY --from=rpi-u-boot / /rpi/u-boot
COPY --from=rpi-firmware / /rpi/rpi-firmware
COPY --from=rpi-firmware-config / /rpi/rpi-firmware-config
COPY --from=rpi-firmware-dt / /rpi/rpi-firmware-dt

# Pinebook
COPY --from=pinebook-u-boot / /pinebookpro/u-boot
RUN luet install -y static/grub-efi --system-target /raw/grub
RUN luet install -y static/grub-config --system-target /raw/grubconfig
RUN luet install -y static/grub-artifacts --system-target /raw/grubartifacts

# Generic
COPY --from=grub-efi / /arm/grub/efi
COPY --from=grub-config / /arm/grub/config
COPY --from=grub-artifacts / /arm/grub/artifacts

RUN zypper ref && zypper dup -y
# remove luet tmp files. Side effect of setting the system-target is that it treats it as a root fs
# so temporal files are stored in each dir
RUN rm -Rf /grub2/var/tmp
RUN rm -Rf /efi/var/tmp
RUN rm -Rf /rpi/var/tmp
RUN rm -Rf /pinebookpro/u-boot/var/tmp
RUN rm -Rf /raw/grub/var/tmp
RUN rm -Rf /raw/grubconfig/var/tmp
RUN rm -Rf /raw/grubartifacts/var/tmp

## ISO Build depedencies
RUN zypper ref && zypper in -y xfsprogs parted util-linux-systemd e2fsprogs curl util-linux udev rsync grub2 dosfstools grub2-x86_64-efi squashfs mtools xorriso lvm2 zstd
RUN mkdir /config

# Arm image build deps
RUN zypper in -y jq docker git curl gptfdisk kpartx sudo
# Netboot
RUN zypper in -y cdrtools
# cloud images
RUN zypper in -y bc qemu-tools

# ISO build config
COPY ./config.yaml /config/manifest.yaml
COPY ./entrypoint.sh /entrypoint.sh
Expand All @@ -101,5 +76,6 @@ COPY ./netboot.sh /netboot.sh

COPY defaults.yaml /defaults.yaml

COPY --from=enki /usr/bin/enki /usr/bin/enki

ENTRYPOINT [ "/entrypoint.sh" ]
6 changes: 2 additions & 4 deletions tools-image/arm/boards/rpi64.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@ TEMPDIR="$(mktemp -d)"
echo $TEMPDIR
mount "${device}p1" "${TEMPDIR}"

for dir in /rpi/u-boot /rpi/rpi-firmware /rpi/rpi-firmware-config /rpi/rpi-firmware-dt
do
cp -rfv ${dir}/* $TEMPDIR
done
# Copy all rpi files
cp -rfv /rpi/* $TEMPDIR

umount "${TEMPDIR}"
6 changes: 3 additions & 3 deletions tools-image/build-arm-image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -321,9 +321,9 @@ cp -rfv ${STATEDIR}/cOS/active.img ${RECOVERY}/cOS/recovery.img
tune2fs -L ${SYSTEM_LABEL} ${RECOVERY}/cOS/recovery.img

# Install real grub config to recovery
cp -rfv /arm/grub/config/* $RECOVERY
cp -rfv /raw/grubconfig/* $RECOVERY
mkdir -p $RECOVERY/grub2/fonts
cp -rfv /arm/grub/artifacts/* $RECOVERY/grub2
cp -rfv /raw/grubartifacts/* $RECOVERY/grub2
mv $RECOVERY/grub2/*pf2 $RECOVERY/grub2/fonts

sync
Expand All @@ -336,7 +336,7 @@ if [ -z "$EFI" ]; then
exit 1
fi

cp -rfv /arm/grub/efi/* $EFI
cp -rfv /efi/* $EFI
if [ -n "$EFI" ] && [ -n "$efi_dir" ]; then
echo "Copy $efi_dir to EFI directory"
cp -rfv $efi_dir/* $EFI
Expand Down
19 changes: 19 additions & 0 deletions tools-image/enki/Earthfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
VERSION 0.7

# renovate: datasource=docker depName=golang
ARG --global GO_VERSION=1.20-alpine3.18

test:
FROM golang:$GO_VERSION
RUN apk add rsync gcc musl-dev docker jq
WORKDIR /build
COPY . .
RUN go mod download
ARG TEST_PATHS=./...
ARG LABEL_FILTER=
ENV CGO_ENABLED=1
# Some test require the docker sock exposed
WITH DOCKER
RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "$LABEL_FILTER" -v --fail-fast --race --covermode=atomic --coverprofile=coverage.out --coverpkg=github.com/kairos-io/enki/... -p -r $TEST_PATHS
END
SAVE ARTIFACT coverage.out AS LOCAL coverage.out
122 changes: 122 additions & 0 deletions tools-image/enki/cmd/build-iso.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package cmd

import (
"fmt"
"os/exec"

"github.com/kairos-io/enki/pkg/action"
"github.com/kairos-io/enki/pkg/config"
"github.com/kairos-io/enki/pkg/utils"
v1 "github.com/kairos-io/kairos-agent/v2/pkg/types/v1"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"k8s.io/mount-utils"
)

// NewBuildISOCmd returns a new instance of the build-iso subcommand and appends it to
// the root command.
func NewBuildISOCmd() *cobra.Command {
c := &cobra.Command{
Use: "build-iso SOURCE",
Short: "Build bootable installation media ISOs",
Long: "Build bootable installation media ISOs\n\n" +
"SOURCE - should be provided as uri in following format <sourceType>:<sourceName>\n" +
" * <sourceType> - might be [\"dir\", \"file\", \"oci\", \"docker\"], as default is \"docker\"\n" +
" * <sourceName> - is path to file or directory, image name with tag version",
Args: cobra.MaximumNArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
return CheckRoot()
},
RunE: func(cmd *cobra.Command, args []string) error {
path, err := exec.LookPath("mount")
if err != nil {
return err
}
mounter := mount.New(path)

cfg, err := config.ReadConfigBuild(viper.GetString("config-dir"), cmd.Flags(), mounter)
if err != nil {
cfg.Logger.Errorf("Error reading config: %s\n", err)
}

flags := cmd.Flags()

// Set this after parsing of the flags, so it fails on parsing and prints usage properly
cmd.SilenceUsage = true
cmd.SilenceErrors = true // Do not propagate errors down the line, we control them
spec, err := config.ReadBuildISO(cfg, flags)
if err != nil {
cfg.Logger.Errorf("invalid install command setup %v", err)
return err
}

if len(args) == 1 {
imgSource, err := v1.NewSrcFromURI(args[0])
if err != nil {
cfg.Logger.Errorf("not a valid rootfs source image argument: %s", args[0])
return err
}
spec.RootFS = []*v1.ImageSource{imgSource}
} else if len(spec.RootFS) == 0 {
errmsg := "rootfs source image for building ISO was not provided"
cfg.Logger.Errorf(errmsg)
return fmt.Errorf(errmsg)
}

// Repos and overlays can't be unmarshaled directly as they require
// to be merged on top and flags do not match any config value key
oRootfs, _ := flags.GetString("overlay-rootfs")
oUEFI, _ := flags.GetString("overlay-uefi")
oISO, _ := flags.GetString("overlay-iso")

if oRootfs != "" {
if ok, err := utils.Exists(cfg.Fs, oRootfs); ok {
spec.RootFS = append(spec.RootFS, v1.NewDirSrc(oRootfs))
} else {
cfg.Logger.Errorf("Invalid value for overlay-rootfs")
return fmt.Errorf("Invalid path '%s': %v", oRootfs, err)
}
}
if oUEFI != "" {
if ok, err := utils.Exists(cfg.Fs, oUEFI); ok {
spec.UEFI = append(spec.UEFI, v1.NewDirSrc(oUEFI))
} else {
cfg.Logger.Errorf("Invalid value for overlay-uefi")
return fmt.Errorf("Invalid path '%s': %v", oUEFI, err)
}
}
if oISO != "" {
if ok, err := utils.Exists(cfg.Fs, oISO); ok {
spec.Image = append(spec.Image, v1.NewDirSrc(oISO))
} else {
cfg.Logger.Errorf("Invalid value for overlay-iso")
return fmt.Errorf("Invalid path '%s': %v", oISO, err)
}
}

buildISO := action.NewBuildISOAction(cfg, spec)
err = buildISO.ISORun()
if err != nil {
cfg.Logger.Errorf(err.Error())
return err
}

return nil
},
}
c.Flags().StringP("name", "n", "", "Basename of the generated ISO file")
c.Flags().StringP("output", "o", "", "Output directory (defaults to current directory)")
c.Flags().Bool("date", false, "Adds a date suffix into the generated ISO file")
c.Flags().String("overlay-rootfs", "", "Path of the overlayed rootfs data")
c.Flags().String("overlay-uefi", "", "Path of the overlayed uefi data")
c.Flags().String("overlay-iso", "", "Path of the overlayed iso data")
c.Flags().String("label", "", "Label of the ISO volume")
archType := newEnumFlag([]string{"x86_64", "arm64"}, "x86_64")
c.Flags().Bool("squash-no-compression", true, "Disable squashfs compression.")
c.Flags().VarP(archType, "arch", "a", "Arch to build the image for")
return c
}

func init() {
rootCmd.AddCommand(NewBuildISOCmd())
}
Loading

0 comments on commit ce8fe1f

Please sign in to comment.