Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Backport "Add Support for Persistent OVMF Settings in Pillar" #4262

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pkg/pillar/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
FROM lfedge/eve-fscrypt:0b7cc0d9d620e47fc54e21d56cb8a5cd224f9c9b as fscrypt

FROM lfedge/eve-uefi:d821658883d6748d8bbf0d6640c62288e3ce8c6f as uefi-build
FROM lfedge/eve-dom0-ztools:417d4ff6a57d2317c9e65166274b0ea6f6da16e2 as zfs
RUN mkdir /out
# copy zfs-related files from dom0-ztools using prepared list of files
Expand Down Expand Up @@ -80,6 +81,9 @@ RUN set -e && for patch in /sys-patches/*.patch; do \
patch -p0 --no-backup-if-mismatch -r /tmp/deleteme.rej < "$patch"; \
done

RUN mkdir -p /out/usr/lib/xen/boot
COPY --from=uefi-build /OVMF_VARS.fd /out/usr/lib/xen/boot/ovmf_vars.bin

# we need zfs files on running system
COPY --from=zfs /out /out

Expand Down
83 changes: 79 additions & 4 deletions pkg/pillar/hypervisor/kvm.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/lf-edge/eve/pkg/pillar/agentlog"
"github.com/lf-edge/eve/pkg/pillar/containerd"
"github.com/lf-edge/eve/pkg/pillar/types"
fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
uuid "github.com/satori/go.uuid"
"github.com/sirupsen/logrus"
)
Expand Down Expand Up @@ -72,7 +73,9 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
kernel-irqchip = "on"
{{- end -}}
{{- if .DomainConfig.BootLoader }}
{{- if ne .VirtualizationMode "FML" }}
firmware = "{{.DomainConfig.BootLoader}}"
{{- end }}
{{- end -}}
{{- if .DomainConfig.Kernel }}
kernel = "{{.DomainConfig.Kernel}}"
Expand All @@ -85,7 +88,7 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
{{- end -}}
{{- if .DomainConfig.ExtraArgs }}
append = "{{.DomainConfig.ExtraArgs}}"
{{ end }}
{{- end }}
{{if ne .Machine "virt" }}
[global]
driver = "kvm-pit"
Expand All @@ -109,7 +112,22 @@ const qemuConfTemplate = `# This file is automatically generated by domainmgr
[device]
driver = "intel-iommu"
caching-mode = "on"
{{ end }}
{{- end }}

{{- if eq .VirtualizationMode "FML" }}

[drive "drive-ovmf-code"]
if = "pflash"
format = "raw"
readonly = "on"
file = "{{.DomainConfig.BootLoader}}"

[drive "drive-ovmf-vars"]
if = "pflash"
format = "raw"
file = "{{.DomainConfig.BootLoaderSettingsFile}}"
{{- end }}

[realtime]
mlock = "off"

Expand Down Expand Up @@ -621,12 +639,56 @@ func vmmOverhead(domainName string, domainUUID uuid.UUID, domainRAMSize int64, v
return overhead, nil
}

func getOVMFSettingsFile(domainName string) string {
return types.OVMFSettingsDir + "/" + domainName + "_ovmf_vars.bin"
}

func prepareOVMFSettings(domainName string) (string, error) {
// Create the OVMF settings directory if it does not exist
if err := os.MkdirAll(types.OVMFSettingsDir, 0755); err != nil {
return "", logError("failed to create OVMF settings directory: %v", err)
}
// Create a copy of the ovmf_vars.bin file in <domainName>_ovmf_vars.bin
ovmfSettingsFile := getOVMFSettingsFile(domainName)
if _, err := os.Stat(ovmfSettingsFile); os.IsNotExist(err) {
if err := fileutils.CopyFile(types.OVMFSettingsTemplate, ovmfSettingsFile); err != nil {
return "", logError("failed to copy ovmf_vars.bin file: %v", err)
}
}
// Set the RW permissions for the OVMF settings file
if err := os.Chmod(ovmfSettingsFile, 0666); err != nil {
return "", logError("failed to set RW permissions for ovmf_vars.bin file: %v", err)
}
return ovmfSettingsFile, nil
}

func cleanupOVMFSettings(domainName string) error {
ovmfVarsFile := getOVMFSettingsFile(domainName)
if _, err := os.Stat(ovmfVarsFile); err == nil {
if err := os.Remove(ovmfVarsFile); err != nil {
return logError("failed to remove ovmf_vars.bin file: %v", err)
}
}
return nil
}

// Setup sets up kvm
func (ctx kvmContext) Setup(status types.DomainStatus, config types.DomainConfig,
aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {

diskStatusList := status.DiskStatusList
domainName := status.DomainName
domainUUID := status.UUIDandVersion.UUID

// Before we start building the domain config, we need to prepare the OVMF settings
if config.VirtualizationMode == types.FML {
ovmfSettingsFile, err := prepareOVMFSettings(domainName)
if err != nil {
return logError("failed to setup OVMF settings for domain %s: %v", status.DomainName, err)
}
config.BootLoaderSettingsFile = ovmfSettingsFile
}

// first lets build the domain config
if err := ctx.CreateDomConfig(domainName, config, status, diskStatusList, aa, file); err != nil {
return logError("failed to build domain config: %v", err)
Expand Down Expand Up @@ -679,11 +741,16 @@ func (ctx kvmContext) Setup(status types.DomainStatus, config types.DomainConfig

func (ctx kvmContext) CreateDomConfig(domainName string, config types.DomainConfig, status types.DomainStatus,
diskStatusList []types.DiskStatus, aa *types.AssignableAdapters, file *os.File) error {
virtualizationMode := ""
if config.VirtualizationMode == types.FML {
virtualizationMode = "FML"
}
tmplCtx := struct {
Machine string
Machine string
VirtualizationMode string
types.DomainConfig
types.DomainStatus
}{ctx.devicemodel, config, status}
}{ctx.devicemodel, virtualizationMode, config, status}
tmplCtx.DomainConfig.Memory = (config.Memory + 1023) / 1024
tmplCtx.DomainConfig.DisplayName = domainName

Expand Down Expand Up @@ -973,6 +1040,14 @@ func (ctx kvmContext) Cleanup(domainName string) error {
return fmt.Errorf("error waiting for Qmp absent for domain %s: %v", domainName, err)
}

// Cleanup OVMF settings
// XXX it should be a check for FML mode based on some config/status option. But we have
// only domain name here. So we check if the OVMF settings file exists.
if _, err := os.Stat(getOVMFSettingsFile(domainName)); err == nil {
if err := cleanupOVMFSettings(domainName); err != nil {
return fmt.Errorf("failed to cleanup OVMF settings for domain %s: %v", domainName, err)
}
}
return nil
}

Expand Down
19 changes: 10 additions & 9 deletions pkg/pillar/types/domainmgrtypes.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,15 +195,16 @@ func (config DomainConfig) LogKey() string {
// StorageConfigList. For example, a Target of "kernel" means to set/override
// the Kernel attribute below.
type VmConfig struct {
Kernel string // default ""
Ramdisk string // default ""
Memory int // in kbytes; Rounded up to Mbytes for xen
MaxMem int // in kbytes; Default equal to 'Memory', so no ballooning for xen
VCpus int // default 1
MaxCpus int // default VCpus
RootDev string // default "/dev/xvda1"
ExtraArgs string // added to bootargs
BootLoader string // default ""
Kernel string // default ""
Ramdisk string // default ""
Memory int // in kbytes; Rounded up to Mbytes for xen
MaxMem int // in kbytes; Default equal to 'Memory', so no ballooning for xen
VCpus int // default 1
MaxCpus int // default VCpus
RootDev string // default "/dev/xvda1"
ExtraArgs string // added to bootargs
BootLoader string // default ""
BootLoaderSettingsFile string // used to pass bootloader settings file, for example OVMF_VARS.fd
// For CPU pinning
CPUs string // default "", list of "1,2"
// Needed for device passthru
Expand Down
5 changes: 5 additions & 0 deletions pkg/pillar/types/locationconsts.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,11 @@ const (

// ContainerdContentDir - path to containerd`s content store
ContainerdContentDir = SealedDirName + "/containerd/io.containerd.content.v1.content"

// OVMFSettingsDir - directory for OVMF settings, they are stored in per-domain files
OVMFSettingsDir = PersistDir + "/ovmf"
// OVMFSettingsTemplate - template file for OVMF settings
OVMFSettingsTemplate = "/usr/lib/xen/boot/ovmf_vars.bin"
)

var (
Expand Down