Merge pull request #300 from ytsssun/cherry-pick-4.0.1

Cherry pick commits to 4.0.x branch

Twoliter.toml

@@ -1,5 +1,5 @@
 schema-version = 1
-release-version = "4.0.0"
+release-version = "4.0.1"
 
 [vendor.bottlerocket]
 registry = "public.ecr.aws/bottlerocket"

packages/amazon-ecs-cni-plugins/amazon-ecs-cni-plugins.spec

@@ -4,8 +4,10 @@
 %global ecscni_gitrev 53a8481891251e66e35847554d52a13fc7c4fd03
 
 Name: %{_cross_os}amazon-ecs-cni-plugins
-Version: %{ecscni_gitrev}
+# https://github.com/aws/amazon-ecs-cni-plugins/blob/53a8481891251e66e35847554d52a13fc7c4fd03/VERSION#L1
+Version: 2020.09.0
 Release: 1%{?dist}
+Epoch: 1
 Summary: Networking plugins for ECS task networking
 License: Apache-2.0
 URL: https://%{ecscni_goimport}

packages/libnvidia-container/Cargo.toml

@@ -12,8 +12,8 @@ path = "../packages.rs"
 releases-url = "https://github.com/NVIDIA/libnvidia-container/releases"
 
 [[package.metadata.build-package.external-files]]
-url = "https://github.com/NVIDIA/libnvidia-container/archive/v1.17.1/libnvidia-container-1.17.1.tar.gz"
-sha512 = "7c643c3b119767e188752dd32314bd214ada1510f56617f9a20bc0483604d77215bcf6ce93c3b5a5111b30cf228df6d76625d1c1f2f2ce95d77e229d6c6120c4"
+url = "https://github.com/NVIDIA/libnvidia-container/archive/v1.17.3/libnvidia-container-1.17.3.tar.gz"
+sha512 = "24293e369fea42ebe64163464f600808c0d18e8b4efeea12095de22e16d43837cb6441f46baf52e8c966810c76b0f5045737a96d173e2ecf8cd15fff37cd4c4f"
 
 [[package.metadata.build-package.external-files]]
 url = "https://github.com/NVIDIA/nvidia-modprobe/archive/550.54.14/nvidia-modprobe-550.54.14.tar.gz"

packages/libnvidia-container/libnvidia-container.spec

@@ -1,7 +1,7 @@
 %global nvidia_modprobe_version 550.54.14
 
 Name: %{_cross_os}libnvidia-container
-Version: 1.17.1
+Version: 1.17.3
 Release: 1%{?dist}
 Epoch: 1
 Summary: NVIDIA container runtime library

packages/nvidia-container-toolkit/Cargo.toml

@@ -12,8 +12,8 @@ path = "../packages.rs"
 releases-url = "https://github.com/NVIDIA/nvidia-container-toolkit/releases"
 
 [[package.metadata.build-package.external-files]]
-url = "https://github.com/NVIDIA/nvidia-container-toolkit/archive/v1.17.1/nvidia-container-toolkit-1.17.1.tar.gz"
-sha512 = "166c0e4644196688dbc7036020e8d2ff1b99e8c164d921fa698655033da6e1a3a76d83d8e8a27bc0bd967469c2a9fb4c2764dc0148af6c8f195c49b3c7c871d1"
+url = "https://github.com/NVIDIA/nvidia-container-toolkit/archive/v1.17.3/nvidia-container-toolkit-1.17.3.tar.gz"
+sha512 = "8c7a4290a1decc448c72e9a09213e0dc4e418ec633cefb16bb6b01fef7c502d23ed72cc1f3cc6583cad07feae5ca3cf44dad73e1274e042e3b26bdc7a4152b95"
 
 [build-dependencies]
 glibc = { path = "../glibc" }

packages/nvidia-container-toolkit/nvidia-container-toolkit.spec

@@ -2,7 +2,7 @@
 %global gorepo nvidia-container-toolkit
 %global goimport %{goproject}/%{gorepo}
 
-%global gover 1.17.1
+%global gover 1.17.3
 %global rpmver %{gover}
 
 Name: %{_cross_os}nvidia-container-toolkit

packages/release/release.spec

@@ -88,6 +88,7 @@ Source1101: systemd-resolved-service-env.conf
 Source1102: systemd-networkd-service-env.conf
 Source1103: systemd-logind-inhibit-maxdelay.conf
 Source1104: aws-config.conf
+Source1105: wait-for-selinux-policy.conf
 
 # network link rules
 Source1200: 80-release.link
@@ -256,6 +257,9 @@ install -p -m 0644 %{S:207} %{buildroot}%{_cross_templatedir}/aws-credentials
 install -p -m 0644 %{S:208} %{buildroot}%{_cross_templatedir}/modules-load
 install -p -m 0644 %{S:209} %{buildroot}%{_cross_templatedir}/log4j-hotpatch-enabled
 
+install -d %{buildroot}%{_cross_unitdir}/systemd-udev-trigger.service.d/
+install -p -m 0644 %{S:1105} %{buildroot}%{_cross_unitdir}/systemd-udev-trigger.service.d/00-selinux.conf
+
 install -d %{buildroot}%{_cross_udevrulesdir}
 install -p -m 0644 %{S:1300} %{buildroot}%{_cross_udevrulesdir}/61-mount-cdrom.rules
 
@@ -323,6 +327,8 @@ ln -s preconfigured.target %{buildroot}%{_cross_unitdir}/default.target
 %{_cross_unitdir}/systemd-networkd.service.d/00-env.conf
 %dir %{_cross_unitdir}/systemd-tmpfiles-setup.service.d
 %{_cross_unitdir}/systemd-tmpfiles-setup.service.d/00-debug.conf
+%dir %{_cross_unitdir}/systemd-udev-trigger.service.d
+%{_cross_unitdir}/systemd-udev-trigger.service.d/00-selinux.conf
 %dir %{_cross_templatedir}
 %{_cross_templatedir}/modprobe-conf
 %{_cross_templatedir}/netdog-toml

packages/release/wait-for-selinux-policy.conf

@@ -0,0 +1,3 @@
+[Unit]
+Wants=selinux-policy-files.service
+After=selinux-policy-files.service

sources/host-ctr/cmd/host-ctr/main.go

@@ -41,6 +41,23 @@ import (
 // Example 4: 777777777777.dkr.ecr-fips.us-west-2.amazonaws.com/my_image:latest
 var ecrRegex = regexp.MustCompile(`(^[a-zA-Z0-9][a-zA-Z0-9-_]*)\.dkr\.ecr(-fips)?\.([a-zA-Z0-9][a-zA-Z0-9-_]*)\.(amazonaws\.com(\.cn)?|cloud\.adc-e\.uk).*`)
 
+// A set of currently supported ECR regions which are not yet present in the golang SDK
+var ecrRefPrefixMapping = map[string]string{
+	"ap-southeast-7": "ecr.aws/arn:aws:ecr:ap-southeast-7:",
+	"eu-isoe-west-1": "ecr.aws/arn:aws-iso-e:ecr:eu-isoe-west-1:",
+	"mx-central-1":   "ecr.aws/arn:aws:ecr:mx-central-1:",
+}
+
+// A set of the currently supported FIPS regions for ECR: https://docs.aws.amazon.com/general/latest/gr/ecr.html
+var fipsSupportedEcrRegionSet = map[string]bool{
+	"us-east-1":     true,
+	"us-east-2":     true,
+	"us-west-1":     true,
+	"us-west-2":     true,
+	"us-gov-east-1": true,
+	"us-gov-west-1": true,
+}
+
 const (
 	// The maximum size of an	image label.
 	imageLabelMaxSize = 4096
@@ -593,35 +610,27 @@ func cleanUp(containerdSocket string, namespace string, containerID string) erro
 	return nil
 }
 
-// parseImageURISpecialRegions mimics the parsing in ecr.ParseImageURI but
-// constructs the canonical ECR references while skipping certain checks.
-// We only do this for special regions that are not yet supported by the aws-go-sdk and for ECR FIPS endpoints.
-// Referenced source: https://github.com/awslabs/amazon-ecr-containerd-resolver/blob/a5058cf091f4fc573813a032db37a9820952f1f9/ecr/ref.go#L70-L71
-func parseImageURISpecialRegions(input string) (ecr.ECRSpec, error) {
-	ecrRefPrefixMapping := map[string]string{
-		"ap-southeast-7": "ecr.aws/arn:aws:ecr:ap-southeast-7:",
-		"eu-isoe-west-1": "ecr.aws/arn:aws-iso-e:ecr:eu-isoe-west-1:",
-		"mx-central-1":   "ecr.aws/arn:aws:ecr:mx-central-1:",
-	}
-	// A set of the currently supported FIPS regions for ECR: https://docs.aws.amazon.com/general/latest/gr/ecr.html
-	fipsSupportedEcrRegionSet := map[string]bool{
-		"us-east-1":     true,
-		"us-east-2":     true,
-		"us-west-1":     true,
-		"us-west-2":     true,
-		"us-gov-east-1": true,
-		"us-gov-west-1": true,
-	}
-	// Matching on account, region
+type parsedECR struct {
+	Region   string
+	Account  string
+	RepoPath string
+	Fips     bool
+}
+
+// parseImageURIAsECR mimics the parsing in ecr.ParseImageURI but only returns metadata pertaining
+// to the parsed URI.
+func parseImageURIAsECR(input string) (*parsedECR, error) {
 	matches := ecrRegex.FindStringSubmatch(input)
+
 	if len(matches) < 3 {
-		return ecr.ECRSpec{}, fmt.Errorf("invalid image URI: %s", input)
+		return nil, fmt.Errorf("invalid image URI: %s", input)
 	}
 	account := matches[1]
+
 	// Need to include the full repository path and the imageID (e.g. /eks/image-name:tag)
 	tokens := strings.SplitN(input, "/", 2)
 	if len(tokens) != 2 {
-		return ecr.ECRSpec{}, fmt.Errorf("invalid image URI: %s", input)
+		return nil, fmt.Errorf("invalid image URI: %s", input)
 	}
 	fullRepoPath := tokens[len(tokens)-1]
 	// Run simple checks on the provided repository.
@@ -633,33 +642,58 @@ func parseImageURISpecialRegions(input string) (ecr.ECRSpec, error) {
 		strings.HasSuffix(fullRepoPath, ":"),
 		// Must not have a partial/unsupplied digest specifier
 		strings.HasSuffix(fullRepoPath, "@"):
-		return ecr.ECRSpec{}, errors.New("incomplete reference provided")
+		return nil, errors.New("incomplete reference provided")
+	}
+
+	isFips := matches[2] == "-fips"
+	region := matches[3]
+
+	return &parsedECR{
+		Region:   region,
+		Account:  account,
+		RepoPath: fullRepoPath,
+		Fips:     isFips,
+	}, nil
+}
+
+// Metadata for specially-treated ECR URIs
+type specialRegions struct {
+	// region => domain mappings
+	EcrRefPrefixMappings map[string]string
+	// The set of regions supporting FIPS
+	FipsSupportedEcrRegions map[string]bool
+}
+
+// parseImageURISpecialRegions mimics the parsing in ecr.ParseImageURI but
+// constructs the canonical ECR references while skipping certain checks.
+// We only do this for special regions that are not yet supported by the aws-go-sdk and for ECR FIPS endpoints.
+// Referenced source: https://github.com/awslabs/amazon-ecr-containerd-resolver/blob/a5058cf091f4fc573813a032db37a9820952f1f9/ecr/ref.go#L70-L71
+func parseImageURISpecialRegions(input string, specialRegions specialRegions) (ecr.ECRSpec, error) {
+	parsedECR, err := parseImageURIAsECR(input)
+	if err != nil {
+		return ecr.ECRSpec{}, err
 	}
 
 	// Return early if the FIPS endpoint is being used. amazon-ecr-containerd-resolver doesn't yet support FIPS urls:
 	// https://github.com/awslabs/amazon-ecr-containerd-resolver/blob/7b72333e780f5a5168936eae79fb89448e2f2a8f/ecr/ref.go#L43
 	// The ecr-prefix helper for admin and control host containers will have already accounted for setting this endpoint
 	// if the region has FIPS support.
-	if matches[2] == "-fips" {
-		region := matches[3]
-		_, isFips := fipsSupportedEcrRegionSet[region]
+	if parsedECR.Fips {
+		_, isFips := specialRegions.FipsSupportedEcrRegions[parsedECR.Region]
 		if !isFips {
-			return ecr.ECRSpec{}, fmt.Errorf("%s: %s", "invalid FIPS region", region)
+			return ecr.ECRSpec{}, fmt.Errorf("%s: %s", "invalid FIPS region", parsedECR.Region)
 		}
-		ecrRefPrefix := fmt.Sprintf("ecr.aws/arn:aws:ecr-fips:%s:", region)
-		return ecr.ParseRef(fmt.Sprintf("%s%s:repository/%s", ecrRefPrefix, account, fullRepoPath))
+		ecrRefPrefix := fmt.Sprintf("ecr.aws/arn:aws:ecr-fips:%s:", parsedECR.Region)
+		return ecr.ParseRef(fmt.Sprintf("%s%s:repository/%s", ecrRefPrefix, parsedECR.Account, parsedECR.RepoPath))
 	}
 
-	// The provided URI does not specify the FIPS endpoint, and the second match is the region.
-	region := matches[2]
-
 	// Get the ECR image reference prefix from the AWS region
-	ecrRefPrefix, ok := ecrRefPrefixMapping[region]
+	ecrRefPrefix, ok := specialRegions.EcrRefPrefixMappings[parsedECR.Region]
 	if !ok {
-		return ecr.ECRSpec{}, fmt.Errorf("%s: %s", "invalid region in internal mapping", region)
+		return ecr.ECRSpec{}, fmt.Errorf("%s: %s", "invalid region in internal mapping", parsedECR.Region)
 	}
 
-	return ecr.ParseRef(fmt.Sprintf("%s%s:repository/%s", ecrRefPrefix, account, fullRepoPath))
+	return ecr.ParseRef(fmt.Sprintf("%s%s:repository/%s", ecrRefPrefix, parsedECR.Account, parsedECR.RepoPath))
 }
 
 // fetchECRRef attempts to resolve the ECR reference from an input source string
@@ -668,7 +702,7 @@ func parseImageURISpecialRegions(input string) (ecr.ECRSpec, error) {
 // attempt to parse again using parseImageURISpecialRegions in this package.
 // This uses a special region reference to build the ECR image references.
 // If both fail, an error is returned.
-func fetchECRRef(ctx context.Context, input string) (ecr.ECRSpec, error) {
+func fetchECRRef(ctx context.Context, input string, specialRegions specialRegions) (ecr.ECRSpec, error) {
 	var spec ecr.ECRSpec
 	spec, err := ecr.ParseImageURI(input)
 	if err == nil {
@@ -677,7 +711,7 @@ func fetchECRRef(ctx context.Context, input string) (ecr.ECRSpec, error) {
 	log.G(ctx).WithError(err).WithField("source", input).Warn("failed to parse ECR reference")
 
 	// The parsing might fail if the AWS region is special, parse again with special handling:
-	spec, err = parseImageURISpecialRegions(input)
+	spec, err = parseImageURISpecialRegions(input, specialRegions)
 	if err == nil {
 		return spec, nil
 	}
@@ -691,7 +725,11 @@ func fetchECRRef(ctx context.Context, input string) (ecr.ECRSpec, error) {
 
 // fetchECRImage does some additional conversions before resolving the image reference and fetches the image.
 func fetchECRImage(ctx context.Context, source string, client *containerd.Client, registryConfigPath string, fetchCachedImageIfExist bool, labels map[string]string) (containerd.Image, error) {
-	ecrRef, err := fetchECRRef(ctx, source)
+	specialRegions := specialRegions{
+		EcrRefPrefixMappings:    ecrRefPrefixMapping,
+		FipsSupportedEcrRegions: fipsSupportedEcrRegionSet,
+	}
+	ecrRef, err := fetchECRRef(ctx, source, specialRegions)
 	if err != nil {
 		return nil, err
 	}
@@ -851,6 +889,12 @@ func withDefaultMounts(containerID string, persistentDir string) oci.SpecOpts {
 			Destination: "/etc/bottlerocket-release",
 			Source:      "/etc/os-release",
 		},
+		// Bottlerocket host certs for the container
+		{
+			Options:     []string{"bind", "ro"},
+			Destination: "/.bottlerocket/certs",
+			Source:      "/etc/pki/tls/certs",
+		},
 		// Bottlerocket RPM inventory available to the container
 		{
 			Options:     []string{"bind", "ro"},