diff --git a/.changelog/24215.txt b/.changelog/24215.txt new file mode 100644 index 00000000000..eb1d22af1ab --- /dev/null +++ b/.changelog/24215.txt @@ -0,0 +1,3 @@ +```release-note:bug +docker: Fix incorrect auth parsing for private registries +``` diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 86b766893f4..da395e6cc9f 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -2481,6 +2481,7 @@ func TestDockerDriver_AuthConfiguration(t *testing.T) { { Repo: "redis:7", AuthConfig: ®istry.AuthConfig{ + Auth: "eyJ1c2VybmFtZSI6InRlc3QiLCJwYXNzd29yZCI6IjEyMzQifQ==", Username: "test", Password: "1234", Email: "", @@ -2490,6 +2491,7 @@ func TestDockerDriver_AuthConfiguration(t *testing.T) { { Repo: "quay.io/redis:7", AuthConfig: ®istry.AuthConfig{ + Auth: "eyJ1c2VybmFtZSI6InRlc3QiLCJwYXNzd29yZCI6IjU2NzgifQ==", Username: "test", Password: "5678", Email: "", @@ -2499,6 +2501,7 @@ func TestDockerDriver_AuthConfiguration(t *testing.T) { { Repo: "other.io/redis:7", AuthConfig: ®istry.AuthConfig{ + Auth: "eyJ1c2VybmFtZSI6InRlc3QiLCJwYXNzd29yZCI6ImFiY2QifQ==", Username: "test", Password: "abcd", Email: "", @@ -2535,6 +2538,7 @@ func TestDockerDriver_AuthFromTaskConfig(t *testing.T) { ServerAddr: "www.foobar.com", }, AuthConfig: ®istry.AuthConfig{ + Auth: "eyJ1c2VybmFtZSI6ImZvbyIsInBhc3N3b3JkIjoiYmFyIn0=", Username: "foo", Password: "bar", Email: "foo@bar.com", @@ -2549,6 +2553,7 @@ func TestDockerDriver_AuthFromTaskConfig(t *testing.T) { ServerAddr: "www.foobar.com", }, AuthConfig: ®istry.AuthConfig{ + Auth: "eyJ1c2VybmFtZSI6ImZvbyIsInBhc3N3b3JkIjoiYmFyIn0=", Username: "foo", Password: "bar", ServerAddress: "www.foobar.com", diff --git a/drivers/docker/utils.go b/drivers/docker/utils.go index 8adc9ee0b08..77a120c5a2f 100644 --- a/drivers/docker/utils.go +++ b/drivers/docker/utils.go @@ -4,6 +4,7 @@ package docker import ( + "encoding/base64" "encoding/json" "fmt" "os" @@ -102,12 +103,19 @@ func authFromTaskConfig(driverConfig *TaskConfig) authBackend { if len(driverConfig.Auth.Username) == 0 && len(driverConfig.Auth.Password) == 0 && len(driverConfig.Auth.Email) == 0 && len(driverConfig.Auth.ServerAddr) == 0 { return nil, nil } - return ®istrytypes.AuthConfig{ + + authConfig := ®istrytypes.AuthConfig{ Username: driverConfig.Auth.Username, Password: driverConfig.Auth.Password, Email: driverConfig.Auth.Email, ServerAddress: driverConfig.Auth.ServerAddr, - }, nil + } + + if err := encodeAuth(authConfig); err != nil { + return nil, err + } + + return authConfig, nil } } @@ -140,6 +148,11 @@ func authFromDockerConfig(file string) authBackend { IdentityToken: dockerAuthConfig.IdentityToken, RegistryToken: dockerAuthConfig.RegistryToken, } + + if err := encodeAuth(auth); err != nil { + return nil, err + } + if authIsEmpty(auth) { return nil, nil } @@ -187,6 +200,9 @@ func authFromHelper(helperName string) authBackend { Username: response["Username"], Password: response["Secret"], } + if err := encodeAuth(auth); err != nil { + return nil, err + } if authIsEmpty(auth) { return nil, nil @@ -195,6 +211,21 @@ func authFromHelper(helperName string) authBackend { } } +// some docker api calls require a base64 encoded basic auth string +func encodeAuth(cfg *registrytypes.AuthConfig) error { + auth := ®istrytypes.AuthConfig{ + Username: cfg.Username, + Password: cfg.Password, + } + encodedJSON, err := json.Marshal(auth) + if err != nil { + return fmt.Errorf("error encoding basic auth: %v", err) + } + + cfg.Auth = base64.URLEncoding.EncodeToString(encodedJSON) + return nil +} + // authIsEmpty returns if auth is nil or an empty structure func authIsEmpty(auth *registrytypes.AuthConfig) bool { if auth == nil { diff --git a/e2e/docker/doc.go b/e2e/docker/doc.go new file mode 100644 index 00000000000..13a1965dccd --- /dev/null +++ b/e2e/docker/doc.go @@ -0,0 +1,5 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +// Package docker contains test cases related to the docker task driver. +package docker diff --git a/e2e/docker/docker_test.go b/e2e/docker/docker_test.go new file mode 100644 index 00000000000..5c9ae19a98b --- /dev/null +++ b/e2e/docker/docker_test.go @@ -0,0 +1,125 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package docker + +import ( + "fmt" + "strconv" + "testing" + "time" + + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/v3/cluster3" + "github.com/hashicorp/nomad/e2e/v3/jobs3" + "github.com/shoenig/test/must" +) + +const ( + registryService = "registry" +) + +func TestDocker(t *testing.T) { + cluster3.Establish(t, + cluster3.Leader(), + cluster3.LinuxClients(1), + ) + + runRegistry(t) + + t.Run("testRedis", testRedis) + t.Run("testAuthBasic", testAuthBasic) + t.Run("testAuthFileStatic", testAuthFileStatic) + t.Run("testAuthHelper", testAuthHelper) +} + +func findService(t *testing.T, name string) (string, int) { + services, _, err := e2eutil.NomadClient(t).Services().Get(name, nil) + must.NoError(t, err, must.Sprintf("failed to find %q service", name)) + must.Len(t, 1, services, must.Sprintf("expected 1 %q service", name)) + return services[0].Address, services[0].Port +} + +func runRegistry(t *testing.T) { + _, regCleanup := jobs3.Submit(t, + "../docker_registry/registry.hcl", + jobs3.Timeout(40*time.Second), // pulls an image + ) + t.Cleanup(regCleanup) + + // lookup registry address + addr, port := findService(t, registryService) + address := fmt.Sprintf("%s:%d", addr, port) + + t.Logf("Setting up insecure private registry at %v", address) + + // run the sed job to fixup the auth.json file with correct address and make + // sure the registry is marked as insecure for docker, otherwise pulls will + // fail + _, sedCleanup := jobs3.Submit(t, + "./input/registry-auths.hcl", + jobs3.Var("registry_address", address), + jobs3.Var("user", "root"), + jobs3.Var("helper_dir", "/usr/local/bin"), + jobs3.Var("auth_dir", "/etc"), + jobs3.Var("docker_conf_dir", "/etc/docker"), + jobs3.WaitComplete("create-files"), + jobs3.Timeout(20*time.Second), + ) + t.Cleanup(sedCleanup) +} + +func testRedis(t *testing.T) { + job, cleanup := jobs3.Submit(t, "./input/redis.hcl") + t.Cleanup(cleanup) + + logs := job.TaskLogs("cache", "redis") + must.StrContains(t, logs.Stdout, "oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo") +} + +func testAuthBasic(t *testing.T) { + // find the private registry service + regAddr, regPort := findService(t, "registry") + + // run the private bash image + bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_basic.hcl", + jobs3.Var("registry_address", regAddr), + jobs3.Var("registry_port", strconv.Itoa(regPort)), + jobs3.WaitComplete("basic"), + ) + t.Cleanup(bashCleanup) + logs := bashJob.TaskLogs("basic", "echo") + must.StrContains(t, logs.Stdout, "The auth basic test is OK!") +} + +func testAuthFileStatic(t *testing.T) { + // find the private registry service + regAddr, regPort := findService(t, "registry") + + // run the private _static bash image + bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_static.hcl", + jobs3.Var("registry_address", regAddr), + jobs3.Var("registry_port", strconv.Itoa(regPort)), + jobs3.WaitComplete("static"), + ) + t.Cleanup(bashCleanup) + logs := bashJob.TaskLogs("static", "echo") + must.StrContains(t, logs.Stdout, "The static auth test is OK!") +} + +func testAuthHelper(t *testing.T) { + // find the private registry service + regAddr, regPort := findService(t, "registry") + + t.Log("registry", regAddr, regPort) + + // run the private _helper bash image + bashJob, bashCleanup := jobs3.Submit(t, "./input/auth_helper.hcl", + jobs3.Var("registry_address", regAddr), + jobs3.Var("registry_port", strconv.Itoa(regPort)), + jobs3.WaitComplete("helper"), + ) + t.Cleanup(bashCleanup) + logs := bashJob.TaskLogs("helper", "echo") + must.StrContains(t, logs.Stdout, "The credentials helper auth test is OK!") +} diff --git a/e2e/docker/input/auth_basic.hcl b/e2e/docker/input/auth_basic.hcl new file mode 100644 index 00000000000..1123a5f6e82 --- /dev/null +++ b/e2e/docker/input/auth_basic.hcl @@ -0,0 +1,75 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This job runs a docker task using a container stored in a private registry +# configured with basic authentication. The registry.hcl job should be running +# and healthy before running this job. The registry_address and registry_port +# HCL variables must be provided. + +variable "registry_address" { + type = string + description = "The HTTP address of the local registry" + default = "localhost" +} + +variable "registry_port" { + type = number + description = "The HTTP port of the local registry" + default = "7511" +} + +variable "registry_username" { + type = string + description = "The Basic Auth username of the local registry" + default = "auth_basic_user" +} + +variable "registry_password" { + type = string + description = "The Basic Auth password of the local registry" + default = "auth_basic_pass" +} + +locals { + registry_auth = base64encode("${var.registry_username}:${var.registry_password}") +} + +job "auth_basic" { + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "basic" { + reschedule { + attempts = 0 + unlimited = false + } + + network { + mode = "host" + } + + task "echo" { + driver = "docker" + + config { + image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_basic:private" + args = ["echo", "The auth basic test is OK!"] + auth_soft_fail = true + + auth { + username = "${var.registry_username}" + password = "${var.registry_password}" + } + } + + resources { + cpu = 100 + memory = 64 + } + } + } +} diff --git a/e2e/docker/input/auth_helper.hcl b/e2e/docker/input/auth_helper.hcl new file mode 100644 index 00000000000..addd30110d7 --- /dev/null +++ b/e2e/docker/input/auth_helper.hcl @@ -0,0 +1,55 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This job runs a docker task using a container stored in a private registry +# configured with credentials helper authentication. The registry.hcl job should +# be running and healthy before running this job. + +variable "registry_address" { + type = string + description = "The HTTP address of the local registry" + default = "localhost" +} + +variable "registry_port" { + type = number + description = "The HTTP port of the local registry" + default = "7511" +} + +job "auth_static" { + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "helper" { + reschedule { + attempts = 0 + unlimited = false + } + + network { + mode = "host" + } + + task "echo" { + driver = "docker" + + config { + image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_helper:private" + args = ["echo", "The credentials helper auth test is OK!"] + + # usename and password come from [docker-credential-]test.sh found on + # $PATH as specified by "helper=test.sh" in plugin config + } + + resources { + cpu = 100 + memory = 64 + } + } + } +} diff --git a/e2e/docker/input/auth_static.hcl b/e2e/docker/input/auth_static.hcl new file mode 100644 index 00000000000..8de6ad0c498 --- /dev/null +++ b/e2e/docker/input/auth_static.hcl @@ -0,0 +1,65 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This job runs a docker task using a container stored in a private registry +# configured with file config static authentication. The registry.hcl job should +# be running and healthy before running this job. + +variable "registry_address" { + type = string + description = "The HTTP address of the local registry" + default = "localhost" +} + +variable "registry_port" { + type = number + description = "The HTTP port of the local registry" + default = "7511" +} + +job "auth_static" { + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "static" { + reschedule { + attempts = 0 + unlimited = false + } + + network { + mode = "host" + } + + task "echo" { + driver = "docker" + + config { + image = "${var.registry_address}:${var.registry_port}/docker.io/library/bash_auth_static:private" + args = ["echo", "The static auth test is OK!"] + + # usename and password come from auth.json in plugin config + } + + resources { + cpu = 100 + memory = 64 + } + } + } +} + +# auth.json (must be pointed to by config=/auth.json) +# +# { +# "auths": { +# "127.0.0.1:7511/docker.io/library/bash_auth_static": { +# "auth": "YXV0aF9zdGF0aWNfdXNlcjphdXRoX3N0YXRpY19wYXNz" +# } +# } +# } + diff --git a/e2e/docker/input/redis.hcl b/e2e/docker/input/redis.hcl new file mode 100644 index 00000000000..911b87c0d9e --- /dev/null +++ b/e2e/docker/input/redis.hcl @@ -0,0 +1,35 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This is a simple redis job using the docker task driver. + +job "redis" { + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "cache" { + network { + port "db" { + to = 6379 + } + } + + task "redis" { + driver = "docker" + + config { + image = "docker.io/library/redis:7" + ports = ["db"] + auth_soft_fail = true + } + + resources { + cpu = 50 + memory = 128 + } + } + } +} diff --git a/e2e/docker/input/registry-auths.hcl b/e2e/docker/input/registry-auths.hcl new file mode 100644 index 00000000000..b8c13415a5a --- /dev/null +++ b/e2e/docker/input/registry-auths.hcl @@ -0,0 +1,170 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# This job runs after the private registry is up and running, when we know +# address and port provided by the bridge network. It is a sysbatch job +# that writes these files on every linux client. +# - /usr/local/bin/docker-credential-test.sh +# - /etc/docker-registry-auth.json + +variable "registry_address" { + type = string + description = "The HTTP address of the local registry" +} + +variable "auth_dir" { + type = string + description = "The destination directory of the auth.json file." + default = "/tmp" +} + +variable "helper_dir" { + type = string + description = "The directory in which test.sh will be written." + default = "/tmp" +} + +variable "docker_conf_dir" { + type = string + description = "The directory in which daemon.json will be written." + default = "/tmp" +} + +variable "user" { + type = string + description = "The user to create files as. Should be root in e2e." + # no default because dealing with root files is annoying locally + # try -var=user=$USER for local development +} + +job "registry-auths" { + type = "sysbatch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "create-files" { + reschedule { + attempts = 0 + unlimited = false + } + + # write out the test.sh file into var.helper_dir + task "create-helper-file" { + driver = "pledge" + user = "${var.user}" + + config { + command = "cp" + args = ["${NOMAD_TASK_DIR}/test.sh", "${var.helper_dir}/docker-credential-test.sh"] + promises = "stdio rpath wpath cpath" + unveil = ["r:${NOMAD_TASK_DIR}/test.sh", "rwc:${var.helper_dir}"] + } + + template { + destination = "local/test.sh" + perms = "755" + data = <