Add support for uploading an ostree commit to Pulp

cmd/osbuild-upload-pulp-ostree/main.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/osbuild/osbuild-composer/internal/upload/pulp"
+)
+
+func check(err error) {
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}
+
+func main() {
+	var filename, apiURL, repository, basePath, credsFile string
+	flag.StringVar(&filename, "archive", "", "ostree archive to upload")
+	flag.StringVar(&apiURL, "url", "", "server URL")
+	flag.StringVar(&repository, "repository", "", "repository name")
+	flag.StringVar(&basePath, "base-path", "", "base path for distribution (if the repository does not already exist)")
+	flag.StringVar(&credsFile, "credentials", "", `file containing credentials (format: {"username": "...", "password": "..."})`)
+	flag.Parse()
+
+	client, err := pulp.NewClientFromFile(apiURL, credsFile)
+	check(err)
+
+	repoURL, err := client.UploadAndDistributeCommit(filename, repository, basePath)
+	check(err)
+
+	fmt.Printf("The commit is available in the repository at %s\n", repoURL)
+}

cmd/osbuild-worker/config.go

@@ -66,6 +66,11 @@ type containersConfig struct {
 	TLSVerify    bool   `toml:"tls_verify"`
 }
 
+type pulpConfig struct {
+	Credentials string `toml:"credentials"`
+	ServerURL   string `toml:"server_address"`
+}
+
 type workerConfig struct {
 	Composer       *composerConfig             `toml:"composer"`
 	Koji           map[string]kojiServerConfig `toml:"koji"`
@@ -76,6 +81,7 @@ type workerConfig struct {
 	Authentication *authenticationConfig       `toml:"authentication"`
 	Containers     *containersConfig           `toml:"containers"`
 	OCI            *ociConfig                  `toml:"oci"`
+	Pulp           *pulpConfig                 `toml:"pulp"`
 	// default value: /api/worker/v1
 	BasePath string `toml:"base_path"`
 	DNFJson  string `toml:"dnf-json"`

cmd/osbuild-worker/config_test.go

@@ -65,6 +65,10 @@ oauth_url = "https://example.com/token"
 client_id = "toucan"
 client_secret = "/etc/osbuild-worker/client_secret"
 offline_token = "/etc/osbuild-worker/offline_token"
+
+[pulp]
+credentials = "/etc/osbuild-worker/pulp-creds"
+server_address = "https://example.com/pulp"
 `,
 			want: &workerConfig{
 				BasePath: "/api/image-builder-worker/v1",
@@ -116,6 +120,10 @@ offline_token = "/etc/osbuild-worker/offline_token"
 					ClientId:         "toucan",
 					ClientSecretPath: "/etc/osbuild-worker/client_secret",
 				},
+				Pulp: &pulpConfig{
+					Credentials: "/etc/osbuild-worker/pulp-creds",
+					ServerURL:   "https://example.com/pulp",
+				},
 			},
 		},
 		{

cmd/osbuild-worker/jobimpl-osbuild.go

@@ -12,12 +12,14 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/osbuild"
 
 	"github.com/osbuild/osbuild-composer/internal/upload/oci"
+	"github.com/osbuild/osbuild-composer/internal/upload/pulp"
 
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
@@ -67,6 +69,11 @@ type OCIConfiguration struct {
 	Namespace    string
 }
 
+type PulpConfiguration struct {
+	CredsFilePath string
+	ServerAddress string
+}
+
 type OSBuildJobImpl struct {
 	Store            string
 	Output           string
@@ -78,6 +85,7 @@ type OSBuildJobImpl struct {
 	AWSBucket        string
 	S3Config         S3Configuration
 	ContainersConfig ContainersConfiguration
+	PulpConfig       PulpConfiguration
 }
 
 // Returns an *awscloud.AWS object with the credentials of the request. If they
@@ -293,6 +301,44 @@ func (impl *OSBuildJobImpl) getContainerClient(destination string, targetOptions
 	return client, nil
 }
 
+// Read server configuration and credentials from the target options and fall
+// back to worker config if they are not set (targetOptions take precedent).
+// Mixing sources is allowed. For example, the server address can be configured
+// in the worker config while the targetOptions provide the credentials (or
+// vice versa).
+func (impl *OSBuildJobImpl) getPulpClient(targetOptions *target.PulpOSTreeTargetOptions) (*pulp.Client, error) {
+
+	var creds *pulp.Credentials
+	// Credentials are considered together. In other words, the username can't
+	// come from a different config source than the password.
+	if targetOptions.Username != "" && targetOptions.Password != "" {
+		creds = &pulp.Credentials{
+			Username: targetOptions.Username,
+			Password: targetOptions.Password,
+		}
+	}
+	address := targetOptions.ServerAddress
+	if address == "" {
+		// fall back to worker configuration for server address
+		address = impl.PulpConfig.ServerAddress
+	}
+	if address == "" {
+		return nil, fmt.Errorf("pulp server address not set")
+	}
+
+	if creds != nil {
+		return pulp.NewClient(address, creds), nil
+	}
+
+	// read from worker configuration
+	if impl.PulpConfig.CredsFilePath == "" {
+		return nil, fmt.Errorf("pulp credentials not set")
+	}
+
+	// use creds file loader helper
+	return pulp.NewClientFromFile(address, impl.PulpConfig.CredsFilePath)
+}
+
 func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 	logWithId := logrus.WithField("jobId", job.Id().String())
 	// Initialize variable needed for reporting back to osbuild-composer.
@@ -1101,6 +1147,23 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 			logWithId.Printf("[container] 🎉 Image uploaded (%s)!", digest.String())
 			targetResult.Options = &target.ContainerTargetResultOptions{URL: client.Target.String(), Digest: digest.String()}
 
+		case *target.PulpOSTreeTargetOptions:
+			targetResult = target.NewPulpOSTreeTargetResult(nil, &artifact)
+			archivePath := filepath.Join(outputDirectory, jobTarget.OsbuildArtifact.ExportName, jobTarget.OsbuildArtifact.ExportFilename)
+
+			client, err := impl.getPulpClient(targetOptions)
+			if err != nil {
+				targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidConfig, err.Error(), nil)
+				break
+			}
+
+			url, err := client.UploadAndDistributeCommit(archivePath, targetOptions.Repository, targetOptions.BasePath)
+			if err != nil {
+				targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, err.Error(), nil)
+				break
+			}
+			targetResult.Options = &target.PulpOSTreeTargetResultOptions{RepoURL: url}
+
 		default:
 			// TODO: we may not want to return completely here with multiple targets, because then no TargetErrors will be added to the JobError details
 			// Nevertheless, all target errors will be still in the OSBuildJobResult.

cmd/osbuild-worker/main.go

@@ -426,6 +426,13 @@ func main() {
 		}
 	}
 
+	var pulpCredsFilePath = ""
+	var pulpAddress = ""
+	if config.Pulp != nil {
+		pulpCredsFilePath = config.Pulp.Credentials
+		pulpAddress = config.Pulp.ServerURL
+	}
+
 	// depsolve jobs can be done during other jobs
 	depsolveCtx, depsolveCtxCancel := context.WithCancel(context.Background())
 	solver := dnfjson.NewBaseSolver(rpmmd_cache)
@@ -487,6 +494,10 @@ func main() {
 				CertPath:     containersCertPath,
 				TLSVerify:    &containersTLSVerify,
 			},
+			PulpConfig: PulpConfiguration{
+				CredsFilePath: pulpCredsFilePath,
+				ServerAddress: pulpAddress,
+			},
 		},
 		worker.JobTypeKojiInit: &KojiInitJobImpl{
 			KojiServers: kojiServers,

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/openshift-online/ocm-sdk-go v0.1.373
 	github.com/oracle/oci-go-sdk/v54 v54.0.0
 	github.com/osbuild/images v0.12.0
+	github.com/osbuild/pulp-client v0.1.0
 	github.com/prometheus/client_golang v1.17.0
 	github.com/segmentio/ksuid v1.0.4
 	github.com/sirupsen/logrus v1.9.3

go.sum

@@ -582,6 +582,8 @@ github.com/oracle/oci-go-sdk/v54 v54.0.0 h1:CDLjeSejv2aDpElAJrhKpi6zvT/zhZCZuXch
 github.com/oracle/oci-go-sdk/v54 v54.0.0/go.mod h1:+t+yvcFGVp+3ZnztnyxqXfQDsMlq8U25faBLa+mqCMc=
 github.com/osbuild/images v0.12.0 h1:NJoZBAThfUhJMbBmHdPmpcimnIK7r9/ItKUi1qDaPBo=
 github.com/osbuild/images v0.12.0/go.mod h1:shwiFpnqCqKFAVR+Polou8BwzzskOmqU7w3ItgSohxM=
+github.com/osbuild/pulp-client v0.1.0 h1:L0C4ezBJGTamN3BKdv+rKLuq/WxXJbsFwz/Hj7aEmJ8=
+github.com/osbuild/pulp-client v0.1.0/go.mod h1:rd/MLdfwwO2cQI1s056h8z32zAi3Bo90XhlAAryIvWc=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

internal/target/pulp_target.go

@@ -0,0 +1,33 @@
+package target
+
+const TargetNamePulpOSTree TargetName = "org.osbuild.pulp.ostree"
+
+type PulpOSTreeTargetOptions struct {
+	// ServerAddress for the pulp instance
+	ServerAddress string `json:"server_address,omitempty"`
+
+	// Repository to import the ostree commit to
+	Repository string `json:"repository"`
+
+	// BasePath for distributing the repository (if new)
+	BasePath string `json:"basepath,omitempty"`
+
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+}
+
+func (PulpOSTreeTargetOptions) isTargetOptions() {}
+
+func NewPulpOSTreeTarget(options *PulpOSTreeTargetOptions) *Target {
+	return newTarget(TargetNamePulpOSTree, options)
+}
+
+type PulpOSTreeTargetResultOptions struct {
+	RepoURL string `json:"repository_url"`
+}
+
+func (PulpOSTreeTargetResultOptions) isTargetResultOptions() {}
+
+func NewPulpOSTreeTargetResult(options *PulpOSTreeTargetResultOptions, artifact *OsbuildArtifact) *TargetResult {
+	return newTargetResult(TargetNamePulpOSTree, options, artifact)
+}

internal/target/target.go

@@ -93,6 +93,8 @@ func (target *Target) UnmarshalJSON(data []byte) error {
 		options = new(ContainerTargetOptions)
 	case TargetNameWorkerServer:
 		options = new(WorkerServerTargetOptions)
+	case TargetNamePulpOSTree:
+		options = new(PulpOSTreeTargetOptions)
 	default:
 		return fmt.Errorf("unexpected target name: %s", rawTarget.Name)
 	}
@@ -278,6 +280,10 @@ func (target Target) MarshalJSON() ([]byte, error) {
 			// the incompatible change.
 			rawOptions, err = json.Marshal(target.Options)
 
+		case *PulpOSTreeTargetOptions:
+			// added after incompatibility change
+			rawOptions, err = json.Marshal(target.Options)
+
 		default:
 			return nil, fmt.Errorf("unexpected target options type: %t", t)
 		}

internal/target/targetresult.go

@@ -76,6 +76,8 @@ func UnmarshalTargetResultOptions(trName TargetName, rawOptions json.RawMessage)
 		options = new(OCIObjectStorageTargetResultOptions)
 	case TargetNameContainer:
 		options = new(ContainerTargetResultOptions)
+	case TargetNamePulpOSTree:
+		options = new(PulpOSTreeTargetResultOptions)
 	default:
 		return nil, fmt.Errorf("unexpected target result name: %s", trName)
 	}