Skip to content

Commit

Permalink
Add v3 support
Browse files Browse the repository at this point in the history 
This does a major refactor on the compose.go functions as well as brings
in a new era of v3 support to Kompose.

Similar to how we utilize libcompose, we utilize docker/cli's "stack
deploy" code which has a built-in v3 parser. We convert the parsed
structure to our own and then convert it to Kubernetes/OpenShift
artifacts.
  • Loading branch information
@cdrage
cdrage committed Jun 13, 2017
1 parent 683ae91 commit 2b58307
Show file tree
Hide file tree
Showing 18 changed files with 2,871 additions and 260 deletions.
303 changes: 43 additions & 260 deletions pkg/loader/compose/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,13 @@ package compose

import (
"fmt"
"net"
"os"
"path/filepath"
"io/ioutil"
"reflect"
"strconv"
"strings"

"k8s.io/kubernetes/pkg/api"
yaml "gopkg.in/yaml.v2"

log "github.com/Sirupsen/logrus"
"github.com/docker/libcompose/config"
"github.com/docker/libcompose/lookup"
"github.com/docker/libcompose/project"
"github.com/fatih/structs"
"github.com/kubernetes-incubator/kompose/pkg/kobject"
Expand Down Expand Up @@ -134,277 +129,65 @@ func checkUnsupportedKey(composeProject *project.Project) []string {
return keysFound
}

// load environment variables from compose file
func loadEnvVars(envars []string) []kobject.EnvVar {
envs := []kobject.EnvVar{}
for _, e := range envars {
character := ""
equalPos := strings.Index(e, "=")
colonPos := strings.Index(e, ":")
switch {
case equalPos == -1 && colonPos == -1:
character = ""
case equalPos == -1 && colonPos != -1:
character = ":"
case equalPos != -1 && colonPos == -1:
character = "="
case equalPos != -1 && colonPos != -1:
if equalPos > colonPos {
character = ":"
} else {
character = "="
}
}

if character == "" {
envs = append(envs, kobject.EnvVar{
Name: e,
Value: os.Getenv(e),
})
} else {
values := strings.SplitN(e, character, 2)
// try to get value from os env
if values[1] == "" {
values[1] = os.Getenv(values[0])
}
envs = append(envs, kobject.EnvVar{
Name: values[0],
Value: values[1],
})
}
}

return envs
}

// Load ports from compose file
func loadPorts(composePorts []string) ([]kobject.Ports, error) {
ports := []kobject.Ports{}
character := ":"
// LoadFile loads a compose file into KomposeObject
func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {

// For each port listed
for _, port := range composePorts {
// Load the json / yaml file in order to get the version value
var version string

// Get the TCP / UDP protocol. Checks to see if it splits in 2 with '/' character.
// ex. 15000:15000/tcp
// else, set a default protocol of using TCP
proto := api.ProtocolTCP
protocolCheck := strings.Split(port, "/")
if len(protocolCheck) == 2 {
if strings.EqualFold("tcp", protocolCheck[1]) {
proto = api.ProtocolTCP
} else if strings.EqualFold("udp", protocolCheck[1]) {
proto = api.ProtocolUDP
} else {
return nil, fmt.Errorf("invalid protocol %q", protocolCheck[1])
}
for _, file := range files {
composeVersion, err := getVersionFromFile(file)
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "Unable to load yaml/json file for version parsing")
}

// Split up the ports / IP without the "/tcp" or "/udp" appended to it
justPorts := strings.Split(protocolCheck[0], character)

if len(justPorts) == 3 {
// ex. 127.0.0.1:80:80

// Get the IP address
hostIP := justPorts[0]
ip := net.ParseIP(hostIP)
if ip.To4() == nil && ip.To16() == nil {
return nil, fmt.Errorf("%q contains an invalid IPv4 or IPv6 IP address", port)
}

// Get the host port
hostPortInt, err := strconv.Atoi(justPorts[1])
if err != nil {
return nil, fmt.Errorf("invalid host port %q valid example: 127.0.0.1:80:80", port)
}

// Get the container port
containerPortInt, err := strconv.Atoi(justPorts[2])
if err != nil {
return nil, fmt.Errorf("invalid container port %q valid example: 127.0.0.1:80:80", port)
}

// Convert to a kobject struct with ports as well as IP
ports = append(ports, kobject.Ports{
HostPort: int32(hostPortInt),
ContainerPort: int32(containerPortInt),
HostIP: hostIP,
Protocol: proto,
})

} else if len(justPorts) == 2 {
// ex. 80:80

// Get the host port
hostPortInt, err := strconv.Atoi(justPorts[0])
if err != nil {
return nil, fmt.Errorf("invalid host port %q valid example: 80:80", port)
}

// Get the container port
containerPortInt, err := strconv.Atoi(justPorts[1])
if err != nil {
return nil, fmt.Errorf("invalid container port %q valid example: 80:80", port)
}

// Convert to a kobject struct and add to the list of ports
ports = append(ports, kobject.Ports{
HostPort: int32(hostPortInt),
ContainerPort: int32(containerPortInt),
Protocol: proto,
})

} else {
// ex. 80

containerPortInt, err := strconv.Atoi(justPorts[0])
if err != nil {
return nil, fmt.Errorf("invalid container port %q valid example: 80", port)
}
ports = append(ports, kobject.Ports{
ContainerPort: int32(containerPortInt),
Protocol: proto,
})
// Check that the previous file loaded matches.
if len(files) > 0 && version != "" && version != composeVersion {
return kobject.KomposeObject{}, errors.New("All Docker Compose files must be of the same version")
}

}
return ports, nil
}

// LoadFile loads compose file into KomposeObject
func (c *Compose) LoadFile(files []string) (kobject.KomposeObject, error) {
komposeObject := kobject.KomposeObject{
ServiceConfigs: make(map[string]kobject.ServiceConfig),
LoadedFrom: "compose",
version = composeVersion
}
context := &project.Context{}
context.ComposeFiles = files

if context.ResourceLookup == nil {
context.ResourceLookup = &lookup.FileResourceLookup{}
}
log.Debugf("Docker Compose version: %s", version)

if context.EnvironmentLookup == nil {
cwd, err := os.Getwd()
// Convert based on version
switch version {
// Use libcompose for 1 or 2
// If blank, it's assumed it's 1 or 2
case "", "1", "1.0", "2", "2.0":
komposeObject, err := parseV1V2(files)
if err != nil {
return kobject.KomposeObject{}, nil
return kobject.KomposeObject{}, err
}
context.EnvironmentLookup = &lookup.ComposableEnvLookup{
Lookups: []config.EnvironmentLookup{
&lookup.EnvfileLookup{
Path: filepath.Join(cwd, ".env"),
},
&lookup.OsEnvLookup{},
},
return komposeObject, nil
// Use docker/cli for 3
case "3", "3.0":
komposeObject, err := parseV3(files)
if err != nil {
return kobject.KomposeObject{}, err
}
return komposeObject, nil
default:
return kobject.KomposeObject{}, fmt.Errorf("Version %s of Docker Compose is not supported. Please use version 1, 2 or 3", version)
}

// load compose file into composeObject
composeObject := project.NewProject(context, nil, nil)
err := composeObject.Parse()
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "composeObject.Parse() failed, Failed to load compose file")
}
}

noSupKeys := checkUnsupportedKey(composeObject)
for _, keyName := range noSupKeys {
log.Warningf("Unsupported %s key - ignoring", keyName)
func getVersionFromFile(file string) (string, error) {
type ComposeVersion struct {
Version string `json:"version"` // This affects YAML as well
}
var version ComposeVersion

for name, composeServiceConfig := range composeObject.ServiceConfigs.All() {
serviceConfig := kobject.ServiceConfig{}
serviceConfig.Image = composeServiceConfig.Image
serviceConfig.Build = composeServiceConfig.Build.Context
newName := normalizeServiceNames(composeServiceConfig.ContainerName)
serviceConfig.ContainerName = newName
if newName != composeServiceConfig.ContainerName {
log.Infof("Container name in service %q has been changed from %q to %q", name, composeServiceConfig.ContainerName, newName)
}
serviceConfig.Command = composeServiceConfig.Entrypoint
serviceConfig.Args = composeServiceConfig.Command
serviceConfig.Dockerfile = composeServiceConfig.Build.Dockerfile
serviceConfig.BuildArgs = composeServiceConfig.Build.Args

envs := loadEnvVars(composeServiceConfig.Environment)
serviceConfig.Environment = envs

//Validate dockerfile path
if filepath.IsAbs(serviceConfig.Dockerfile) {
log.Fatalf("%q defined in service %q is an absolute path, it must be a relative path.", serviceConfig.Dockerfile, name)
}

// load ports
ports, err := loadPorts(composeServiceConfig.Ports)
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "loadPorts failed. "+name+" failed to load ports from compose file")
}
serviceConfig.Port = ports

serviceConfig.WorkingDir = composeServiceConfig.WorkingDir

if composeServiceConfig.Volumes != nil {
for _, volume := range composeServiceConfig.Volumes.Volumes {
v := normalizeServiceNames(volume.String())
serviceConfig.Volumes = append(serviceConfig.Volumes, v)
}
}

// canonical "Custom Labels" handler
// Labels used to influence conversion of kompose will be handled
// from here for docker-compose. Each loader will have such handler.
for key, value := range composeServiceConfig.Labels {
switch key {
case "kompose.service.type":
serviceType, err := handleServiceType(value)
if err != nil {
return kobject.KomposeObject{}, errors.Wrap(err, "handleServiceType failed")
}

serviceConfig.ServiceType = serviceType
case "kompose.service.expose":
serviceConfig.ExposeService = strings.ToLower(value)
}
}

// convert compose labels to annotations
serviceConfig.Annotations = map[string]string(composeServiceConfig.Labels)
serviceConfig.CPUQuota = int64(composeServiceConfig.CPUQuota)
serviceConfig.CapAdd = composeServiceConfig.CapAdd
serviceConfig.CapDrop = composeServiceConfig.CapDrop
serviceConfig.Pid = composeServiceConfig.Pid
serviceConfig.Expose = composeServiceConfig.Expose
serviceConfig.Privileged = composeServiceConfig.Privileged
serviceConfig.Restart = composeServiceConfig.Restart
serviceConfig.User = composeServiceConfig.User
serviceConfig.VolumesFrom = composeServiceConfig.VolumesFrom
serviceConfig.Stdin = composeServiceConfig.StdinOpen
serviceConfig.Tty = composeServiceConfig.Tty
serviceConfig.MemLimit = composeServiceConfig.MemLimit
serviceConfig.TmpFs = composeServiceConfig.Tmpfs
serviceConfig.StopGracePeriod = composeServiceConfig.StopGracePeriod
komposeObject.ServiceConfigs[normalizeServiceNames(name)] = serviceConfig
if normalizeServiceNames(name) != name {
log.Infof("Service name in docker-compose has been changed from %q to %q", name, normalizeServiceNames(name))
}
loadedFile, err := ioutil.ReadFile(file)
if err != nil {
return "", err
}

return komposeObject, nil
}

func handleServiceType(ServiceType string) (string, error) {
switch strings.ToLower(ServiceType) {
case "", "clusterip":
return string(api.ServiceTypeClusterIP), nil
case "nodeport":
return string(api.ServiceTypeNodePort), nil
case "loadbalancer":
return string(api.ServiceTypeLoadBalancer), nil
default:
return "", errors.New("Unknown value " + ServiceType + " , supported values are 'NodePort, ClusterIP or LoadBalancer'")
err = yaml.Unmarshal(loadedFile, &version)
if err != nil {
return "", err
}
}

func normalizeServiceNames(svcName string) string {
return strings.Replace(svcName, "_", "-", -1)
return version.Version, nil
}
37 changes: 37 additions & 0 deletions pkg/loader/compose/compose_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,49 @@ import (
"github.com/kubernetes-incubator/kompose/pkg/kobject"
"k8s.io/kubernetes/pkg/api"

"github.com/docker/cli/cli/compose/types"
"github.com/docker/libcompose/config"
"github.com/docker/libcompose/project"
"github.com/docker/libcompose/yaml"
"github.com/pkg/errors"
)

func TestLoadV3Volumes(t *testing.T) {
vol := types.ServiceVolumeConfig{
Type: "volume",
Source: "/tmp/foobar",
Target: "/tmp/foobar",
ReadOnly: true,
}
volumes := []types.ServiceVolumeConfig{vol}
output := loadV3Volumes(volumes)
expected := "/tmp/foobar:/tmp/foobar:ro"

if output[0] != expected {
t.Errorf("Expected %s, got %s", expected, output[0])
}

}

func TestLoadV3Ports(t *testing.T) {
port := types.ServicePortConfig{
Target: 80,
Published: 80,
Protocol: "TCP",
}
ports := []types.ServicePortConfig{port}
output := loadV3Ports(ports)
expected := kobject.Ports{
HostPort: 80,
ContainerPort: 80,
Protocol: api.Protocol("TCP"),
}

if output[0] != expected {
t.Errorf("Expected %s, got %s", expected, output[0])
}
}

// Test if service types are parsed properly on user input
// give a service type and expect correct input
func TestHandleServiceType(t *testing.T) {
Expand Down
Loading

0 comments on commit 2b58307

Please sign in to comment.