Skip to content
This repository has been archived by the owner on Sep 24, 2021. It is now read-only.

Support portmapping #162

Merged
merged 3 commits into from
Jun 9, 2017
Merged

Support portmapping #162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd4a194
support port mapping
heartlock Jun 7, 2017
3cc4b06
enable e2e test for port mapping
heartlock Jun 7, 2017
6f71d9e
update README
heartlock Jun 8, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 28 additions & 17 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,29 +52,40 @@ systemctl restart hyperd
Setup CNI networking using bridge plugin

```sh
$ sudo mkdir -p /etc/cni/net.d /opt/cni/bin
$ curl -sSL https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-v0.5.2.tgz -o cni-amd64-v0.5.2.tgz
$ sudo tar zxvf cni-amd64-v0.5.2.tgz -C /opt/cni/bin && rm -f cni-amd64-v0.5.2.tgz
$ sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conf <<-EOF
$ sudo mkdir -p /etc/cni/net.d /opt/cni/bin
$ git clone https://github.com/containernetworking/plugins $GOPATH/src/github.com/containernetworking/plugins
$ cd $GOPATH/src/github.com/containernetworking/plugins
$ ./build.sh
$ sudo cp bin/* /opt/cni/bin/
$ sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"name": "mynet",
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.10.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
"plugins": [
{
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.30.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": true
}
]
}
EOF'
$ sudo sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"type": "loopback"
}
EOF'
Expand Down
48 changes: 30 additions & 18 deletions hack/test-e2e.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,32 +88,44 @@ function install_remote_hyperd() {
function configure_cni() {
# install cni
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could you also update the CNI setup in README.md?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

will do

sudo mkdir -p /etc/cni/net.d /opt/cni/bin
curl -sSL https://github.com/containernetworking/cni/releases/download/${CNI_VERSION}/cni-amd64-${CNI_VERSION}.tgz -o cni.tgz
sudo tar zxvf cni.tgz -C /opt/cni/bin
rm -f cni.tgz

git clone https://github.com/containernetworking/plugins $GOPATH/src/github.com/containernetworking/plugins
cd $GOPATH/src/github.com/containernetworking/plugins

./build.sh
sudo cp bin/* /opt/cni/bin/

# create network configure file
sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conf <<-EOF
sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"name": "mynet",
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.30.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
"plugins": [
{
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.30.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": true
}
]
}
EOF'

sudo sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"type": "loopback"
}
EOF'
Expand All @@ -124,7 +136,7 @@ function test_cri() {
go get github.com/kubernetes-incubator/cri-tools/cmd/critest

# run critest
sudo env PATH=$PATH:$GOPATH/bin GOPATH=$GOPATH critest -r=/var/run/frakti.sock --focus="Conformance" --skip="container port" validation
sudo env PATH=$PATH:$GOPATH/bin GOPATH=$GOPATH critest -r=/var/run/frakti.sock --focus="Conformance" validation
}

FRAKTI_LISTEN_ADDR=${FRAKTI_LISTEN_ADDR:-/var/run/frakti.sock}
Expand Down
46 changes: 46 additions & 0 deletions pkg/hyper/network.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,19 @@ import (
cnitypes "github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/types/current"
"github.com/vishvananda/netlink"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
)

// cniPortMapping maps to the standard CNI portmapping Capability
// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md
type cniPortMapping struct {
HostPort int32 `json:"hostPort"`
ContainerPort int32 `json:"containerPort"`
Protocol string `json:"protocol"`
HostIP string `json:"hostIP"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hostIP,omitempty

}

// The network information needed to create HyperContainer
// network device from CNI Result
type NetworkInfo struct {
Expand All @@ -35,6 +46,41 @@ type NetworkInfo struct {
BridgeName string
}

func (h *Runtime) GetPodPortMappings(podID string) ([]*hostport.PortMapping, error) {
// TODO: get portmappings from docker labels for backward compatibility
checkpoint, err := h.checkpointHandler.GetCheckpoint(podID)
if err != nil {
return nil, err

}

portMappings := make([]*hostport.PortMapping, 0, len(checkpoint.Data.PortMappings))
for _, pm := range checkpoint.Data.PortMappings {
proto := toAPIProtocol(*pm.Protocol)
portMappings = append(portMappings, &hostport.PortMapping{
HostPort: *pm.HostPort,
ContainerPort: *pm.ContainerPort,
Protocol: proto,
})

}
return portMappings, nil

}

func toAPIProtocol(protocol Protocol) v1.Protocol {
switch protocol {
case ProtocolTCP:
return v1.ProtocolTCP
case ProtocolUDP:
return v1.ProtocolUDP

}
glog.Warningf("Unknown protocol %q: defaulting to TCP", protocol)
return v1.ProtocolTCP

}

func convertCNIResult2NetworkInfo(result cnitypes.Result) *NetworkInfo {
ret := &NetworkInfo{}

Expand Down
21 changes: 12 additions & 9 deletions pkg/hyper/ocicni/ocicni.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/containernetworking/cni/libcni"
cnitypes "github.com/containernetworking/cni/pkg/types"
"github.com/golang/glog"
kubeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
)

// TODO: upgrade CNI Plugin to its stable realease
Expand Down Expand Up @@ -155,12 +156,12 @@ func (plugin *cniNetworkPlugin) Name() string {
return CNIPluginName
}

func (plugin *cniNetworkPlugin) SetUpPod(podNetnsPath string, podID string) (cnitypes.Result, error) {
func (plugin *cniNetworkPlugin) SetUpPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) (cnitypes.Result, error) {
if err := plugin.checkInitialized(); err != nil {
return nil, err
}

res, err := plugin.getDefaultNetwork().addToNetwork(podNetnsPath, podID)
res, err := plugin.getDefaultNetwork().addToNetwork(podNetnsPath, podID, metadata, capabilities)
if err != nil {
glog.Errorf("Error while adding to cni network: %s", err)
return nil, err
Expand All @@ -170,16 +171,16 @@ func (plugin *cniNetworkPlugin) SetUpPod(podNetnsPath string, podID string) (cni
return res, nil
}

func (plugin *cniNetworkPlugin) TearDownPod(podNetnsPath string, podID string) error {
func (plugin *cniNetworkPlugin) TearDownPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) error {
if err := plugin.checkInitialized(); err != nil {
return err
}

return plugin.getDefaultNetwork().deleteFromNetwork(podNetnsPath, podID)
return plugin.getDefaultNetwork().deleteFromNetwork(podNetnsPath, podID, metadata, capabilities)
}

func (network *cniNetwork) addToNetwork(podNetnsPath string, podID string) (cnitypes.Result, error) {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID)
func (network *cniNetwork) addToNetwork(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, capabilities map[string]interface{}) (cnitypes.Result, error) {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID, metadata, capabilities)
if err != nil {
glog.Errorf("Pod: %s, Netns: %s, Error adding network: %v", podID, podNetnsPath, err)
return nil, err
Expand All @@ -196,8 +197,8 @@ func (network *cniNetwork) addToNetwork(podNetnsPath string, podID string) (cnit
return res, nil
}

func (network *cniNetwork) deleteFromNetwork(podNetnsPath string, podID string) error {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID)
func (network *cniNetwork) deleteFromNetwork(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, capabilities map[string]interface{}) error {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID, metadata, capabilities)
if err != nil {
glog.Errorf("Pod: %s, Netns: %s, Error deleting network: %v", podID, podNetnsPath, err)
return err
Expand All @@ -214,7 +215,7 @@ func (network *cniNetwork) deleteFromNetwork(podNetnsPath string, podID string)
return nil
}

func buildCNIRuntimeConf(podNetnsPath string, podID string) (*libcni.RuntimeConf, error) {
func buildCNIRuntimeConf(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, capabilities map[string]interface{}) (*libcni.RuntimeConf, error) {
glog.V(4).Infof("Got netns path %v", podNetnsPath)
glog.V(4).Infof("Using netns path %v", podNetnsPath)

Expand All @@ -225,7 +226,9 @@ func buildCNIRuntimeConf(podNetnsPath string, podID string) (*libcni.RuntimeConf
Args: [][2]string{
{"IgnoreUnknown", "1"},
{"K8S_POD_NAME", podID},
{"K8S_POD_NAMESPACE", metadata.GetNamespace()},
},
CapabilityArgs: capabilities,
}

return rt, nil
Expand Down
5 changes: 3 additions & 2 deletions pkg/hyper/ocicni/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package ocicni

import (
cnitypes "github.com/containernetworking/cni/pkg/types"
kubeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
)

const (
Expand All @@ -34,10 +35,10 @@ type CNIPlugin interface {
Name() string

// SetUpPod is the method called when the pod is created
SetUpPod(podNetnsPath string, podID string) (cnitypes.Result, error)
SetUpPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) (cnitypes.Result, error)

// TearDownPod is the method called before pod stopped
TearDownPod(podNetnsPath string, podID string) error
TearDownPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) error

// NetworkStatus returns error if the network plugin is in error state
Status() error
Expand Down
40 changes: 36 additions & 4 deletions pkg/hyper/sandbox.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,22 @@ func (h *Runtime) RunPodSandbox(config *kubeapi.PodSandboxConfig) (string, error
userpod.Labels["NETNS"] = netNsPath

// Setup the network
portMappings := config.GetPortMappings()
portMappingsParam := make([]cniPortMapping, 0, len(portMappings))
for _, p := range portMappings {
protocol := kubeapi.Protocol_name[int32(p.Protocol)]
portMappingsParam = append(portMappingsParam, cniPortMapping{
HostPort: p.HostPort,
ContainerPort: p.ContainerPort,
Protocol: strings.ToLower(protocol),
HostIP: p.HostIp,
})
}
capabilities := map[string]interface{}{
"portMappings": portMappingsParam,
}
podId := userpod.Id
result, err := h.netPlugin.SetUpPod(netNsPath, podId)
result, err := h.netPlugin.SetUpPod(netNsPath, podId, config.GetMetadata(), config.GetAnnotations(), capabilities)
if err != nil {
glog.Errorf("Setup network for sandbox %q by cni plugin failed: %v", config.String(), err)
}
Expand Down Expand Up @@ -93,7 +107,7 @@ func (h *Runtime) RunPodSandbox(config *kubeapi.PodSandboxConfig) (string, error
glog.Warningf("Remove pod %q failed: %v", removeError)
}
// destroy the network namespace
if tearError := h.netPlugin.TearDownPod(netNsPath, podID); tearError != nil {
if tearError := h.netPlugin.TearDownPod(netNsPath, podID, config.GetMetadata(), config.GetAnnotations(), capabilities); tearError != nil {
glog.Errorf("Destroy pod %s network namespace failed: %v", podID, tearError)
}
unix.Unmount(netNsPath, unix.MNT_DETACH)
Expand Down Expand Up @@ -190,12 +204,30 @@ func (h *Runtime) StopPodSandbox(podSandboxID string) error {
netNsPath = checkpoint.NetNsPath
}

// Get portMappings from checkpoint
portMappings, err := h.GetPodPortMappings(podSandboxID)
if err != nil {
return fmt.Errorf("could not retrieve port mappings: %v", err)
}
portMappingsParam := make([]cniPortMapping, 0, len(portMappings))
for _, p := range portMappings {
portMappingsParam = append(portMappingsParam, cniPortMapping{
HostPort: p.HostPort,
ContainerPort: p.ContainerPort,
Protocol: strings.ToLower(string(p.Protocol)),
HostIP: p.HostIP,
})
}
capabilities := map[string]interface{}{
"portMappings": portMappingsParam,
}

code, cause, err := h.client.StopPod(podSandboxID)
if err != nil {
glog.Errorf("Stop pod %s failed, code: %d, cause: %s, error: %v", podSandboxID, code, cause, err)
if isPodNotFoundError(err, podSandboxID) {
// destroy the network namespace
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID)
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID, status.GetMetadata(), status.GetAnnotations(), capabilities)
if err != nil {
glog.Errorf("Destroy pod %s network namespace failed: %v", podSandboxID, err)
}
Expand All @@ -213,7 +245,7 @@ func (h *Runtime) StopPodSandbox(podSandboxID string) error {
}

// destroy the network namespace
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID)
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID, status.GetMetadata(), status.GetAnnotations(), capabilities)
if err != nil {
glog.Errorf("Destroy pod %s network namespace failed: %v", podSandboxID, err)
}
Expand Down