Add integration test and update go version

* Use real CNI plugins to test Setup/Remove API. * Update go version from 1.16.x to 1.17.x in CI. * Remove coverage Signed-off-by: Wei Fu <[email protected]>
containerd · Feb 23, 2022 · 49aa5ab · 49aa5ab
1 parent e559bd8
commit 49aa5ab
Show file tree

Hide file tree

Showing 7 changed files with 600 additions and 5 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.16.x
+          go-version: 1.17.x
 
       - name: Set env
         shell: bash
@@ -33,14 +33,18 @@ jobs:
         with:
           working-directory: src/github.com/containerd/go-cni
 
+      - uses: containerd/project-checks@v1
+        with:
+          working-directory: src/github.com/containerd/go-cni/integration
+
   linters:
     name: Linters
     runs-on: ${{ matrix.os }}
     timeout-minutes: 10
 
     strategy:
       matrix:
-        go-version: [1.16.x]
+        go-version: [1.17.x]
         os: [ubuntu-18.04]
 
     steps:
@@ -71,7 +75,7 @@ jobs:
 
       - uses: actions/setup-go@v2
         with:
-          go-version: 1.16.x
+          go-version: 1.17.x
 
       - name: Set env
         shell: bash
@@ -80,6 +84,9 @@ jobs:
           echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
 
       - run: |
-          go test -v -race -covermode=atomic -coverprofile=coverage.txt  ./...
-          bash <(curl -s https://codecov.io/bash)
+          bash -x script/install-cni
+          make clean V=1
+          make bin/integration.test V=1
+          make test V=1
+          sudo make integration V=1
         working-directory: src/github.com/containerd/go-cni
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+/bin/
+coverage.txt
+profile.out
diff --git a/Makefile b/Makefile
@@ -0,0 +1,43 @@
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+TESTFLAGS_PARALLEL ?= 8
+
+EXTRA_TESTFLAGS ?=
+
+# quiet or not
+ifeq ($(V),1)
+	Q =
+else
+	Q = @
+endif
+
+.PHONY: test integration clean help
+
+help: ## this help
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
+
+test: ## run tests, except integration tests and tests that require root
+	$(Q)go test -v -race $(EXTRA_TESTFLAGS) -count=1 ./...
+
+integration: ## run integration test
+	$(Q)bin/integration.test -test.v -test.count=1 -test.root $(EXTRA_TESTFLAGS) -test.parallel $(TESTFLAGS_PARALLEL)
+
+FORCE:
+
+bin/integration.test: FORCE ## build integration test binary into bin
+	$(Q)cd ./integration && go test -race -c . -o ../bin/integration.test
+
+clean: ## clean up binaries
+	$(Q)rm -rf bin/
diff --git a/integration/cni_setup_teardown_linux_test.go b/integration/cni_setup_teardown_linux_test.go
@@ -0,0 +1,263 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Copyright 2018 CNI authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+//
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package integration
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"runtime"
+	"sync"
+	"syscall"
+	"testing"
+
+	"github.com/containerd/continuity/fs"
+	"github.com/containerd/continuity/testutil"
+	"github.com/containerd/go-cni"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	baseNetNSDir = "/var/run/netns/"
+
+	defaultCNIPluginDir = "/opt/cni/bin/"
+
+	cniBridgePluginCfg = `
+{
+  "cniVersion": "1.0.0",
+  "name": "gocni-test",
+  "plugins": [
+    {
+      "type":"bridge",
+      "bridge":"gocni-test0",
+      "isGateway":true,
+      "ipMasq":true,
+      "promiscMode":true,
+      "ipam":{
+        "type":"host-local",
+        "ranges":[
+          [{
+            "subnet":"10.88.0.0/16"
+          }],
+          [{
+            "subnet":"2001:4860:4860::/64"
+          }]
+        ],
+        "routes":[
+          {"dst":"0.0.0.0/0"},
+          {"dst":"::/0"}
+        ]
+       }
+    },
+    {
+      "type":"portmap",
+      "capabilities":{
+        "portMappings":true
+      }
+    }
+  ]
+}
+`
+)
+
+// TestBasicSetupAndRemove tests the cni.Setup/Remove with real bridge and
+// loopback CNI plugins.
+//
+// NOTE:
+//
+// 1. It required that the both bridge and loopback CNI plugins are installed
+// in /opt/cni/bin.
+//
+// 2. Since #76 enables parallel mode, we should enable -race option for this.
+func TestBasicSetupAndRemove(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	// setup config dir
+	tmpPluginConfDir, err := os.MkdirTemp("", t.Name()+"-conf")
+	assert.NoError(t, err, "create temp dir for plugin conf dir")
+	defer os.RemoveAll(tmpPluginConfDir)
+
+	assert.NoError(t,
+		ioutil.WriteFile(
+			path.Join(tmpPluginConfDir, "10-gocni-test-net.conflist"),
+			[]byte(cniBridgePluginCfg),
+			0600,
+		),
+		"init cni config",
+	)
+
+	// copy plugins from /opt/cni/bin
+	tmpPluginDir, err := os.MkdirTemp("", t.Name()+"-bin")
+	assert.NoError(t, err, "create temp dir for plugin bin dir")
+	defer os.RemoveAll(tmpPluginDir)
+
+	assert.NoError(t,
+		fs.CopyDir(tmpPluginDir, defaultCNIPluginDir),
+		"copy %v into %v", defaultCNIPluginDir, tmpPluginDir)
+
+	nsPath, done, err := createNetNS()
+	assert.NoError(t, err, "create temp netns")
+	defer func() {
+		assert.NoError(t, done(), "cleanup temp netns")
+	}()
+
+	defaultIfName := "eth0"
+	ctx := context.Background()
+	id := t.Name()
+
+	for idx, opts := range [][]cni.Opt{
+		// Use default plugin dir
+		{
+			cni.WithMinNetworkCount(2),
+			cni.WithPluginConfDir(tmpPluginConfDir),
+		},
+		// Use customize plugin dir
+		{
+			cni.WithMinNetworkCount(2),
+			cni.WithPluginConfDir(tmpPluginConfDir),
+			cni.WithPluginDir([]string{
+				tmpPluginDir,
+			}),
+		},
+	} {
+		l, err := cni.New(opts...)
+		assert.NoError(t, err, "[%v] initialize cni library", idx)
+
+		assert.NoError(t,
+			l.Load(cni.WithLoNetwork, cni.WithDefaultConf),
+			"[%v] load cni configuration", idx,
+		)
+
+		// Setup network
+		result, err := l.Setup(ctx, id, nsPath)
+		assert.NoError(t, err, "[%v] setup network for namespace %v", idx, nsPath)
+
+		ip := result.Interfaces[defaultIfName].IPConfigs[0].IP.String()
+		t.Logf("[%v] ip is %v", idx, ip)
+
+		assert.NoError(t,
+			l.Remove(ctx, id, nsPath),
+			"[%v] teardown network for namespace %v", idx, nsPath,
+		)
+	}
+}
+
+// createNetNS returns temp netns path.
+//
+// NOTE: It is based on https://github.com/containernetworking/plugins/blob/v1.0.1/pkg/testutils/netns_linux.go.
+// That can prevent from introducing unnessary dependencies in go.mod.
+func createNetNS() (_ string, _ func() error, retErr error) {
+	b := make([]byte, 16)
+	if _, err := rand.Reader.Read(b); err != nil {
+		return "", nil, fmt.Errorf("failed to generate random netns name: %w", err)
+	}
+
+	// Create the directory for mounting network namespaces
+	// This needs to be a shared mountpoint in case it is mounted in to
+	// other namespaces (containers)
+	if err := os.MkdirAll(baseNetNSDir, 0755); err != nil {
+		return "", nil, fmt.Errorf("failed to init base netns dir %s: %v", baseNetNSDir, err)
+	}
+
+	// create an empty file at the mount point
+	nsName := fmt.Sprintf("gocni-test-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+	nsPath := path.Join(baseNetNSDir, nsName)
+	mountPointFd, err := os.Create(nsPath)
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to create temp nspath %s: %v", nsPath, err)
+	}
+	mountPointFd.Close()
+
+	defer func() {
+		if retErr != nil {
+			_ = os.RemoveAll(nsPath)
+		}
+	}()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// do namespace work in a dedicated goroutine, so that we can safely
+	// Lock/Unlock OSThread without upsetting the lock/unlock state of
+	// the caller of this function
+	go (func() {
+		defer wg.Done()
+
+		// Don't unlock. By not unlocking, golang will kill the OS thread
+		// when the goroutine is done (>= go1.10). Since <= go1.10 has
+		// been deprecated, we don't need to get current net ns and
+		// reset.
+		runtime.LockOSThread()
+
+		// create a new netns on the current thread
+		if err = syscall.Unshare(syscall.CLONE_NEWNET); err != nil {
+			return
+		}
+
+		// bind mount the netns from the current thread (from /proc) onto the
+		// mount point. This causes the namespace to persist, even when there
+		// are no threads in the ns.
+		err = syscall.Mount(getCurrentThreadNetNSPath(), nsPath, "none", syscall.MS_BIND, "")
+		if err != nil {
+			err = fmt.Errorf("failed to bind mount ns at %s: %w", nsPath, err)
+		}
+	})()
+	wg.Wait()
+
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to create net namespace: %w", err)
+	}
+
+	return nsPath, func() error {
+		if err := syscall.Unmount(nsPath, 0); err != nil {
+			return fmt.Errorf("failed to unmount netns: at %s: %v", nsPath, err)
+		}
+
+		if err := os.Remove(nsPath); err != nil {
+			return fmt.Errorf("failed to remove nspath %s: %v", nsPath, err)
+		}
+		return nil
+	}, nil
+}
+
+// getCurrentThreadNetNSPath copied from pkg/ns
+//
+// NOTE: It is from https://github.com/containernetworking/plugins/blob/v1.0.1/pkg/testutils/netns_linux.go.
+func getCurrentThreadNetNSPath() string {
+	// /proc/self/ns/net returns the namespace of the main thread, not
+	// of whatever thread this goroutine is running on.  Make sure we
+	// use the thread's net namespace since the thread is switching around
+	return fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid())
+}
diff --git a/integration/go.mod b/integration/go.mod
@@ -0,0 +1,22 @@
+module github.com/containerd/go-cni/integration
+
+go 1.17
+
+require (
+	github.com/containerd/continuity v0.2.2
+	github.com/containerd/go-cni v0.0.0-00010101000000-000000000000
+	github.com/stretchr/testify v1.7.0
+)
+
+require (
+	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/containernetworking/cni v1.0.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.7.0 // indirect
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+)
+
+replace github.com/containerd/go-cni => ../