Merge pull request #8860 E2E: rescheduling tests

e2e/e2e_test.go

@@ -22,6 +22,7 @@ import (
 	_ "github.com/hashicorp/nomad/e2e/nomad09upgrade"
 	_ "github.com/hashicorp/nomad/e2e/nomadexec"
 	_ "github.com/hashicorp/nomad/e2e/podman"
+	_ "github.com/hashicorp/nomad/e2e/rescheduling"
 	_ "github.com/hashicorp/nomad/e2e/spread"
 	_ "github.com/hashicorp/nomad/e2e/systemsched"
 	_ "github.com/hashicorp/nomad/e2e/taskevents"

e2e/e2eutil/cli.go

@@ -0,0 +1,101 @@
+package e2eutil
+
+import (
+	"fmt"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+// Command sends a command line argument to Nomad and returns the unbuffered
+// stdout as a string (or, if there's an error, the stderr)
+func Command(cmd string, args ...string) (string, error) {
+	out, err := exec.Command(cmd, args...).CombinedOutput()
+	return string(out), err
+}
+
+// GetField returns the value of an output field (ex. the "Submit Date" field
+// of `nomad job status :id`)
+func GetField(output, key string) (string, error) {
+	re := regexp.MustCompile(`(?m)^` + key + ` += (.*)$`)
+	match := re.FindStringSubmatch(output)
+	if match == nil {
+		return "", fmt.Errorf("could not find field %q", key)
+	}
+	return match[1], nil
+}
+
+// GetSection returns a section, with its field header but without its title.
+// (ex. the Allocations section of `nomad job status :id`)
+func GetSection(output, key string) (string, error) {
+
+	// golang's regex engine doesn't support negative lookahead, so
+	// we can't stop at 2 newlines if we also want a section that includes
+	// single newlines. so split on the section title, and then split a second time
+	// on \n\n
+	re := regexp.MustCompile(`(?ms)^` + key + `\n(.*)`)
+	match := re.FindStringSubmatch(output)
+	if match == nil {
+		return "", fmt.Errorf("could not find section %q", key)
+	}
+	tail := match[1]
+	return strings.Split(tail, "\n\n")[0], nil
+}
+
+// ParseColumns maps the CLI output for a columized section (without title) to
+// a slice of key->value pairs for each row in that section.
+// (ex. the Allocations section of `nomad job status :id`)
+func ParseColumns(section string) ([]map[string]string, error) {
+	parsed := []map[string]string{}
+
+	// field names and values are deliminated by two or more spaces, but can have a
+	// single space themselves. compress all the delimiters into a tab so we can
+	// break the fields on that
+	re := regexp.MustCompile(" {2,}")
+	section = re.ReplaceAllString(section, "\t")
+	rows := strings.Split(section, "\n")
+
+	breakFields := func(row string) []string {
+		return strings.FieldsFunc(row, func(c rune) bool { return c == '\t' })
+	}
+
+	fieldNames := breakFields(rows[0])
+
+	for _, row := range rows[1:] {
+		if row == "" {
+			continue
+		}
+		r := map[string]string{}
+		vals := breakFields(row)
+		for i, val := range vals {
+			if i >= len(fieldNames) {
+				return parsed, fmt.Errorf("section is misaligned with header\n%v", section)
+			}
+
+			r[fieldNames[i]] = val
+		}
+		parsed = append(parsed, r)
+	}
+	return parsed, nil
+}
+
+// ParseFields maps the CLI output for a key-value section (without title) to
+// map of the key->value pairs in that section
+// (ex. the Latest Deployment section of `nomad job status :id`)
+func ParseFields(section string) (map[string]string, error) {
+	parsed := map[string]string{}
+	rows := strings.Split(strings.TrimSpace(section), "\n")
+	for _, row := range rows {
+		kv := strings.Split(row, "=")
+		if len(kv) == 0 {
+			continue
+		}
+		key := strings.TrimSpace(kv[0])
+		if len(kv) == 1 {
+			parsed[key] = ""
+		} else {
+			parsed[key] = strings.TrimSpace(strings.Join(kv[1:], " "))
+		}
+	}
+	return parsed, nil
+}

e2e/rescheduling/helpers.go

@@ -0,0 +1,134 @@
+package rescheduling
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/hashicorp/nomad/e2e/framework"
+	"github.com/hashicorp/nomad/testutil"
+)
+
+// allocStatuses returns a slice of client statuses
+func allocStatuses(f *framework.F, jobID string) []string {
+
+	out, err := e2eutil.Command("nomad", "job", "status", "-verbose", "-all-allocs", jobID)
+	f.NoError(err, "nomad job status failed", err)
+	section, err := e2eutil.GetSection(out, "Allocations")
+	f.NoError(err, "could not find Allocations section", err)
+
+	allocs, err := e2eutil.ParseColumns(section)
+	f.NoError(err, "could not parse Allocations section", err)
+
+	statuses := []string{}
+	for _, alloc := range allocs {
+		statuses = append(statuses, alloc["Status"])
+	}
+
+	return statuses
+}
+
+// allocStatusesRescheduled is a helper function that pulls
+// out client statuses only from rescheduled allocs.
+func allocStatusesRescheduled(f *framework.F, jobID string) []string {
+
+	out, err := e2eutil.Command("nomad", "job", "status", "-verbose", jobID)
+	f.NoError(err, "nomad job status failed", err)
+	section, err := e2eutil.GetSection(out, "Allocations")
+	f.NoError(err, "could not find Allocations section", err)
+
+	allocs, err := e2eutil.ParseColumns(section)
+	f.NoError(err, "could not parse Allocations section", err)
+
+	statuses := []string{}
+	for _, alloc := range allocs {
+
+		allocID := alloc["ID"]
+
+		// reschedule tracker isn't exposed in the normal CLI output
+		out, err := e2eutil.Command("nomad", "alloc", "status", "-json", allocID)
+		f.NoError(err, "nomad alloc status failed", err)
+
+		dec := json.NewDecoder(strings.NewReader(out))
+		alloc := &api.Allocation{}
+		err = dec.Decode(alloc)
+		f.NoError(err, "could not decode alloc status JSON: %w", err)
+
+		if (alloc.RescheduleTracker != nil &&
+			len(alloc.RescheduleTracker.Events) > 0) || alloc.FollowupEvalID != "" {
+			statuses = append(statuses, alloc.ClientStatus)
+		}
+	}
+	return statuses
+}
+
+// register is a helper that registers a jobspec with a unique ID
+// and records that ID in the testcase for later cleanup
+func register(f *framework.F, jobFile, jobID string) {
+
+	cmd := exec.Command("nomad", "job", "run", "-")
+	stdin, err := cmd.StdinPipe()
+	f.NoError(err, fmt.Sprintf("could not open stdin?: %v", err))
+
+	content, err := ioutil.ReadFile(jobFile)
+	f.NoError(err, fmt.Sprintf("could not open job file: %v", err))
+
+	// hack off the first line to replace with our unique ID
+	var re = regexp.MustCompile(`^job "\w+" \{`)
+	jobspec := re.ReplaceAllString(string(content),
+		fmt.Sprintf("job \"%s\" {", jobID))
+
+	go func() {
+		defer stdin.Close()
+		io.WriteString(stdin, jobspec)
+	}()
+
+	out, err := cmd.CombinedOutput()
+	f.NoError(err, "could not register job: %v\n%v", err, string(out))
+}
+
+func waitForAllocStatusComparison(query func() ([]string, error), comparison func([]string) bool) error {
+	var got []string
+	var err error
+	testutil.WaitForResultRetries(30, func() (bool, error) {
+		time.Sleep(time.Millisecond * 100)
+		got, err = query()
+		if err != nil {
+			return false, err
+		}
+		return comparison(got), nil
+	}, func(e error) {
+		err = fmt.Errorf("alloc status check failed: got %#v", got)
+	})
+	return err
+}
+
+func waitForLastDeploymentStatus(f *framework.F, jobID, status string) error {
+	var got string
+	var err error
+	testutil.WaitForResultRetries(30, func() (bool, error) {
+		time.Sleep(time.Millisecond * 100)
+
+		out, err := e2eutil.Command("nomad", "job", "status", jobID)
+		f.NoError(err, "could not get job status: %v\n%v", err, out)
+
+		section, err := e2eutil.GetSection(out, "Latest Deployment")
+		f.NoError(err, "could not find Latest Deployment section", err)
+
+		fields, err := e2eutil.ParseFields(section)
+		f.NoError(err, "could not parse Latest Deployment section", err)
+
+		got = fields["Status"]
+		return got == status, nil
+	}, func(e error) {
+		err = fmt.Errorf("deployment status check failed: got %#v", got)
+	})
+	return err
+}