From ee1b091e35282fb8bb918f15255e5773615e8982 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Wed, 15 Jan 2020 20:51:57 +0000 Subject: [PATCH 01/30] WIP: adding ScalingPolicy to api/structs and state store --- api/tasks.go | 16 +++ command/agent/job_endpoint.go | 20 +++- jobspec/parse_group.go | 47 ++++++++ nomad/fsm.go | 8 ++ nomad/job_endpoint.go | 2 + nomad/mock/mock.go | 14 +++ nomad/state/schema.go | 62 ++++++++++ nomad/state/state_store.go | 201 ++++++++++++++++++++++++++++++++ nomad/state/state_store_test.go | 164 +++++++++++++++++++++++++- nomad/structs/funcs.go | 15 +++ nomad/structs/structs.go | 39 +++++++ 11 files changed, 585 insertions(+), 3 deletions(-) diff --git a/api/tasks.go b/api/tasks.go index 2e6e64d8828..d4856ee2eda 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -409,6 +409,18 @@ func (vm *VolumeMount) Canonicalize() { } } +// ScalingPolicy is the user-specified API object for an autoscaling policy +type ScalingPolicy struct { + Policy map[string]interface{} + Enabled *bool +} + +func (p *ScalingPolicy) Canonicalize() { + if p.Enabled == nil { + p.Enabled = boolToPtr(true) + } +} + // TaskGroup is the unit of scheduling. type TaskGroup struct { Name *string @@ -427,6 +439,7 @@ type TaskGroup struct { Meta map[string]string Services []*Service ShutdownDelay *time.Duration `mapstructure:"shutdown_delay"` + Scaling *ScalingPolicy } // NewTaskGroup creates a new TaskGroup. @@ -543,6 +556,9 @@ func (g *TaskGroup) Canonicalize(job *Job) { for _, s := range g.Services { s.Canonicalize(nil, g, job) } + if g.Scaling != nil { + g.Scaling.Canonicalize() + } } // Constrain is used to add a constraint to a task group. diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index b76329c5691..0f3d50042c1 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/golang/snappy" + "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/jobspec" @@ -685,7 +686,7 @@ func ApiJobToStructJob(job *api.Job) *structs.Job { j.TaskGroups = make([]*structs.TaskGroup, l) for i, taskGroup := range job.TaskGroups { tg := &structs.TaskGroup{} - ApiTgToStructsTG(taskGroup, tg) + ApiTgToStructsTG(j, taskGroup, tg) j.TaskGroups[i] = tg } } @@ -693,7 +694,7 @@ func ApiJobToStructJob(job *api.Job) *structs.Job { return j } -func ApiTgToStructsTG(taskGroup *api.TaskGroup, tg *structs.TaskGroup) { +func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.TaskGroup) { tg.Name = *taskGroup.Name tg.Count = *taskGroup.Count tg.Meta = taskGroup.Meta @@ -733,6 +734,11 @@ func ApiTgToStructsTG(taskGroup *api.TaskGroup, tg *structs.TaskGroup) { } } + if taskGroup.Scaling != nil { + target := fmt.Sprintf("%s/%s", job.ID, tg.Name) + tg.Scaling = ApiScalingPolicyToStructs(job, target, taskGroup.Scaling) + } + tg.EphemeralDisk = &structs.EphemeralDisk{ Sticky: *taskGroup.EphemeralDisk.Sticky, SizeMB: *taskGroup.EphemeralDisk.SizeMB, @@ -1217,3 +1223,13 @@ func ApiSpreadToStructs(a1 *api.Spread) *structs.Spread { } return ret } + +func ApiScalingPolicyToStructs(job *structs.Job, target string, a1 *api.ScalingPolicy) *structs.ScalingPolicy { + return &structs.ScalingPolicy{ + Namespace: job.Namespace, + JobID: job.ID, + Target: target, + Enabled: *a1.Enabled, + Policy: a1.Policy, + } +} diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index 86c078658b1..a5f222fda3a 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -55,6 +55,7 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { "network", "service", "volume", + "scaling", } if err := helper.CheckHCLKeys(listVal, valid); err != nil { return multierror.Prefix(err, fmt.Sprintf("'%s' ->", n)) @@ -78,6 +79,7 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { delete(m, "network") delete(m, "service") delete(m, "volume") + delete(m, "scaling") // Build the group with the basic decode var g api.TaskGroup @@ -181,6 +183,13 @@ func parseGroups(result *api.Job, list *ast.ObjectList) error { } } + // Parse scaling policy + if o := listVal.Filter("scaling"); len(o.Items) > 0 { + if err := parseScalingPolicy(&g.Scaling, o); err != nil { + return multierror.Prefix(err, "scaling ->") + } + } + // Parse tasks if o := listVal.Filter("task"); len(o.Items) > 0 { if err := parseTasks(&g.Tasks, o); err != nil { @@ -309,3 +318,41 @@ func parseVolumes(out *map[string]*api.VolumeRequest, list *ast.ObjectList) erro return nil } + +func parseScalingPolicy(out **api.ScalingPolicy, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) > 1 { + return fmt.Errorf("only one 'scaling' block allowed") + } + + // Get our resource object + o := list.Items[0] + + valid := []string{ + "policy", + "enabled", + } + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return err + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + var result api.ScalingPolicy + dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + WeaklyTypedInput: true, + Result: &result, + }) + if err != nil { + return err + } + if err := dec.Decode(m); err != nil { + return err + } + + *out = &result + return nil +} diff --git a/nomad/fsm.go b/nomad/fsm.go index 2dbaee2f571..f2ee8d361dd 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -545,6 +545,14 @@ func (n *nomadFSM) applyUpsertJob(buf []byte, index uint64) interface{} { } } + // cgbaker: FINISH + // create/update any scaling policies, remove old policies + // scalingPolicies := req.Job.GetScalingPolicies() + // for p := range scalingPolicies { + // n.state.UpsertACLPolicies() + // n.state.Upsert + // } + return nil } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index a0b17323add..844c311a8c7 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -189,6 +189,8 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis return err } + // cgbaker: FINISH: validate the scaling policies (e.g., can't change policy ID) + // Ensure that the job has permissions for the requested Vault tokens policies := args.Job.VaultPolicies() if len(policies) != 0 { diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index cfdd18a98a4..943264ea6b4 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -1253,3 +1253,17 @@ func ACLManagementToken() *structs.ACLToken { ModifyIndex: 20, } } + +func ScalingPolicy() *structs.ScalingPolicy { + return &structs.ScalingPolicy{ + Namespace: structs.DefaultNamespace, + Target: uuid.Generate(), + JobID: uuid.Generate(), + Policy: map[string]interface{}{ + "a": "b", + }, + Enabled: true, + CreateIndex: 10, + ModifyIndex: 20, + } +} diff --git a/nomad/state/schema.go b/nomad/state/schema.go index b58bf68551e..078022f96a5 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -49,6 +49,7 @@ func init() { clusterMetaTableSchema, csiVolumeTableSchema, csiPluginTableSchema, + scalingPolicyTableSchema, }...) } @@ -728,3 +729,64 @@ func csiPluginTableSchema() *memdb.TableSchema { }, } } + +// scalingPolicyTableSchema returns the MemDB schema for the policy table. +// This table is used to store the policies which are referenced by tokens +func scalingPolicyTableSchema() *memdb.TableSchema { + return &memdb.TableSchema{ + Name: "scaling_policy", + Indexes: map[string]*memdb.IndexSchema{ + // Primary index is used for job management + // and simple direct lookup. Target is required to be + // unique within a namespace. + "id": { + Name: "id", + AllowMissing: false, + Unique: true, + + // Use a compound index so the tuple of (Namespace, Target) is + // uniquely identifying + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &memdb.StringFieldIndex{ + Field: "Namespace", + }, + + &memdb.StringFieldIndex{ + Field: "Target", + }, + }, + }, + }, + // Job index is used to lookup scaling policies by job + "job": { + Name: "job", + AllowMissing: false, + Unique: false, + + // Use a compound index so the tuple of (Namespace, JobID) is + // uniquely identifying + Indexer: &memdb.CompoundIndex{ + Indexes: []memdb.Indexer{ + &memdb.StringFieldIndex{ + Field: "Namespace", + }, + + &memdb.StringFieldIndex{ + Field: "JobID", + }, + }, + }, + }, + // Used to filter by enabled + "enabled": { + Name: "enabled", + AllowMissing: false, + Unique: false, + Indexer: &memdb.FieldSetIndex{ + Field: "Enabled", + }, + }, + }, + } +} diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 04fae83ad5f..4ef7eca3966 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -4650,6 +4650,207 @@ func (s *StateStore) setClusterMetadata(txn *memdb.Txn, meta *structs.ClusterMet return nil } +// UpsertScalingPolicy is used to insert a new scaling policy. +func (s *StateStore) UpsertScalingPolicies(index uint64, scalingPolicies []*structs.ScalingPolicy) error { + txn := s.db.Txn(true) + defer txn.Abort() + + for _, scalingPolicy := range scalingPolicies { + // Check if the scaling policy already exists + existing, err := txn.First("scaling_policy", "id", scalingPolicy.Namespace, scalingPolicy.Target) + if err != nil { + return fmt.Errorf("scaling policy lookup failed: %v", err) + } + + // Setup the indexes correctly + if existing != nil { + scalingPolicy.CreateIndex = existing.(*structs.ScalingPolicy).CreateIndex + scalingPolicy.ModifyIndex = index + } else { + scalingPolicy.CreateIndex = index + scalingPolicy.ModifyIndex = index + } + + // Insert the scaling policy + if err := txn.Insert("scaling_policy", scalingPolicy); err != nil { + return err + } + } + + // Update the indexes table for scaling policy + if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + + txn.Commit() + return nil +} + +// DeleteScalingPolicies is used to delete a set of scaling policies by ID +func (s *StateStore) DeleteScalingPolicies(index uint64, namespace string, targets []string) error { + txn := s.db.Txn(true) + defer txn.Abort() + + if len(targets) == 0 { + return nil + } + + for _, tgt := range targets { + // Lookup the scaling policy + existing, err := txn.First("scaling_policy", "id", namespace, tgt) + if err != nil { + return fmt.Errorf("scaling policy lookup failed: %v", err) + } + if existing == nil { + return fmt.Errorf("scaling policy not found") + } + + // Delete the scaling policy + if err := txn.Delete("scaling_policy", existing); err != nil { + return fmt.Errorf("scaling policy delete failed: %v", err) + } + } + + if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + + txn.Commit() + return nil +} + +func (s *StateStore) ScalingPoliciesByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { + txn := s.db.Txn(false) + + // Walk the entire scaling policy table + iter, err := txn.Get("scaling_policy", "id_prefix", namespace) + if err != nil { + return nil, err + } + + ws.Add(iter.WatchCh()) + return iter, nil +} + +// func (s *StateStore) ScalingPoliciesByIDPrefix(ws memdb.WatchSet, namespace, deploymentID string) (memdb.ResultIterator, error) { +// txn := s.db.Txn(false) +// +// // Walk the entire deployments table +// iter, err := txn.Get("scaling_policy", "id_prefix", deploymentID) +// if err != nil { +// return nil, err +// } +// +// ws.Add(iter.WatchCh()) +// +// // Wrap the iterator in a filter +// wrap := memdb.NewFilterIterator(iter, scalingPolicyNamespaceFilter(namespace)) +// return wrap, nil +// } + +// // scalingPolicyNamespaceFilter returns a filter function that filters all +// // scalingPolicy not in the given namespace. +// func scalingPolicyNamespaceFilter(namespace string) func(interface{}) bool { +// return func(raw interface{}) bool { +// d, ok := raw.(*structs.Deployment) +// if !ok { +// return true +// } +// +// return d.Namespace != namespace +// } +// } + +func (s *StateStore) ScalingPolicyByTarget(ws memdb.WatchSet, namespace, target string) (*structs.ScalingPolicy, error) { + txn := s.db.Txn(false) + return s.scalingPolicyByIDImpl(ws, namespace, target, txn) +} + +func (s *StateStore) scalingPolicyByIDImpl(ws memdb.WatchSet, namespace, target string, + txn *memdb.Txn) (*structs.ScalingPolicy, error) { + watchCh, existing, err := txn.FirstWatch("scaling_policy", "id", namespace, target) + if err != nil { + return nil, fmt.Errorf("scaling_policy lookup failed: %v", err) + } + ws.Add(watchCh) + + if existing != nil { + return existing.(*structs.ScalingPolicy), nil + } + + return nil, nil +} + +// func (s *StateStore) DeploymentsByJobID(ws memdb.WatchSet, namespace, jobID string, all bool) ([]*structs.Deployment, error) { +// txn := s.db.Txn(false) +// +// var job *structs.Job +// // Read job from state store +// _, existing, err := txn.FirstWatch("jobs", "id", namespace, jobID) +// if err != nil { +// return nil, fmt.Errorf("job lookup failed: %v", err) +// } +// if existing != nil { +// job = existing.(*structs.Job) +// } +// +// // Get an iterator over the deployments +// iter, err := txn.Get("deployment", "job", namespace, jobID) +// if err != nil { +// return nil, err +// } +// +// ws.Add(iter.WatchCh()) +// +// var out []*structs.Deployment +// for { +// raw := iter.Next() +// if raw == nil { +// break +// } +// d := raw.(*structs.Deployment) +// +// // If the allocation belongs to a job with the same ID but a different +// // create index and we are not getting all the allocations whose Jobs +// // matches the same Job ID then we skip it +// if !all && job != nil && d.JobCreateIndex != job.CreateIndex { +// continue +// } +// out = append(out, d) +// } +// +// return out, nil +// } + +// // LatestDeploymentByJobID returns the latest deployment for the given job. The +// // latest is determined strictly by CreateIndex. +// func (s *StateStore) LatestDeploymentByJobID(ws memdb.WatchSet, namespace, jobID string) (*structs.Deployment, error) { +// txn := s.db.Txn(false) +// +// // Get an iterator over the deployments +// iter, err := txn.Get("deployment", "job", namespace, jobID) +// if err != nil { +// return nil, err +// } +// +// ws.Add(iter.WatchCh()) +// +// var out *structs.Deployment +// for { +// raw := iter.Next() +// if raw == nil { +// break +// } +// +// d := raw.(*structs.Deployment) +// if out == nil || out.CreateIndex < d.CreateIndex { +// out = d +// } +// } +// +// return out, nil +// } + // StateSnapshot is used to provide a point-in-time snapshot type StateSnapshot struct { StateStore diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 13dc6093bcb..253888793bc 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -7420,7 +7420,7 @@ func TestStateStore_DeleteACLPolicy(t *testing.T) { t.Fatalf("err: %v", err) } - // Ensure we see both policies + // Ensure we see neither policy count := 0 for { raw := iter.Next() @@ -7919,6 +7919,168 @@ func TestStateStore_ClusterMetadataRestore(t *testing.T) { require.Equal(now, out.CreateTime) } +func TestStateStore_UpsertScalingPolicy(t *testing.T) { + t.Parallel() + require := require.New(t) + + state := testStateStore(t) + policy := mock.ScalingPolicy() + policy2 := mock.ScalingPolicy() + + ws := memdb.NewWatchSet() + _, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + require.NoError(err) + + _, err = state.ScalingPolicyByTarget(ws, policy.Namespace, policy2.Target) + require.NoError(err) + + err = state.UpsertScalingPolicies(1000, []*structs.ScalingPolicy{policy, policy2}) + require.NoError(err) + require.True(watchFired(ws)) + + ws = memdb.NewWatchSet() + out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + require.NoError(err) + require.Equal(policy, out) + + out, err = state.ScalingPolicyByTarget(ws, policy2.Namespace, policy2.Target) + require.NoError(err) + require.Equal(policy2, out) + + iter, err := state.ScalingPoliciesByNamespace(ws, policy.Namespace) + require.NoError(err) + + // Ensure we see both policies + count := 0 + for { + raw := iter.Next() + if raw == nil { + break + } + count++ + } + require.Equal(2, count) + + index, err := state.Index("scaling_policy") + require.NoError(err) + require.True(1000 == index) + require.False(watchFired(ws)) +} + +func TestStateStore_DeleteScalingPolicies(t *testing.T) { + t.Parallel() + + require := require.New(t) + + state := testStateStore(t) + policy := mock.ScalingPolicy() + policy2 := mock.ScalingPolicy() + + // Create the policy + err := state.UpsertScalingPolicies(1000, []*structs.ScalingPolicy{policy, policy2}) + require.NoError(err) + + // Create a watcher + ws := memdb.NewWatchSet() + _, err = state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + require.NoError(err) + + // Delete the policy + err = state.DeleteScalingPolicies(1001, policy.Namespace, []string{policy.Target, policy2.Target}) + require.NoError(err) + + // Ensure watching triggered + require.True(watchFired(ws)) + + // Ensure we don't get the objects back + ws = memdb.NewWatchSet() + out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + require.NoError(err) + require.Nil(out) + + ws = memdb.NewWatchSet() + out, err = state.ScalingPolicyByTarget(ws, policy2.Namespace, policy2.Target) + require.NoError(err) + require.Nil(out) + + // Ensure we see both policies + iter, err := state.ScalingPoliciesByNamespace(ws, policy.Namespace) + require.NoError(err) + count := 0 + for { + raw := iter.Next() + if raw == nil { + break + } + count++ + } + require.Equal(0, count) + + index, err := state.Index("scaling_policy") + require.NoError(err) + require.True(1001 == index) + require.False(watchFired(ws)) +} + +func TestStateStore_ScalingPoliciesByNamespace(t *testing.T) { + t.Parallel() + + require := require.New(t) + + state := testStateStore(t) + policyA1 := mock.ScalingPolicy() + policyA2 := mock.ScalingPolicy() + policyB1 := mock.ScalingPolicy() + policyB2 := mock.ScalingPolicy() + policyB1.Namespace = "different-namespace" + policyB2.Namespace = policyB1.Namespace + + // Create the policies + var baseIndex uint64 = 1000 + err := state.UpsertScalingPolicies(baseIndex, []*structs.ScalingPolicy{policyA1, policyA2, policyB1, policyB2}) + require.NoError(err) + + iter, err := state.ScalingPoliciesByNamespace(nil, policyA1.Namespace) + require.NoError(err) + + // Ensure we see expected policies + count := 0 + found := []string{} + for { + raw := iter.Next() + if raw == nil { + break + } + count++ + found = append(found, raw.(*structs.ScalingPolicy).Target) + } + require.Equal(2, count) + sort.Strings(found) + expect := []string{policyA1.Target, policyA2.Target} + sort.Strings(expect) + require.Equal(expect, found) + + iter, err = state.ScalingPoliciesByNamespace(nil, policyB1.Namespace) + require.NoError(err) + + // Ensure we see expected policies + count = 0 + found = []string{} + for { + raw := iter.Next() + if raw == nil { + break + } + count++ + found = append(found, raw.(*structs.ScalingPolicy).Target) + } + require.Equal(2, count) + sort.Strings(found) + expect = []string{policyB1.Target, policyB2.Target} + sort.Strings(expect) + require.Equal(expect, found) +} + func TestStateStore_Abandon(t *testing.T) { t.Parallel() diff --git a/nomad/structs/funcs.go b/nomad/structs/funcs.go index f79cb6c067a..889a7ce4dd7 100644 --- a/nomad/structs/funcs.go +++ b/nomad/structs/funcs.go @@ -252,6 +252,21 @@ func CopySliceNodeScoreMeta(s []*NodeScoreMeta) []*NodeScoreMeta { return c } +func CopyScalingPolicy(p *ScalingPolicy) *ScalingPolicy { + if p == nil { + return nil + } + + c := ScalingPolicy{ + Namespace: p.Namespace, + Target: p.Target, + JobID: p.JobID, + Policy: p.Policy, + Enabled: p.Enabled, + } + return &c +} + // VaultPoliciesSet takes the structure returned by VaultPolicies and returns // the set of required policies func VaultPoliciesSet(policies map[string]map[string]*Vault) []string { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 3c1fa2b6ca0..d1d3d616726 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -4003,6 +4003,8 @@ func (j *Job) SpecChanged(new *Job) bool { c.JobModifyIndex = j.JobModifyIndex c.SubmitTime = j.SubmitTime + // cgbaker: FINISH: probably need some consideration of scaling policy ID here + // Deep equals the jobs return !reflect.DeepEqual(j, c) } @@ -4576,6 +4578,39 @@ const ( ReasonWithinPolicy = "Restart within policy" ) +// ScalingPolicy specifies the scaling policy for a scaling target +type ScalingPolicy struct { + // Namespace is the namespace for the containing job + Namespace string + + // Target is the scaling target; there can be only one policy per scaling target + Target string + + // JobID is the ID of the parent job; there can be multiple policies per job + JobID string + + // Policy is an opaque description of the scaling policy, passed to the autoscaler + Policy map[string]interface{} + + // Enabled indicates whether this policy has been enabled/disabled + Enabled bool + + CreateIndex uint64 + ModifyIndex uint64 +} + +func (j *Job) GetScalingPolicies() []*ScalingPolicy { + ret := []*ScalingPolicy{} + + for _, tg := range j.TaskGroups { + if tg.Scaling != nil { + ret = append(ret, tg.Scaling) + } + } + + return ret +} + // RestartPolicy configures how Tasks are restarted when they crash or fail. type RestartPolicy struct { // Attempts is the number of restart that will occur in an interval. @@ -4927,6 +4962,9 @@ type TaskGroup struct { // all the tasks contained. Constraints []*Constraint + // Scaling is the list of autoscaling policies for the TaskGroup + Scaling *ScalingPolicy + //RestartPolicy of a TaskGroup RestartPolicy *RestartPolicy @@ -4980,6 +5018,7 @@ func (tg *TaskGroup) Copy() *TaskGroup { ntg.Affinities = CopySliceAffinities(ntg.Affinities) ntg.Spreads = CopySliceSpreads(ntg.Spreads) ntg.Volumes = CopyMapVolumeRequest(ntg.Volumes) + ntg.Scaling = CopyScalingPolicy(ntg.Scaling) // Copy the network objects if tg.Networks != nil { From 0762386a5043542298b7817c65c812da5373673f Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Thu, 16 Jan 2020 14:40:54 +0000 Subject: [PATCH 02/30] wip: upsert/delete scaling policies on job upsert/delete --- command/agent/job_endpoint.go | 6 +- nomad/fsm.go | 8 -- nomad/state/state_store.go | 198 +++++++++++++++----------------- nomad/state/state_store_test.go | 87 ++++++++++++-- nomad/structs/structs.go | 8 +- 5 files changed, 177 insertions(+), 130 deletions(-) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 0f3d50042c1..279ff1a08ee 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -735,8 +735,7 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta } if taskGroup.Scaling != nil { - target := fmt.Sprintf("%s/%s", job.ID, tg.Name) - tg.Scaling = ApiScalingPolicyToStructs(job, target, taskGroup.Scaling) + tg.Scaling = ApiScalingPolicyToStructs(job, taskGroup.Scaling).TargetTaskGroup(job, tg) } tg.EphemeralDisk = &structs.EphemeralDisk{ @@ -1224,11 +1223,10 @@ func ApiSpreadToStructs(a1 *api.Spread) *structs.Spread { return ret } -func ApiScalingPolicyToStructs(job *structs.Job, target string, a1 *api.ScalingPolicy) *structs.ScalingPolicy { +func ApiScalingPolicyToStructs(job *structs.Job, a1 *api.ScalingPolicy) *structs.ScalingPolicy { return &structs.ScalingPolicy{ Namespace: job.Namespace, JobID: job.ID, - Target: target, Enabled: *a1.Enabled, Policy: a1.Policy, } diff --git a/nomad/fsm.go b/nomad/fsm.go index f2ee8d361dd..2dbaee2f571 100644 --- a/nomad/fsm.go +++ b/nomad/fsm.go @@ -545,14 +545,6 @@ func (n *nomadFSM) applyUpsertJob(buf []byte, index uint64) interface{} { } } - // cgbaker: FINISH - // create/update any scaling policies, remove old policies - // scalingPolicies := req.Job.GetScalingPolicies() - // for p := range scalingPolicies { - // n.state.UpsertACLPolicies() - // n.state.Upsert - // } - return nil } diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 4ef7eca3966..12667e38e3a 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -1177,6 +1177,10 @@ func (s *StateStore) upsertJobImpl(index uint64, job *structs.Job, keepVersion b return fmt.Errorf("unable to upsert job into job_version table: %v", err) } + if err := s.updateJobScalingPolicies(index, job, txn); err != nil { + return fmt.Errorf("unable to update job scaling policies: %v", err) + } + // Insert the job if err := txn.Insert("jobs", job); err != nil { return fmt.Errorf("job insert failed: %v", err) @@ -1273,12 +1277,20 @@ func (s *StateStore) DeleteJobTxn(index uint64, namespace, jobID string, txn Txn // Delete the job summary if _, err = txn.DeleteAll("job_summary", "id", namespace, jobID); err != nil { - return fmt.Errorf("deleing job summary failed: %v", err) + return fmt.Errorf("deleting job summary failed: %v", err) } if err := txn.Insert("index", &IndexEntry{"job_summary", index}); err != nil { return fmt.Errorf("index update failed: %v", err) } + // Delete the job scaling policies + if _, err := txn.DeleteAll("scaling_policy", "job", namespace, jobID); err != nil { + return fmt.Errorf("deleting job scaling policies failed: %v", err) + } + if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } + return nil } @@ -3980,6 +3992,46 @@ func (s *StateStore) updateSummaryWithJob(index uint64, job *structs.Job, return nil } +// updateJobScalingPolicies upserts any scaling policies contained in the job and removes +// any previous scaling policies that were removed from the job +func (s *StateStore) updateJobScalingPolicies(index uint64, job *structs.Job, txn *memdb.Txn) error { + + ws := memdb.NewWatchSet() + + scalingPolicies := job.GetScalingPolicies() + newTargets := map[string]struct{}{} + for _, p := range scalingPolicies { + newTargets[p.Target] = struct{}{} + } + // find existing policies that need to be deleted + deletedPolicies := []string{} + iter, err := s.ScalingPoliciesByJobTxn(ws, job.Namespace, job.ID, txn) + if err != nil { + return fmt.Errorf("ScalingPoliciesByJob lookup failed: %v", err) + } + for { + raw := iter.Next() + if raw == nil { + break + } + oldPolicy := raw.(*structs.ScalingPolicy) + if _, ok := newTargets[oldPolicy.Target]; !ok { + deletedPolicies = append(deletedPolicies, oldPolicy.Target) + } + } + err = s.DeleteScalingPoliciesTxn(index, job.Namespace, deletedPolicies, txn) + if err != nil { + return fmt.Errorf("DeleteScalingPolicies of removed policies failed: %v", err) + } + + err = s.UpsertScalingPoliciesTxn(index, scalingPolicies, txn) + if err != nil { + return fmt.Errorf("UpsertScalingPolicies of policies failed: %v", err) + } + + return nil +} + // updateDeploymentWithAlloc is used to update the deployment state associated // with the given allocation. The passed alloc may be updated if the deployment // status has changed to capture the modify index at which it has changed. @@ -4655,6 +4707,18 @@ func (s *StateStore) UpsertScalingPolicies(index uint64, scalingPolicies []*stru txn := s.db.Txn(true) defer txn.Abort() + if err := s.UpsertScalingPoliciesTxn(index, scalingPolicies, txn); err != nil { + return err + } + + txn.Commit() + return nil +} + +// upsertScalingPolicy is used to insert a new scaling policy. +func (s *StateStore) UpsertScalingPoliciesTxn(index uint64, scalingPolicies []*structs.ScalingPolicy, + txn *memdb.Txn) error { + for _, scalingPolicy := range scalingPolicies { // Check if the scaling policy already exists existing, err := txn.First("scaling_policy", "id", scalingPolicy.Namespace, scalingPolicy.Target) @@ -4682,15 +4746,23 @@ func (s *StateStore) UpsertScalingPolicies(index uint64, scalingPolicies []*stru return fmt.Errorf("index update failed: %v", err) } - txn.Commit() return nil } -// DeleteScalingPolicies is used to delete a set of scaling policies by ID func (s *StateStore) DeleteScalingPolicies(index uint64, namespace string, targets []string) error { txn := s.db.Txn(true) defer txn.Abort() + err := s.DeleteScalingPoliciesTxn(index, namespace, targets, txn) + if err == nil { + txn.Commit() + } + + return err +} + +// DeleteScalingPolicies is used to delete a set of scaling policies by ID +func (s *StateStore) DeleteScalingPoliciesTxn(index uint64, namespace string, targets []string, txn *memdb.Txn) error { if len(targets) == 0 { return nil } @@ -4715,14 +4787,12 @@ func (s *StateStore) DeleteScalingPolicies(index uint64, namespace string, targe return fmt.Errorf("index update failed: %v", err) } - txn.Commit() return nil } func (s *StateStore) ScalingPoliciesByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { txn := s.db.Txn(false) - // Walk the entire scaling policy table iter, err := txn.Get("scaling_policy", "id_prefix", namespace) if err != nil { return nil, err @@ -4732,42 +4802,26 @@ func (s *StateStore) ScalingPoliciesByNamespace(ws memdb.WatchSet, namespace str return iter, nil } -// func (s *StateStore) ScalingPoliciesByIDPrefix(ws memdb.WatchSet, namespace, deploymentID string) (memdb.ResultIterator, error) { -// txn := s.db.Txn(false) -// -// // Walk the entire deployments table -// iter, err := txn.Get("scaling_policy", "id_prefix", deploymentID) -// if err != nil { -// return nil, err -// } -// -// ws.Add(iter.WatchCh()) -// -// // Wrap the iterator in a filter -// wrap := memdb.NewFilterIterator(iter, scalingPolicyNamespaceFilter(namespace)) -// return wrap, nil -// } - -// // scalingPolicyNamespaceFilter returns a filter function that filters all -// // scalingPolicy not in the given namespace. -// func scalingPolicyNamespaceFilter(namespace string) func(interface{}) bool { -// return func(raw interface{}) bool { -// d, ok := raw.(*structs.Deployment) -// if !ok { -// return true -// } -// -// return d.Namespace != namespace -// } -// } +func (s *StateStore) ScalingPoliciesByJob(ws memdb.WatchSet, namespace, jobID string) (memdb.ResultIterator, error) { + txn := s.db.Txn(false) + return s.ScalingPoliciesByJobTxn(ws, namespace, jobID, txn) +} + +func (s *StateStore) ScalingPoliciesByJobTxn(ws memdb.WatchSet, namespace, jobID string, + txn *memdb.Txn) (memdb.ResultIterator, error) { + + iter, err := txn.Get("scaling_policy", "job", namespace, jobID) + if err != nil { + return nil, err + } + + ws.Add(iter.WatchCh()) + return iter, nil +} func (s *StateStore) ScalingPolicyByTarget(ws memdb.WatchSet, namespace, target string) (*structs.ScalingPolicy, error) { txn := s.db.Txn(false) - return s.scalingPolicyByIDImpl(ws, namespace, target, txn) -} -func (s *StateStore) scalingPolicyByIDImpl(ws memdb.WatchSet, namespace, target string, - txn *memdb.Txn) (*structs.ScalingPolicy, error) { watchCh, existing, err := txn.FirstWatch("scaling_policy", "id", namespace, target) if err != nil { return nil, fmt.Errorf("scaling_policy lookup failed: %v", err) @@ -4781,76 +4835,6 @@ func (s *StateStore) scalingPolicyByIDImpl(ws memdb.WatchSet, namespace, target return nil, nil } -// func (s *StateStore) DeploymentsByJobID(ws memdb.WatchSet, namespace, jobID string, all bool) ([]*structs.Deployment, error) { -// txn := s.db.Txn(false) -// -// var job *structs.Job -// // Read job from state store -// _, existing, err := txn.FirstWatch("jobs", "id", namespace, jobID) -// if err != nil { -// return nil, fmt.Errorf("job lookup failed: %v", err) -// } -// if existing != nil { -// job = existing.(*structs.Job) -// } -// -// // Get an iterator over the deployments -// iter, err := txn.Get("deployment", "job", namespace, jobID) -// if err != nil { -// return nil, err -// } -// -// ws.Add(iter.WatchCh()) -// -// var out []*structs.Deployment -// for { -// raw := iter.Next() -// if raw == nil { -// break -// } -// d := raw.(*structs.Deployment) -// -// // If the allocation belongs to a job with the same ID but a different -// // create index and we are not getting all the allocations whose Jobs -// // matches the same Job ID then we skip it -// if !all && job != nil && d.JobCreateIndex != job.CreateIndex { -// continue -// } -// out = append(out, d) -// } -// -// return out, nil -// } - -// // LatestDeploymentByJobID returns the latest deployment for the given job. The -// // latest is determined strictly by CreateIndex. -// func (s *StateStore) LatestDeploymentByJobID(ws memdb.WatchSet, namespace, jobID string) (*structs.Deployment, error) { -// txn := s.db.Txn(false) -// -// // Get an iterator over the deployments -// iter, err := txn.Get("deployment", "job", namespace, jobID) -// if err != nil { -// return nil, err -// } -// -// ws.Add(iter.WatchCh()) -// -// var out *structs.Deployment -// for { -// raw := iter.Next() -// if raw == nil { -// break -// } -// -// d := raw.(*structs.Deployment) -// if out == nil || out.CreateIndex < d.CreateIndex { -// out = d -// } -// } -// -// return out, nil -// } - // StateSnapshot is used to provide a point-in-time snapshot type StateSnapshot struct { StateStore diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 253888793bc..fb64f0f840c 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -7967,6 +7967,45 @@ func TestStateStore_UpsertScalingPolicy(t *testing.T) { require.False(watchFired(ws)) } +func TestStateStore_UpsertJob_UpsertScalingPolicies(t *testing.T) { + t.Parallel() + + require := require.New(t) + + state := testStateStore(t) + job := mock.Job() + // create policy and register against task group + policy := &structs.ScalingPolicy{ + Namespace: job.Namespace, + JobID: job.ID, + Policy: map[string]interface{}{}, + Enabled: true, + } + policy.TargetTaskGroup(job, job.TaskGroups[0]) + job.TaskGroups[0].Scaling = policy + + // Create a watchset so we can test that upsert fires the watch + ws := memdb.NewWatchSet() + out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + require.NoError(err) + require.Nil(out) + + var newIndex uint64 = 1000 + err = state.UpsertJob(newIndex, job) + require.NoError(err) + require.True(watchFired(ws), "watch did not fire") + + ws = memdb.NewWatchSet() + out, err = state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + require.NoError(err) + require.NotNil(out) + require.Equal(newIndex, out.CreateIndex) + require.Equal(newIndex, out.ModifyIndex) + + index, err := state.Index("scaling_policy") + require.Equal(newIndex, index) +} + func TestStateStore_DeleteScalingPolicies(t *testing.T) { t.Parallel() @@ -8022,25 +8061,53 @@ func TestStateStore_DeleteScalingPolicies(t *testing.T) { require.False(watchFired(ws)) } -func TestStateStore_ScalingPoliciesByNamespace(t *testing.T) { +func TestStateStore_DeleteJob_ChildScalingPolicies(t *testing.T) { t.Parallel() require := require.New(t) state := testStateStore(t) - policyA1 := mock.ScalingPolicy() - policyA2 := mock.ScalingPolicy() + + job := mock.Job() + + err := state.UpsertJob(1000, job) + require.NoError(err) + + policy := mock.ScalingPolicy() + policy.JobID = job.ID + err = state.UpsertScalingPolicies(1001, []*structs.ScalingPolicy{policy}) + require.NoError(err) + + // Delete the job + err = state.DeleteJob(1002, job.Namespace, job.ID) + require.NoError(err) + + // Ensure the scaling policy was deleted + ws := memdb.NewWatchSet() + out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + require.NoError(err) + require.Nil(out) + index, err := state.Index("scaling_policy") + require.True(index > 1001) +} + +func TestStateStore_ScalingPoliciesByJob(t *testing.T) { + t.Parallel() + + require := require.New(t) + + state := testStateStore(t) + policyA := mock.ScalingPolicy() policyB1 := mock.ScalingPolicy() policyB2 := mock.ScalingPolicy() - policyB1.Namespace = "different-namespace" - policyB2.Namespace = policyB1.Namespace + policyB1.JobID = policyB2.JobID // Create the policies var baseIndex uint64 = 1000 - err := state.UpsertScalingPolicies(baseIndex, []*structs.ScalingPolicy{policyA1, policyA2, policyB1, policyB2}) + err := state.UpsertScalingPolicies(baseIndex, []*structs.ScalingPolicy{policyA, policyB1, policyB2}) require.NoError(err) - iter, err := state.ScalingPoliciesByNamespace(nil, policyA1.Namespace) + iter, err := state.ScalingPoliciesByJob(nil, policyA.Namespace, policyA.JobID) require.NoError(err) // Ensure we see expected policies @@ -8054,13 +8121,13 @@ func TestStateStore_ScalingPoliciesByNamespace(t *testing.T) { count++ found = append(found, raw.(*structs.ScalingPolicy).Target) } - require.Equal(2, count) + require.Equal(1, count) sort.Strings(found) - expect := []string{policyA1.Target, policyA2.Target} + expect := []string{policyA.Target} sort.Strings(expect) require.Equal(expect, found) - iter, err = state.ScalingPoliciesByNamespace(nil, policyB1.Namespace) + iter, err = state.ScalingPoliciesByJob(nil, policyB1.Namespace, policyB1.JobID) require.NoError(err) // Ensure we see expected policies diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index d1d3d616726..3f9e2d32c67 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -4599,8 +4599,14 @@ type ScalingPolicy struct { ModifyIndex uint64 } +func (p *ScalingPolicy) TargetTaskGroup(job *Job, tg *TaskGroup) *ScalingPolicy { + p.Target = fmt.Sprintf("%s/%s", job.ID, tg.Name) + return p +} + +// GetScalingPolicies returns a slice of all scaling scaling policies for this job func (j *Job) GetScalingPolicies() []*ScalingPolicy { - ret := []*ScalingPolicy{} + ret := make([]*ScalingPolicy, 0) for _, tg := range j.TaskGroups { if tg.Scaling != nil { From e6d75cab9f4eb36f74338da6ff31c2e922d9636a Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Thu, 16 Jan 2020 15:32:00 +0000 Subject: [PATCH 03/30] wip: was incorrectly parsing ScalingPolicy --- jobspec/parse_group.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index a5f222fda3a..44b25185e5e 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -328,6 +328,14 @@ func parseScalingPolicy(out **api.ScalingPolicy, list *ast.ObjectList) error { // Get our resource object o := list.Items[0] + // We need this later + var listVal *ast.ObjectList + if ot, ok := o.Val.(*ast.ObjectType); ok { + listVal = ot.List + } else { + return fmt.Errorf("should be an object") + } + valid := []string{ "policy", "enabled", @@ -340,6 +348,7 @@ func parseScalingPolicy(out **api.ScalingPolicy, list *ast.ObjectList) error { if err := hcl.DecodeObject(&m, o.Val); err != nil { return err } + delete(m, "policy") var result api.ScalingPolicy dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ @@ -353,6 +362,20 @@ func parseScalingPolicy(out **api.ScalingPolicy, list *ast.ObjectList) error { return err } + // If we have policy, then parse that + if o := listVal.Filter("policy"); len(o.Items) > 0 { + for _, o := range o.Elem().Items { + var m map[string]interface{} + if err := hcl.DecodeObject(&m, o.Val); err != nil { + return err + } + + if err := mapstructure.WeakDecode(m, &result.Policy); err != nil { + return err + } + } + } + *out = &result return nil } From 7ba9b9401853ef17b8cd8bf232c5522d065278c4 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Thu, 16 Jan 2020 15:44:40 +0000 Subject: [PATCH 04/30] wip: test for scaling policy parsing --- jobspec/parse_test.go | 23 +++++++++++++++++++++ jobspec/test-fixtures/tg-scaling-policy.hcl | 13 ++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 jobspec/test-fixtures/tg-scaling-policy.hcl diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 924d3ab719e..5eda7063c7d 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1129,6 +1129,29 @@ func TestParse(t *testing.T) { }, false, }, + + { + "tg-scaling-policy.hcl", + &api.Job{ + ID: helper.StringToPtr("elastic"), + Name: helper.StringToPtr("elastic"), + TaskGroups: []*api.TaskGroup{ + { + Name: helper.StringToPtr("group"), + Scaling: &api.ScalingPolicy{ + Policy: map[string]interface{}{ + "foo": "bar", + "b": true, + "val": 5, + "f": .1, + }, + Enabled: helper.BoolToPtr(false), + }, + }, + }, + }, + false, + }, } for _, tc := range cases { diff --git a/jobspec/test-fixtures/tg-scaling-policy.hcl b/jobspec/test-fixtures/tg-scaling-policy.hcl new file mode 100644 index 00000000000..2875892b3f1 --- /dev/null +++ b/jobspec/test-fixtures/tg-scaling-policy.hcl @@ -0,0 +1,13 @@ +job "elastic" { + group "group" { + scaling { + enabled = false + policy { + foo = "bar" + b = true + val = 5 + f = .1 + } + } + } +} From a715eb7820542140ce3044d2212895bed6029de5 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Thu, 16 Jan 2020 20:12:52 +0000 Subject: [PATCH 05/30] wip: added policy get endpoint, added UUID to policy --- command/agent/http.go | 3 + command/agent/job_endpoint.go | 9 - command/agent/scaling_endpoint.go | 118 ++ command/agent/scaling_endpoint_test.go | 2174 ++++++++++++++++++++++++ nomad/job_endpoint.go | 2 - nomad/mock/mock.go | 15 + nomad/scaling_endpoint.go | 228 +++ nomad/scaling_endpoint_test.go | 344 ++++ nomad/server.go | 3 + nomad/state/schema.go | 17 +- nomad/state/state_store.go | 43 +- nomad/state/state_store_test.go | 13 +- nomad/structs/funcs.go | 13 +- nomad/structs/structs.go | 59 +- 14 files changed, 2994 insertions(+), 47 deletions(-) create mode 100644 command/agent/scaling_endpoint.go create mode 100644 command/agent/scaling_endpoint_test.go create mode 100644 nomad/scaling_endpoint.go create mode 100644 nomad/scaling_endpoint_test.go diff --git a/command/agent/http.go b/command/agent/http.go index 68cf7b46afd..003f99a7a8e 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -298,6 +298,9 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/regions", s.wrap(s.RegionListRequest)) + s.mux.HandleFunc("/v1/scaling/policies", s.wrap(s.ScalingPoliciesRequest)) + s.mux.HandleFunc("/v1/scaling/policy/", s.wrap(s.ScalingPolicySpecificRequest)) + s.mux.HandleFunc("/v1/status/leader", s.wrap(s.StatusLeaderRequest)) s.mux.HandleFunc("/v1/status/peers", s.wrap(s.StatusPeersRequest)) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 279ff1a08ee..83a020bc939 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1222,12 +1222,3 @@ func ApiSpreadToStructs(a1 *api.Spread) *structs.Spread { } return ret } - -func ApiScalingPolicyToStructs(job *structs.Job, a1 *api.ScalingPolicy) *structs.ScalingPolicy { - return &structs.ScalingPolicy{ - Namespace: job.Namespace, - JobID: job.ID, - Enabled: *a1.Enabled, - Policy: a1.Policy, - } -} diff --git a/command/agent/scaling_endpoint.go b/command/agent/scaling_endpoint.go new file mode 100644 index 00000000000..54bf220c508 --- /dev/null +++ b/command/agent/scaling_endpoint.go @@ -0,0 +1,118 @@ +package agent + +import ( + "net/http" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/nomad/structs" +) + +func (s *HTTPServer) ScalingPoliciesRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + switch req.Method { + case "GET": + return s.scalingPoliciesListRequest(resp, req) + default: + return nil, CodedError(405, ErrInvalidMethod) + } +} + +func (s *HTTPServer) scalingPoliciesListRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + args := structs.ScalingPolicyListRequest{} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + var out structs.ScalingPolicyListResponse + if err := s.agent.RPC("Scaling.ListPolicies", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + if out.Policies == nil { + out.Policies = make([]*structs.ScalingPolicyListStub, 0) + } + return out.Policies, nil +} + +func (s *HTTPServer) ScalingPolicySpecificRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + path := strings.TrimPrefix(req.URL.Path, "/v1/scaling/policy/") + switch { + default: + return s.scalingPolicyCRUD(resp, req, path) + } +} + +// func (s *HTTPServer) ValidateJobRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { +// // Ensure request method is POST or PUT +// if !(req.Method == "POST" || req.Method == "PUT") { +// return nil, CodedError(405, ErrInvalidMethod) +// } +// +// var validateRequest api.JobValidateRequest +// if err := decodeBody(req, &validateRequest); err != nil { +// return nil, CodedError(400, err.Error()) +// } +// if validateRequest.Job == nil { +// return nil, CodedError(400, "Job must be specified") +// } +// +// job := ApiJobToStructJob(validateRequest.Job) +// +// args := structs.JobValidateRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: validateRequest.Region, +// }, +// } +// s.parseWriteRequest(req, &args.WriteRequest) +// args.Namespace = job.Namespace +// +// var out structs.JobValidateResponse +// if err := s.agent.RPC("Job.Validate", &args, &out); err != nil { +// return nil, err +// } +// +// return out, nil +// } + +func (s *HTTPServer) scalingPolicyCRUD(resp http.ResponseWriter, req *http.Request, + policyID string) (interface{}, error) { + switch req.Method { + case "GET": + return s.scalingPolicyQuery(resp, req, policyID) + default: + return nil, CodedError(405, ErrInvalidMethod) + } +} + +func (s *HTTPServer) scalingPolicyQuery(resp http.ResponseWriter, req *http.Request, + policyID string) (interface{}, error) { + args := structs.ScalingPolicySpecificRequest{ + ID: policyID, + } + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + var out structs.SingleScalingPolicyResponse + if err := s.agent.RPC("Scaling.GetPolicy", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + if out.Policy == nil { + return nil, CodedError(404, "policy not found") + } + + return out.Policy, nil +} + +func ApiScalingPolicyToStructs(job *structs.Job, a1 *api.ScalingPolicy) *structs.ScalingPolicy { + return &structs.ScalingPolicy{ + Namespace: job.Namespace, + JobID: job.ID, + Enabled: *a1.Enabled, + Policy: a1.Policy, + } +} diff --git a/command/agent/scaling_endpoint_test.go b/command/agent/scaling_endpoint_test.go new file mode 100644 index 00000000000..c0d44737ced --- /dev/null +++ b/command/agent/scaling_endpoint_test.go @@ -0,0 +1,2174 @@ +package agent + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestHTTP_ScalingPoliciesList(t *testing.T) { + t.Parallel() + httpTest(t, nil, func(s *TestAgent) { + for i := 0; i < 3; i++ { + // Create the job + job, _ := mock.JobWithScalingPolicy() + + args := structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: structs.DefaultNamespace, + }, + } + var resp structs.JobRegisterResponse + if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { + t.Fatalf("err: %v", err) + } + } + + // Make the HTTP request + req, err := http.NewRequest("GET", "/v1/scaling/policies", nil) + if err != nil { + t.Fatalf("err: %v", err) + } + respW := httptest.NewRecorder() + + // Make the request + obj, err := s.Server.ScalingPoliciesRequest(respW, req) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Check for the index + if respW.Header().Get("X-Nomad-Index") == "" { + t.Fatalf("missing index") + } + if respW.Header().Get("X-Nomad-KnownLeader") != "true" { + t.Fatalf("missing known leader") + } + if respW.Header().Get("X-Nomad-LastContact") == "" { + t.Fatalf("missing last contact") + } + + // Check the list + l := obj.([]*structs.ScalingPolicyListStub) + if len(l) != 3 { + t.Fatalf("bad: %#v", l) + } + }) +} + +func TestHTTP_ScalingPolicyGet(t *testing.T) { + t.Parallel() + require := require.New(t) + httpTest(t, nil, func(s *TestAgent) { + // Create the job + job, p := mock.JobWithScalingPolicy() + args := structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: structs.DefaultNamespace, + }, + } + var resp structs.JobRegisterResponse + err := s.Agent.RPC("Job.Register", &args, &resp) + require.NoError(err) + + // Make the HTTP request + req, err := http.NewRequest("GET", "/v1/scaling/policy/"+p.ID, nil) + require.NoError(err) + respW := httptest.NewRecorder() + + // Make the request + obj, err := s.Server.ScalingPolicySpecificRequest(respW, req) + require.NoError(err) + + // Check for the index + if respW.Header().Get("X-Nomad-Index") == "" { + t.Fatalf("missing index") + } + if respW.Header().Get("X-Nomad-KnownLeader") != "true" { + t.Fatalf("missing known leader") + } + if respW.Header().Get("X-Nomad-LastContact") == "" { + t.Fatalf("missing last contact") + } + + // Check the policy + require.Equal(p.ID, obj.(*structs.ScalingPolicy).ID) + }) +} + +// func TestHTTP_JobQuery_Payload(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := mock.Job() +// +// // Insert Payload compressed +// expected := []byte("hello world") +// compressed := snappy.Encode(nil, expected) +// job.Payload = compressed +// +// // Directly manipulate the state +// state := s.Agent.server.State() +// if err := state.UpsertJob(1000, job); err != nil { +// t.Fatalf("Failed to upsert job: %v", err) +// } +// +// // Make the HTTP request +// req, err := http.NewRequest("GET", "/v1/job/"+job.ID, nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { +// t.Fatalf("missing known leader") +// } +// if respW.Header().Get("X-Nomad-LastContact") == "" { +// t.Fatalf("missing last contact") +// } +// +// // Check the job +// j := obj.(*structs.Job) +// if j.ID != job.ID { +// t.Fatalf("bad: %#v", j) +// } +// +// // Check the payload is decompressed +// if !reflect.DeepEqual(j.Payload, expected) { +// t.Fatalf("Payload not decompressed properly; got %#v; want %#v", j.Payload, expected) +// } +// }) +// } +// +// func TestHTTP_JobUpdate(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := MockJob() +// args := api.JobRegisterRequest{ +// Job: job, +// WriteRequest: api.WriteRequest{ +// Region: "global", +// Namespace: api.DefaultNamespace, +// }, +// } +// buf := encodeReq(args) +// +// // Make the HTTP request +// req, err := http.NewRequest("PUT", "/v1/job/"+*job.ID, buf) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// dereg := obj.(structs.JobRegisterResponse) +// if dereg.EvalID == "" { +// t.Fatalf("bad: %v", dereg) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// +// // Check the job is registered +// getReq := structs.JobSpecificRequest{ +// JobID: *job.ID, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var getResp structs.SingleJobResponse +// if err := s.Agent.RPC("Job.GetJob", &getReq, &getResp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// if getResp.Job == nil { +// t.Fatalf("job does not exist") +// } +// }) +// } +// +// func TestHTTP_JobUpdateRegion(t *testing.T) { +// t.Parallel() +// +// cases := []struct { +// Name string +// ConfigRegion string +// APIRegion string +// ExpectedRegion string +// }{ +// { +// Name: "api region takes precedence", +// ConfigRegion: "not-global", +// APIRegion: "north-america", +// ExpectedRegion: "north-america", +// }, +// { +// Name: "config region is set", +// ConfigRegion: "north-america", +// APIRegion: "", +// ExpectedRegion: "north-america", +// }, +// { +// Name: "api region is set", +// ConfigRegion: "", +// APIRegion: "north-america", +// ExpectedRegion: "north-america", +// }, +// { +// Name: "defaults to node region global if no region is provided", +// ConfigRegion: "", +// APIRegion: "", +// ExpectedRegion: "global", +// }, +// { +// Name: "defaults to node region not-global if no region is provided", +// ConfigRegion: "", +// APIRegion: "", +// ExpectedRegion: "not-global", +// }, +// } +// +// for _, tc := range cases { +// t.Run(tc.Name, func(t *testing.T) { +// httpTest(t, func(c *Config) { c.Region = tc.ExpectedRegion }, func(s *TestAgent) { +// // Create the job +// job := MockRegionalJob() +// +// if tc.ConfigRegion == "" { +// job.Region = nil +// } else { +// job.Region = &tc.ConfigRegion +// } +// +// args := api.JobRegisterRequest{ +// Job: job, +// WriteRequest: api.WriteRequest{ +// Namespace: api.DefaultNamespace, +// Region: tc.APIRegion, +// }, +// } +// +// buf := encodeReq(args) +// +// // Make the HTTP request +// url := "/v1/job/" + *job.ID +// +// req, err := http.NewRequest("PUT", url, buf) +// require.NoError(t, err) +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// require.NoError(t, err) +// +// // Check the response +// dereg := obj.(structs.JobRegisterResponse) +// require.NotEmpty(t, dereg.EvalID) +// +// // Check for the index +// require.NotEmpty(t, respW.Header().Get("X-Nomad-Index"), "missing index") +// +// // Check the job is registered +// getReq := structs.JobSpecificRequest{ +// JobID: *job.ID, +// QueryOptions: structs.QueryOptions{ +// Region: tc.ExpectedRegion, +// Namespace: structs.DefaultNamespace, +// }, +// } +// var getResp structs.SingleJobResponse +// err = s.Agent.RPC("Job.GetJob", &getReq, &getResp) +// require.NoError(t, err) +// require.NotNil(t, getResp.Job, "job does not exist") +// require.Equal(t, tc.ExpectedRegion, getResp.Job.Region) +// }) +// }) +// } +// } +// +// func TestHTTP_JobDelete(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := mock.Job() +// args := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Make the HTTP request to do a soft delete +// req, err := http.NewRequest("DELETE", "/v1/job/"+job.ID, nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// dereg := obj.(structs.JobDeregisterResponse) +// if dereg.EvalID == "" { +// t.Fatalf("bad: %v", dereg) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// +// // Check the job is still queryable +// getReq1 := structs.JobSpecificRequest{ +// JobID: job.ID, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var getResp1 structs.SingleJobResponse +// if err := s.Agent.RPC("Job.GetJob", &getReq1, &getResp1); err != nil { +// t.Fatalf("err: %v", err) +// } +// if getResp1.Job == nil { +// t.Fatalf("job doesn't exists") +// } +// if !getResp1.Job.Stop { +// t.Fatalf("job should be marked as stop") +// } +// +// // Make the HTTP request to do a purge delete +// req2, err := http.NewRequest("DELETE", "/v1/job/"+job.ID+"?purge=true", nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW.Flush() +// +// // Make the request +// obj, err = s.Server.JobSpecificRequest(respW, req2) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// dereg = obj.(structs.JobDeregisterResponse) +// if dereg.EvalID == "" { +// t.Fatalf("bad: %v", dereg) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// +// // Check the job is gone +// getReq2 := structs.JobSpecificRequest{ +// JobID: job.ID, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var getResp2 structs.SingleJobResponse +// if err := s.Agent.RPC("Job.GetJob", &getReq2, &getResp2); err != nil { +// t.Fatalf("err: %v", err) +// } +// if getResp2.Job != nil { +// t.Fatalf("job still exists") +// } +// }) +// } +// +// func TestHTTP_JobForceEvaluate(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := mock.Job() +// args := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Make the HTTP request +// req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/evaluate", nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// reg := obj.(structs.JobRegisterResponse) +// if reg.EvalID == "" { +// t.Fatalf("bad: %v", reg) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// }) +// } +// +// func TestHTTP_JobEvaluate_ForceReschedule(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := mock.Job() +// args := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// jobEvalReq := api.JobEvaluateRequest{ +// JobID: job.ID, +// EvalOptions: api.EvalOptions{ +// ForceReschedule: true, +// }, +// } +// +// buf := encodeReq(jobEvalReq) +// +// // Make the HTTP request +// req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/evaluate", buf) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// reg := obj.(structs.JobRegisterResponse) +// if reg.EvalID == "" { +// t.Fatalf("bad: %v", reg) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// }) +// } +// +// func TestHTTP_JobEvaluations(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := mock.Job() +// args := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Make the HTTP request +// req, err := http.NewRequest("GET", "/v1/job/"+job.ID+"/evaluations", nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// evals := obj.([]*structs.Evaluation) +// // Can be multiple evals, use the last one, since they are in order +// idx := len(evals) - 1 +// if len(evals) < 0 || evals[idx].ID != resp.EvalID { +// t.Fatalf("bad: %v", evals) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { +// t.Fatalf("missing known leader") +// } +// if respW.Header().Get("X-Nomad-LastContact") == "" { +// t.Fatalf("missing last contact") +// } +// }) +// } +// +// func TestHTTP_JobAllocations(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// alloc1 := mock.Alloc() +// args := structs.JobRegisterRequest{ +// Job: alloc1.Job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Directly manipulate the state +// expectedDisplayMsg := "test message" +// testEvent := structs.NewTaskEvent("test event").SetMessage(expectedDisplayMsg) +// var events []*structs.TaskEvent +// events = append(events, testEvent) +// taskState := &structs.TaskState{Events: events} +// alloc1.TaskStates = make(map[string]*structs.TaskState) +// alloc1.TaskStates["test"] = taskState +// state := s.Agent.server.State() +// err := state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Make the HTTP request +// req, err := http.NewRequest("GET", "/v1/job/"+alloc1.Job.ID+"/allocations?all=true", nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// allocs := obj.([]*structs.AllocListStub) +// if len(allocs) != 1 && allocs[0].ID != alloc1.ID { +// t.Fatalf("bad: %v", allocs) +// } +// displayMsg := allocs[0].TaskStates["test"].Events[0].DisplayMessage +// assert.Equal(t, expectedDisplayMsg, displayMsg) +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { +// t.Fatalf("missing known leader") +// } +// if respW.Header().Get("X-Nomad-LastContact") == "" { +// t.Fatalf("missing last contact") +// } +// }) +// } +// +// func TestHTTP_JobDeployments(t *testing.T) { +// assert := assert.New(t) +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// j := mock.Job() +// args := structs.JobRegisterRequest{ +// Job: j, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// assert.Nil(s.Agent.RPC("Job.Register", &args, &resp), "JobRegister") +// +// // Directly manipulate the state +// state := s.Agent.server.State() +// d := mock.Deployment() +// d.JobID = j.ID +// d.JobCreateIndex = resp.JobModifyIndex +// +// assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") +// +// // Make the HTTP request +// req, err := http.NewRequest("GET", "/v1/job/"+j.ID+"/deployments", nil) +// assert.Nil(err, "HTTP") +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// assert.Nil(err, "JobSpecificRequest") +// +// // Check the response +// deploys := obj.([]*structs.Deployment) +// assert.Len(deploys, 1, "deployments") +// assert.Equal(d.ID, deploys[0].ID, "deployment id") +// +// assert.NotZero(respW.Header().Get("X-Nomad-Index"), "missing index") +// assert.Equal("true", respW.Header().Get("X-Nomad-KnownLeader"), "missing known leader") +// assert.NotZero(respW.Header().Get("X-Nomad-LastContact"), "missing last contact") +// }) +// } +// +// func TestHTTP_JobDeployment(t *testing.T) { +// assert := assert.New(t) +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// j := mock.Job() +// args := structs.JobRegisterRequest{ +// Job: j, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// assert.Nil(s.Agent.RPC("Job.Register", &args, &resp), "JobRegister") +// +// // Directly manipulate the state +// state := s.Agent.server.State() +// d := mock.Deployment() +// d.JobID = j.ID +// d.JobCreateIndex = resp.JobModifyIndex +// assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") +// +// // Make the HTTP request +// req, err := http.NewRequest("GET", "/v1/job/"+j.ID+"/deployment", nil) +// assert.Nil(err, "HTTP") +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// assert.Nil(err, "JobSpecificRequest") +// +// // Check the response +// out := obj.(*structs.Deployment) +// assert.NotNil(out, "deployment") +// assert.Equal(d.ID, out.ID, "deployment id") +// +// assert.NotZero(respW.Header().Get("X-Nomad-Index"), "missing index") +// assert.Equal("true", respW.Header().Get("X-Nomad-KnownLeader"), "missing known leader") +// assert.NotZero(respW.Header().Get("X-Nomad-LastContact"), "missing last contact") +// }) +// } +// +// func TestHTTP_JobVersions(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := mock.Job() +// args := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// job2 := mock.Job() +// job2.ID = job.ID +// job2.Priority = 100 +// +// args2 := structs.JobRegisterRequest{ +// Job: job2, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp2 structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args2, &resp2); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Make the HTTP request +// req, err := http.NewRequest("GET", "/v1/job/"+job.ID+"/versions?diffs=true", nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// vResp := obj.(structs.JobVersionsResponse) +// versions := vResp.Versions +// if len(versions) != 2 { +// t.Fatalf("got %d versions; want 2", len(versions)) +// } +// +// if v := versions[0]; v.Version != 1 || v.Priority != 100 { +// t.Fatalf("bad %v", v) +// } +// +// if v := versions[1]; v.Version != 0 { +// t.Fatalf("bad %v", v) +// } +// +// if len(vResp.Diffs) != 1 { +// t.Fatalf("bad %v", vResp) +// } +// +// // Check for the index +// if respW.Header().Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { +// t.Fatalf("missing known leader") +// } +// if respW.Header().Get("X-Nomad-LastContact") == "" { +// t.Fatalf("missing last contact") +// } +// }) +// } +// +// func TestHTTP_PeriodicForce(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create and register a periodic job. +// job := mock.PeriodicJob() +// args := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Make the HTTP request +// req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/periodic/force", nil) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check for the index +// if respW.HeaderMap.Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// +// // Check the response +// r := obj.(structs.PeriodicForceResponse) +// if r.EvalID == "" { +// t.Fatalf("bad: %#v", r) +// } +// }) +// } +// +// func TestHTTP_JobPlan(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := MockJob() +// args := api.JobPlanRequest{ +// Job: job, +// Diff: true, +// WriteRequest: api.WriteRequest{ +// Region: "global", +// Namespace: api.DefaultNamespace, +// }, +// } +// buf := encodeReq(args) +// +// // Make the HTTP request +// req, err := http.NewRequest("PUT", "/v1/job/"+*job.ID+"/plan", buf) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// plan := obj.(structs.JobPlanResponse) +// if plan.Annotations == nil { +// t.Fatalf("bad: %v", plan) +// } +// +// if plan.Diff == nil { +// t.Fatalf("bad: %v", plan) +// } +// }) +// } +// +// func TestHTTP_JobPlanRegion(t *testing.T) { +// t.Parallel() +// +// cases := []struct { +// Name string +// ConfigRegion string +// APIRegion string +// ExpectedRegion string +// }{ +// { +// Name: "api region takes precedence", +// ConfigRegion: "not-global", +// APIRegion: "north-america", +// ExpectedRegion: "north-america", +// }, +// { +// Name: "config region is set", +// ConfigRegion: "north-america", +// APIRegion: "", +// ExpectedRegion: "north-america", +// }, +// { +// Name: "api region is set", +// ConfigRegion: "", +// APIRegion: "north-america", +// ExpectedRegion: "north-america", +// }, +// { +// Name: "falls back to default if no region is provided", +// ConfigRegion: "", +// APIRegion: "", +// ExpectedRegion: "global", +// }, +// } +// +// for _, tc := range cases { +// t.Run(tc.Name, func(t *testing.T) { +// httpTest(t, func(c *Config) { c.Region = tc.ExpectedRegion }, func(s *TestAgent) { +// // Create the job +// job := MockRegionalJob() +// +// if tc.ConfigRegion == "" { +// job.Region = nil +// } else { +// job.Region = &tc.ConfigRegion +// } +// +// args := api.JobPlanRequest{ +// Job: job, +// Diff: true, +// WriteRequest: api.WriteRequest{ +// Region: tc.APIRegion, +// Namespace: api.DefaultNamespace, +// }, +// } +// buf := encodeReq(args) +// +// // Make the HTTP request +// req, err := http.NewRequest("PUT", "/v1/job/"+*job.ID+"/plan", buf) +// require.NoError(t, err) +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// require.NoError(t, err) +// +// // Check the response +// plan := obj.(structs.JobPlanResponse) +// require.NotNil(t, plan.Annotations) +// require.NotNil(t, plan.Diff) +// }) +// }) +// } +// } +// +// func TestHTTP_JobDispatch(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the parameterized job +// job := mock.BatchJob() +// job.ParameterizedJob = &structs.ParameterizedJobConfig{} +// +// args := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var resp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Make the request +// respW := httptest.NewRecorder() +// args2 := structs.JobDispatchRequest{ +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// buf := encodeReq(args2) +// +// // Make the HTTP request +// req2, err := http.NewRequest("PUT", "/v1/job/"+job.ID+"/dispatch", buf) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW.Flush() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req2) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// dispatch := obj.(structs.JobDispatchResponse) +// if dispatch.EvalID == "" { +// t.Fatalf("bad: %v", dispatch) +// } +// +// if dispatch.DispatchedJobID == "" { +// t.Fatalf("bad: %v", dispatch) +// } +// }) +// } +// +// func TestHTTP_JobRevert(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job and register it twice +// job := mock.Job() +// regReq := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var regResp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Change the job to get a new version +// job.Datacenters = append(job.Datacenters, "foo") +// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// args := structs.JobRevertRequest{ +// JobID: job.ID, +// JobVersion: 0, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// buf := encodeReq(args) +// +// // Make the HTTP request +// req, err := http.NewRequest("PUT", "/v1/job/"+job.ID+"/revert", buf) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// revertResp := obj.(structs.JobRegisterResponse) +// if revertResp.EvalID == "" { +// t.Fatalf("bad: %v", revertResp) +// } +// +// // Check for the index +// if respW.HeaderMap.Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// }) +// } +// +// func TestHTTP_JobStable(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job and register it twice +// job := mock.Job() +// regReq := structs.JobRegisterRequest{ +// Job: job, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// var regResp structs.JobRegisterResponse +// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// args := structs.JobStabilityRequest{ +// JobID: job.ID, +// JobVersion: 0, +// Stable: true, +// WriteRequest: structs.WriteRequest{ +// Region: "global", +// Namespace: structs.DefaultNamespace, +// }, +// } +// buf := encodeReq(args) +// +// // Make the HTTP request +// req, err := http.NewRequest("PUT", "/v1/job/"+job.ID+"/stable", buf) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.JobSpecificRequest(respW, req) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// +// // Check the response +// stableResp := obj.(structs.JobStabilityResponse) +// if stableResp.Index == 0 { +// t.Fatalf("bad: %v", stableResp) +// } +// +// // Check for the index +// if respW.HeaderMap.Get("X-Nomad-Index") == "" { +// t.Fatalf("missing index") +// } +// }) +// } +// +// func TestJobs_ApiJobToStructsJob(t *testing.T) { +// apiJob := &api.Job{ +// Stop: helper.BoolToPtr(true), +// Region: helper.StringToPtr("global"), +// Namespace: helper.StringToPtr("foo"), +// ID: helper.StringToPtr("foo"), +// ParentID: helper.StringToPtr("lol"), +// Name: helper.StringToPtr("name"), +// Type: helper.StringToPtr("service"), +// Priority: helper.IntToPtr(50), +// AllAtOnce: helper.BoolToPtr(true), +// Datacenters: []string{"dc1", "dc2"}, +// Constraints: []*api.Constraint{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// }, +// }, +// Affinities: []*api.Affinity{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// Weight: helper.Int8ToPtr(50), +// }, +// }, +// Update: &api.UpdateStrategy{ +// Stagger: helper.TimeToPtr(1 * time.Second), +// MaxParallel: helper.IntToPtr(5), +// HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual), +// MinHealthyTime: helper.TimeToPtr(1 * time.Minute), +// HealthyDeadline: helper.TimeToPtr(3 * time.Minute), +// ProgressDeadline: helper.TimeToPtr(3 * time.Minute), +// AutoRevert: helper.BoolToPtr(false), +// Canary: helper.IntToPtr(1), +// }, +// Spreads: []*api.Spread{ +// { +// Attribute: "${meta.rack}", +// Weight: helper.Int8ToPtr(100), +// SpreadTarget: []*api.SpreadTarget{ +// { +// Value: "r1", +// Percent: 50, +// }, +// }, +// }, +// }, +// Periodic: &api.PeriodicConfig{ +// Enabled: helper.BoolToPtr(true), +// Spec: helper.StringToPtr("spec"), +// SpecType: helper.StringToPtr("cron"), +// ProhibitOverlap: helper.BoolToPtr(true), +// TimeZone: helper.StringToPtr("test zone"), +// }, +// ParameterizedJob: &api.ParameterizedJobConfig{ +// Payload: "payload", +// MetaRequired: []string{"a", "b"}, +// MetaOptional: []string{"c", "d"}, +// }, +// Payload: []byte("payload"), +// Meta: map[string]string{ +// "foo": "bar", +// }, +// TaskGroups: []*api.TaskGroup{ +// { +// Name: helper.StringToPtr("group1"), +// Count: helper.IntToPtr(5), +// Constraints: []*api.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Affinities: []*api.Affinity{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// Weight: helper.Int8ToPtr(100), +// }, +// }, +// RestartPolicy: &api.RestartPolicy{ +// Interval: helper.TimeToPtr(1 * time.Second), +// Attempts: helper.IntToPtr(5), +// Delay: helper.TimeToPtr(10 * time.Second), +// Mode: helper.StringToPtr("delay"), +// }, +// ReschedulePolicy: &api.ReschedulePolicy{ +// Interval: helper.TimeToPtr(12 * time.Hour), +// Attempts: helper.IntToPtr(5), +// DelayFunction: helper.StringToPtr("constant"), +// Delay: helper.TimeToPtr(30 * time.Second), +// Unlimited: helper.BoolToPtr(true), +// MaxDelay: helper.TimeToPtr(20 * time.Minute), +// }, +// Migrate: &api.MigrateStrategy{ +// MaxParallel: helper.IntToPtr(12), +// HealthCheck: helper.StringToPtr("task_events"), +// MinHealthyTime: helper.TimeToPtr(12 * time.Hour), +// HealthyDeadline: helper.TimeToPtr(12 * time.Hour), +// }, +// Spreads: []*api.Spread{ +// { +// Attribute: "${node.datacenter}", +// Weight: helper.Int8ToPtr(100), +// SpreadTarget: []*api.SpreadTarget{ +// { +// Value: "dc1", +// Percent: 100, +// }, +// }, +// }, +// }, +// EphemeralDisk: &api.EphemeralDisk{ +// SizeMB: helper.IntToPtr(100), +// Sticky: helper.BoolToPtr(true), +// Migrate: helper.BoolToPtr(true), +// }, +// Update: &api.UpdateStrategy{ +// HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Checks), +// MinHealthyTime: helper.TimeToPtr(2 * time.Minute), +// HealthyDeadline: helper.TimeToPtr(5 * time.Minute), +// ProgressDeadline: helper.TimeToPtr(5 * time.Minute), +// AutoRevert: helper.BoolToPtr(true), +// }, +// Meta: map[string]string{ +// "key": "value", +// }, +// Services: []*api.Service{ +// { +// Name: "groupserviceA", +// Tags: []string{"a", "b"}, +// CanaryTags: []string{"d", "e"}, +// PortLabel: "1234", +// Meta: map[string]string{ +// "servicemeta": "foobar", +// }, +// CheckRestart: &api.CheckRestart{ +// Limit: 4, +// Grace: helper.TimeToPtr(11 * time.Second), +// }, +// Checks: []api.ServiceCheck{ +// { +// Id: "hello", +// Name: "bar", +// Type: "http", +// Command: "foo", +// Args: []string{"a", "b"}, +// Path: "/check", +// Protocol: "http", +// PortLabel: "foo", +// AddressMode: "driver", +// GRPCService: "foo.Bar", +// GRPCUseTLS: true, +// Interval: 4 * time.Second, +// Timeout: 2 * time.Second, +// InitialStatus: "ok", +// CheckRestart: &api.CheckRestart{ +// Limit: 3, +// IgnoreWarnings: true, +// }, +// TaskName: "task1", +// }, +// }, +// Connect: &api.ConsulConnect{ +// Native: false, +// SidecarService: &api.ConsulSidecarService{ +// Tags: []string{"f", "g"}, +// Port: "9000", +// }, +// }, +// }, +// }, +// Tasks: []*api.Task{ +// { +// Name: "task1", +// Leader: true, +// Driver: "docker", +// User: "mary", +// Config: map[string]interface{}{ +// "lol": "code", +// }, +// Env: map[string]string{ +// "hello": "world", +// }, +// Constraints: []*api.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Affinities: []*api.Affinity{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// Weight: helper.Int8ToPtr(50), +// }, +// }, +// +// Services: []*api.Service{ +// { +// Id: "id", +// Name: "serviceA", +// Tags: []string{"1", "2"}, +// CanaryTags: []string{"3", "4"}, +// PortLabel: "foo", +// Meta: map[string]string{ +// "servicemeta": "foobar", +// }, +// CheckRestart: &api.CheckRestart{ +// Limit: 4, +// Grace: helper.TimeToPtr(11 * time.Second), +// }, +// Checks: []api.ServiceCheck{ +// { +// Id: "hello", +// Name: "bar", +// Type: "http", +// Command: "foo", +// Args: []string{"a", "b"}, +// Path: "/check", +// Protocol: "http", +// PortLabel: "foo", +// AddressMode: "driver", +// GRPCService: "foo.Bar", +// GRPCUseTLS: true, +// Interval: 4 * time.Second, +// Timeout: 2 * time.Second, +// InitialStatus: "ok", +// CheckRestart: &api.CheckRestart{ +// Limit: 3, +// IgnoreWarnings: true, +// }, +// }, +// { +// Id: "check2id", +// Name: "check2", +// Type: "tcp", +// PortLabel: "foo", +// Interval: 4 * time.Second, +// Timeout: 2 * time.Second, +// }, +// }, +// }, +// }, +// Resources: &api.Resources{ +// CPU: helper.IntToPtr(100), +// MemoryMB: helper.IntToPtr(10), +// Networks: []*api.NetworkResource{ +// { +// IP: "10.10.11.1", +// MBits: helper.IntToPtr(10), +// ReservedPorts: []api.Port{ +// { +// Label: "http", +// Value: 80, +// }, +// }, +// DynamicPorts: []api.Port{ +// { +// Label: "ssh", +// Value: 2000, +// }, +// }, +// }, +// }, +// Devices: []*api.RequestedDevice{ +// { +// Name: "nvidia/gpu", +// Count: helper.Uint64ToPtr(4), +// Constraints: []*api.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Affinities: []*api.Affinity{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// Weight: helper.Int8ToPtr(50), +// }, +// }, +// }, +// { +// Name: "gpu", +// Count: nil, +// }, +// }, +// }, +// Meta: map[string]string{ +// "lol": "code", +// }, +// KillTimeout: helper.TimeToPtr(10 * time.Second), +// KillSignal: "SIGQUIT", +// LogConfig: &api.LogConfig{ +// MaxFiles: helper.IntToPtr(10), +// MaxFileSizeMB: helper.IntToPtr(100), +// }, +// Artifacts: []*api.TaskArtifact{ +// { +// GetterSource: helper.StringToPtr("source"), +// GetterOptions: map[string]string{ +// "a": "b", +// }, +// GetterMode: helper.StringToPtr("dir"), +// RelativeDest: helper.StringToPtr("dest"), +// }, +// }, +// Vault: &api.Vault{ +// Policies: []string{"a", "b", "c"}, +// Env: helper.BoolToPtr(true), +// ChangeMode: helper.StringToPtr("c"), +// ChangeSignal: helper.StringToPtr("sighup"), +// }, +// Templates: []*api.Template{ +// { +// SourcePath: helper.StringToPtr("source"), +// DestPath: helper.StringToPtr("dest"), +// EmbeddedTmpl: helper.StringToPtr("embedded"), +// ChangeMode: helper.StringToPtr("change"), +// ChangeSignal: helper.StringToPtr("signal"), +// Splay: helper.TimeToPtr(1 * time.Minute), +// Perms: helper.StringToPtr("666"), +// LeftDelim: helper.StringToPtr("abc"), +// RightDelim: helper.StringToPtr("def"), +// Envvars: helper.BoolToPtr(true), +// VaultGrace: helper.TimeToPtr(3 * time.Second), +// }, +// }, +// DispatchPayload: &api.DispatchPayloadConfig{ +// File: "fileA", +// }, +// }, +// }, +// }, +// }, +// VaultToken: helper.StringToPtr("token"), +// Status: helper.StringToPtr("status"), +// StatusDescription: helper.StringToPtr("status_desc"), +// Version: helper.Uint64ToPtr(10), +// CreateIndex: helper.Uint64ToPtr(1), +// ModifyIndex: helper.Uint64ToPtr(3), +// JobModifyIndex: helper.Uint64ToPtr(5), +// } +// +// expected := &structs.Job{ +// Stop: true, +// Region: "global", +// Namespace: "foo", +// ID: "foo", +// ParentID: "lol", +// Name: "name", +// Type: "service", +// Priority: 50, +// AllAtOnce: true, +// Datacenters: []string{"dc1", "dc2"}, +// Constraints: []*structs.Constraint{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// }, +// }, +// Affinities: []*structs.Affinity{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// Weight: 50, +// }, +// }, +// Spreads: []*structs.Spread{ +// { +// Attribute: "${meta.rack}", +// Weight: 100, +// SpreadTarget: []*structs.SpreadTarget{ +// { +// Value: "r1", +// Percent: 50, +// }, +// }, +// }, +// }, +// Update: structs.UpdateStrategy{ +// Stagger: 1 * time.Second, +// MaxParallel: 5, +// }, +// Periodic: &structs.PeriodicConfig{ +// Enabled: true, +// Spec: "spec", +// SpecType: "cron", +// ProhibitOverlap: true, +// TimeZone: "test zone", +// }, +// ParameterizedJob: &structs.ParameterizedJobConfig{ +// Payload: "payload", +// MetaRequired: []string{"a", "b"}, +// MetaOptional: []string{"c", "d"}, +// }, +// Payload: []byte("payload"), +// Meta: map[string]string{ +// "foo": "bar", +// }, +// TaskGroups: []*structs.TaskGroup{ +// { +// Name: "group1", +// Count: 5, +// Constraints: []*structs.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Affinities: []*structs.Affinity{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// Weight: 100, +// }, +// }, +// RestartPolicy: &structs.RestartPolicy{ +// Interval: 1 * time.Second, +// Attempts: 5, +// Delay: 10 * time.Second, +// Mode: "delay", +// }, +// Spreads: []*structs.Spread{ +// { +// Attribute: "${node.datacenter}", +// Weight: 100, +// SpreadTarget: []*structs.SpreadTarget{ +// { +// Value: "dc1", +// Percent: 100, +// }, +// }, +// }, +// }, +// ReschedulePolicy: &structs.ReschedulePolicy{ +// Interval: 12 * time.Hour, +// Attempts: 5, +// DelayFunction: "constant", +// Delay: 30 * time.Second, +// Unlimited: true, +// MaxDelay: 20 * time.Minute, +// }, +// Migrate: &structs.MigrateStrategy{ +// MaxParallel: 12, +// HealthCheck: "task_events", +// MinHealthyTime: 12 * time.Hour, +// HealthyDeadline: 12 * time.Hour, +// }, +// EphemeralDisk: &structs.EphemeralDisk{ +// SizeMB: 100, +// Sticky: true, +// Migrate: true, +// }, +// Update: &structs.UpdateStrategy{ +// Stagger: 1 * time.Second, +// MaxParallel: 5, +// HealthCheck: structs.UpdateStrategyHealthCheck_Checks, +// MinHealthyTime: 2 * time.Minute, +// HealthyDeadline: 5 * time.Minute, +// ProgressDeadline: 5 * time.Minute, +// AutoRevert: true, +// AutoPromote: false, +// Canary: 1, +// }, +// Meta: map[string]string{ +// "key": "value", +// }, +// Services: []*structs.Service{ +// { +// Name: "groupserviceA", +// Tags: []string{"a", "b"}, +// CanaryTags: []string{"d", "e"}, +// PortLabel: "1234", +// AddressMode: "auto", +// Meta: map[string]string{ +// "servicemeta": "foobar", +// }, +// Checks: []*structs.ServiceCheck{ +// { +// Name: "bar", +// Type: "http", +// Command: "foo", +// Args: []string{"a", "b"}, +// Path: "/check", +// Protocol: "http", +// PortLabel: "foo", +// AddressMode: "driver", +// GRPCService: "foo.Bar", +// GRPCUseTLS: true, +// Interval: 4 * time.Second, +// Timeout: 2 * time.Second, +// InitialStatus: "ok", +// CheckRestart: &structs.CheckRestart{ +// Grace: 11 * time.Second, +// Limit: 3, +// IgnoreWarnings: true, +// }, +// TaskName: "task1", +// }, +// }, +// Connect: &structs.ConsulConnect{ +// Native: false, +// SidecarService: &structs.ConsulSidecarService{ +// Tags: []string{"f", "g"}, +// Port: "9000", +// }, +// }, +// }, +// }, +// Tasks: []*structs.Task{ +// { +// Name: "task1", +// Driver: "docker", +// Leader: true, +// User: "mary", +// Config: map[string]interface{}{ +// "lol": "code", +// }, +// Constraints: []*structs.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Affinities: []*structs.Affinity{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// Weight: 50, +// }, +// }, +// Env: map[string]string{ +// "hello": "world", +// }, +// Services: []*structs.Service{ +// { +// Name: "serviceA", +// Tags: []string{"1", "2"}, +// CanaryTags: []string{"3", "4"}, +// PortLabel: "foo", +// AddressMode: "auto", +// Meta: map[string]string{ +// "servicemeta": "foobar", +// }, +// Checks: []*structs.ServiceCheck{ +// { +// Name: "bar", +// Type: "http", +// Command: "foo", +// Args: []string{"a", "b"}, +// Path: "/check", +// Protocol: "http", +// PortLabel: "foo", +// AddressMode: "driver", +// Interval: 4 * time.Second, +// Timeout: 2 * time.Second, +// InitialStatus: "ok", +// GRPCService: "foo.Bar", +// GRPCUseTLS: true, +// CheckRestart: &structs.CheckRestart{ +// Limit: 3, +// Grace: 11 * time.Second, +// IgnoreWarnings: true, +// }, +// }, +// { +// Name: "check2", +// Type: "tcp", +// PortLabel: "foo", +// Interval: 4 * time.Second, +// Timeout: 2 * time.Second, +// CheckRestart: &structs.CheckRestart{ +// Limit: 4, +// Grace: 11 * time.Second, +// }, +// }, +// }, +// }, +// }, +// Resources: &structs.Resources{ +// CPU: 100, +// MemoryMB: 10, +// Networks: []*structs.NetworkResource{ +// { +// IP: "10.10.11.1", +// MBits: 10, +// ReservedPorts: []structs.Port{ +// { +// Label: "http", +// Value: 80, +// }, +// }, +// DynamicPorts: []structs.Port{ +// { +// Label: "ssh", +// Value: 2000, +// }, +// }, +// }, +// }, +// Devices: []*structs.RequestedDevice{ +// { +// Name: "nvidia/gpu", +// Count: 4, +// Constraints: []*structs.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Affinities: []*structs.Affinity{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// Weight: 50, +// }, +// }, +// }, +// { +// Name: "gpu", +// Count: 1, +// }, +// }, +// }, +// Meta: map[string]string{ +// "lol": "code", +// }, +// KillTimeout: 10 * time.Second, +// KillSignal: "SIGQUIT", +// LogConfig: &structs.LogConfig{ +// MaxFiles: 10, +// MaxFileSizeMB: 100, +// }, +// Artifacts: []*structs.TaskArtifact{ +// { +// GetterSource: "source", +// GetterOptions: map[string]string{ +// "a": "b", +// }, +// GetterMode: "dir", +// RelativeDest: "dest", +// }, +// }, +// Vault: &structs.Vault{ +// Policies: []string{"a", "b", "c"}, +// Env: true, +// ChangeMode: "c", +// ChangeSignal: "sighup", +// }, +// Templates: []*structs.Template{ +// { +// SourcePath: "source", +// DestPath: "dest", +// EmbeddedTmpl: "embedded", +// ChangeMode: "change", +// ChangeSignal: "SIGNAL", +// Splay: 1 * time.Minute, +// Perms: "666", +// LeftDelim: "abc", +// RightDelim: "def", +// Envvars: true, +// VaultGrace: 3 * time.Second, +// }, +// }, +// DispatchPayload: &structs.DispatchPayloadConfig{ +// File: "fileA", +// }, +// }, +// }, +// }, +// }, +// +// VaultToken: "token", +// } +// +// structsJob := ApiJobToStructJob(apiJob) +// +// if diff := pretty.Diff(expected, structsJob); len(diff) > 0 { +// t.Fatalf("bad:\n%s", strings.Join(diff, "\n")) +// } +// +// systemAPIJob := &api.Job{ +// Stop: helper.BoolToPtr(true), +// Region: helper.StringToPtr("global"), +// Namespace: helper.StringToPtr("foo"), +// ID: helper.StringToPtr("foo"), +// ParentID: helper.StringToPtr("lol"), +// Name: helper.StringToPtr("name"), +// Type: helper.StringToPtr("system"), +// Priority: helper.IntToPtr(50), +// AllAtOnce: helper.BoolToPtr(true), +// Datacenters: []string{"dc1", "dc2"}, +// Constraints: []*api.Constraint{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// }, +// }, +// TaskGroups: []*api.TaskGroup{ +// { +// Name: helper.StringToPtr("group1"), +// Count: helper.IntToPtr(5), +// Constraints: []*api.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// RestartPolicy: &api.RestartPolicy{ +// Interval: helper.TimeToPtr(1 * time.Second), +// Attempts: helper.IntToPtr(5), +// Delay: helper.TimeToPtr(10 * time.Second), +// Mode: helper.StringToPtr("delay"), +// }, +// EphemeralDisk: &api.EphemeralDisk{ +// SizeMB: helper.IntToPtr(100), +// Sticky: helper.BoolToPtr(true), +// Migrate: helper.BoolToPtr(true), +// }, +// Meta: map[string]string{ +// "key": "value", +// }, +// Tasks: []*api.Task{ +// { +// Name: "task1", +// Leader: true, +// Driver: "docker", +// User: "mary", +// Config: map[string]interface{}{ +// "lol": "code", +// }, +// Env: map[string]string{ +// "hello": "world", +// }, +// Constraints: []*api.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Resources: &api.Resources{ +// CPU: helper.IntToPtr(100), +// MemoryMB: helper.IntToPtr(10), +// Networks: []*api.NetworkResource{ +// { +// IP: "10.10.11.1", +// MBits: helper.IntToPtr(10), +// ReservedPorts: []api.Port{ +// { +// Label: "http", +// Value: 80, +// }, +// }, +// DynamicPorts: []api.Port{ +// { +// Label: "ssh", +// Value: 2000, +// }, +// }, +// }, +// }, +// }, +// Meta: map[string]string{ +// "lol": "code", +// }, +// KillTimeout: helper.TimeToPtr(10 * time.Second), +// KillSignal: "SIGQUIT", +// LogConfig: &api.LogConfig{ +// MaxFiles: helper.IntToPtr(10), +// MaxFileSizeMB: helper.IntToPtr(100), +// }, +// Artifacts: []*api.TaskArtifact{ +// { +// GetterSource: helper.StringToPtr("source"), +// GetterOptions: map[string]string{ +// "a": "b", +// }, +// GetterMode: helper.StringToPtr("dir"), +// RelativeDest: helper.StringToPtr("dest"), +// }, +// }, +// DispatchPayload: &api.DispatchPayloadConfig{ +// File: "fileA", +// }, +// }, +// }, +// }, +// }, +// Status: helper.StringToPtr("status"), +// StatusDescription: helper.StringToPtr("status_desc"), +// Version: helper.Uint64ToPtr(10), +// CreateIndex: helper.Uint64ToPtr(1), +// ModifyIndex: helper.Uint64ToPtr(3), +// JobModifyIndex: helper.Uint64ToPtr(5), +// } +// +// expectedSystemJob := &structs.Job{ +// Stop: true, +// Region: "global", +// Namespace: "foo", +// ID: "foo", +// ParentID: "lol", +// Name: "name", +// Type: "system", +// Priority: 50, +// AllAtOnce: true, +// Datacenters: []string{"dc1", "dc2"}, +// Constraints: []*structs.Constraint{ +// { +// LTarget: "a", +// RTarget: "b", +// Operand: "c", +// }, +// }, +// TaskGroups: []*structs.TaskGroup{ +// { +// Name: "group1", +// Count: 5, +// Constraints: []*structs.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// RestartPolicy: &structs.RestartPolicy{ +// Interval: 1 * time.Second, +// Attempts: 5, +// Delay: 10 * time.Second, +// Mode: "delay", +// }, +// EphemeralDisk: &structs.EphemeralDisk{ +// SizeMB: 100, +// Sticky: true, +// Migrate: true, +// }, +// Meta: map[string]string{ +// "key": "value", +// }, +// Tasks: []*structs.Task{ +// { +// Name: "task1", +// Driver: "docker", +// Leader: true, +// User: "mary", +// Config: map[string]interface{}{ +// "lol": "code", +// }, +// Constraints: []*structs.Constraint{ +// { +// LTarget: "x", +// RTarget: "y", +// Operand: "z", +// }, +// }, +// Env: map[string]string{ +// "hello": "world", +// }, +// Resources: &structs.Resources{ +// CPU: 100, +// MemoryMB: 10, +// Networks: []*structs.NetworkResource{ +// { +// IP: "10.10.11.1", +// MBits: 10, +// ReservedPorts: []structs.Port{ +// { +// Label: "http", +// Value: 80, +// }, +// }, +// DynamicPorts: []structs.Port{ +// { +// Label: "ssh", +// Value: 2000, +// }, +// }, +// }, +// }, +// }, +// Meta: map[string]string{ +// "lol": "code", +// }, +// KillTimeout: 10 * time.Second, +// KillSignal: "SIGQUIT", +// LogConfig: &structs.LogConfig{ +// MaxFiles: 10, +// MaxFileSizeMB: 100, +// }, +// Artifacts: []*structs.TaskArtifact{ +// { +// GetterSource: "source", +// GetterOptions: map[string]string{ +// "a": "b", +// }, +// GetterMode: "dir", +// RelativeDest: "dest", +// }, +// }, +// DispatchPayload: &structs.DispatchPayloadConfig{ +// File: "fileA", +// }, +// }, +// }, +// }, +// }, +// } +// +// systemStructsJob := ApiJobToStructJob(systemAPIJob) +// +// if diff := pretty.Diff(expectedSystemJob, systemStructsJob); len(diff) > 0 { +// t.Fatalf("bad:\n%s", strings.Join(diff, "\n")) +// } +// } +// +// func TestJobs_ApiJobToStructsJobUpdate(t *testing.T) { +// apiJob := &api.Job{ +// Update: &api.UpdateStrategy{ +// Stagger: helper.TimeToPtr(1 * time.Second), +// MaxParallel: helper.IntToPtr(5), +// HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual), +// MinHealthyTime: helper.TimeToPtr(1 * time.Minute), +// HealthyDeadline: helper.TimeToPtr(3 * time.Minute), +// ProgressDeadline: helper.TimeToPtr(3 * time.Minute), +// AutoRevert: helper.BoolToPtr(false), +// AutoPromote: nil, +// Canary: helper.IntToPtr(1), +// }, +// TaskGroups: []*api.TaskGroup{ +// { +// Update: &api.UpdateStrategy{ +// Canary: helper.IntToPtr(2), +// AutoRevert: helper.BoolToPtr(true), +// }, +// }, { +// Update: &api.UpdateStrategy{ +// Canary: helper.IntToPtr(3), +// AutoPromote: helper.BoolToPtr(true), +// }, +// }, +// }, +// } +// +// structsJob := ApiJobToStructJob(apiJob) +// +// // Update has been moved from job down to the groups +// jobUpdate := structs.UpdateStrategy{ +// Stagger: 1000000000, +// MaxParallel: 5, +// HealthCheck: "", +// MinHealthyTime: 0, +// HealthyDeadline: 0, +// ProgressDeadline: 0, +// AutoRevert: false, +// AutoPromote: false, +// Canary: 0, +// } +// +// // But the groups inherit settings from the job update +// group1 := structs.UpdateStrategy{ +// Stagger: 1000000000, +// MaxParallel: 5, +// HealthCheck: "manual", +// MinHealthyTime: 60000000000, +// HealthyDeadline: 180000000000, +// ProgressDeadline: 180000000000, +// AutoRevert: true, +// AutoPromote: false, +// Canary: 2, +// } +// +// group2 := structs.UpdateStrategy{ +// Stagger: 1000000000, +// MaxParallel: 5, +// HealthCheck: "manual", +// MinHealthyTime: 60000000000, +// HealthyDeadline: 180000000000, +// ProgressDeadline: 180000000000, +// AutoRevert: false, +// AutoPromote: true, +// Canary: 3, +// } +// +// require.Equal(t, jobUpdate, structsJob.Update) +// require.Equal(t, group1, *structsJob.TaskGroups[0].Update) +// require.Equal(t, group2, *structsJob.TaskGroups[1].Update) +// } +// +// // TestHTTP_JobValidate_SystemMigrate asserts that a system job with a migrate +// // stanza fails to validate but does not panic (see #5477). +// func TestHTTP_JobValidate_SystemMigrate(t *testing.T) { +// t.Parallel() +// httpTest(t, nil, func(s *TestAgent) { +// // Create the job +// job := &api.Job{ +// Region: helper.StringToPtr("global"), +// Datacenters: []string{"dc1"}, +// ID: helper.StringToPtr("systemmigrate"), +// Name: helper.StringToPtr("systemmigrate"), +// TaskGroups: []*api.TaskGroup{ +// {Name: helper.StringToPtr("web")}, +// }, +// +// // System job... +// Type: helper.StringToPtr("system"), +// +// // ...with an empty migrate stanza +// Migrate: &api.MigrateStrategy{}, +// } +// +// args := api.JobValidateRequest{ +// Job: job, +// WriteRequest: api.WriteRequest{Region: "global"}, +// } +// buf := encodeReq(args) +// +// // Make the HTTP request +// req, err := http.NewRequest("PUT", "/v1/validate/job", buf) +// require.NoError(t, err) +// respW := httptest.NewRecorder() +// +// // Make the request +// obj, err := s.Server.ValidateJobRequest(respW, req) +// require.NoError(t, err) +// +// // Check the response +// resp := obj.(structs.JobValidateResponse) +// require.Contains(t, resp.Error, `Job type "system" does not allow migrate block`) +// }) +// } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 844c311a8c7..a0b17323add 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -189,8 +189,6 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis return err } - // cgbaker: FINISH: validate the scaling policies (e.g., can't change policy ID) - // Ensure that the job has permissions for the requested Vault tokens policies := args.Job.VaultPolicies() if len(policies) != 0 { diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 943264ea6b4..fb5b24f3d1c 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -1256,6 +1256,7 @@ func ACLManagementToken() *structs.ACLToken { func ScalingPolicy() *structs.ScalingPolicy { return &structs.ScalingPolicy{ + ID: uuid.Generate(), Namespace: structs.DefaultNamespace, Target: uuid.Generate(), JobID: uuid.Generate(), @@ -1267,3 +1268,17 @@ func ScalingPolicy() *structs.ScalingPolicy { ModifyIndex: 20, } } + +func JobWithScalingPolicy() (*structs.Job, *structs.ScalingPolicy) { + job := Job() + policy := &structs.ScalingPolicy{ + ID: uuid.Generate(), + Namespace: job.Namespace, + JobID: job.ID, + Policy: map[string]interface{}{}, + Enabled: true, + } + policy.TargetTaskGroup(job, job.TaskGroups[0]) + job.TaskGroups[0].Scaling = policy + return job, policy +} diff --git a/nomad/scaling_endpoint.go b/nomad/scaling_endpoint.go new file mode 100644 index 00000000000..d9ed4849dfe --- /dev/null +++ b/nomad/scaling_endpoint.go @@ -0,0 +1,228 @@ +package nomad + +import ( + "time" + + metrics "github.com/armon/go-metrics" + log "github.com/hashicorp/go-hclog" + memdb "github.com/hashicorp/go-memdb" + + "github.com/hashicorp/nomad/nomad/state" + "github.com/hashicorp/nomad/nomad/structs" +) + +// Scaling endpoint is used for listing and retrieving scaling policies +type Scaling struct { + srv *Server + logger log.Logger +} + +// ListPolicies is used to list the policies +func (a *Scaling) ListPolicies(args *structs.ScalingPolicyListRequest, + reply *structs.ScalingPolicyListResponse) error { + + if done, err := a.srv.forward("Scaling.ListPolicies", args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "scaling", "list_policies"}, time.Now()) + + // Check management level permissions + // acl, err := a.srv.ResolveToken(args.AuthToken) + // if err != nil { + // return err + // } else if acl == nil { + // return structs.ErrPermissionDenied + // } + + // If it is not a management token determine the policies that may be listed + // mgt := acl.IsManagement() + // var policies map[string]struct{} + // if !mgt { + // token, err := a.requestACLToken(args.AuthToken) + // if err != nil { + // return err + // } + // if token == nil { + // return structs.ErrTokenNotFound + // } + // + // policies = make(map[string]struct{}, len(token.Policies)) + // for _, p := range token.Policies { + // policies[p] = struct{}{} + // } + // } + + // Setup the blocking query + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, state *state.StateStore) error { + // Iterate over all the policies + iter, err := state.ScalingPoliciesByNamespace(ws, args.Namespace) + if err != nil { + return err + } + + // Convert all the policies to a list stub + reply.Policies = nil + for { + raw := iter.Next() + if raw == nil { + break + } + policy := raw.(*structs.ScalingPolicy) + // if _, ok := policies[policy.Target]; ok || mgt { + // reply.Policies = append(reply.Policies, policy.Stub()) + // } + reply.Policies = append(reply.Policies, policy.Stub()) + } + + // Use the last index that affected the policy table + index, err := state.Index("scaling_policy") + if err != nil { + return err + } + + // Ensure we never set the index to zero, otherwise a blocking query cannot be used. + // We floor the index at one, since realistically the first write must have a higher index. + if index == 0 { + index = 1 + } + reply.Index = index + return nil + }} + return a.srv.blockingRPC(&opts) +} + +// GetPolicy is used to get a specific policy +func (a *Scaling) GetPolicy(args *structs.ScalingPolicySpecificRequest, + reply *structs.SingleScalingPolicyResponse) error { + + if done, err := a.srv.forward("Scaling.GetPolicy", args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "scaling", "get_policy"}, time.Now()) + + // Check management level permissions + // acl, err := a.srv.ResolveToken(args.AuthToken) + // if err != nil { + // return err + // } else if acl == nil { + // return structs.ErrPermissionDenied + // } + + // If it is not a management token determine the policies that may be listed + // mgt := acl.IsManagement() + // var policies map[string]struct{} + // if !mgt { + // token, err := a.requestACLToken(args.AuthToken) + // if err != nil { + // return err + // } + // if token == nil { + // return structs.ErrTokenNotFound + // } + // + // policies = make(map[string]struct{}, len(token.Policies)) + // for _, p := range token.Policies { + // policies[p] = struct{}{} + // } + // } + + // Setup the blocking query + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, state *state.StateStore) error { + // Iterate over all the policies + p, err := state.ScalingPolicyByID(ws, args.ID) + if err != nil { + return err + } + + reply.Policy = p + + // Use the last index that affected the policy table + index, err := state.Index("scaling_policy") + if err != nil { + return err + } + + // Ensure we never set the index to zero, otherwise a blocking query cannot be used. + // We floor the index at one, since realistically the first write must have a higher index. + if index == 0 { + index = 1 + } + reply.Index = index + return nil + }} + return a.srv.blockingRPC(&opts) +} + +// func (a *ACL) requestACLToken(secretID string) (*structs.ACLToken, error) { +// if secretID == "" { +// return structs.AnonymousACLToken, nil +// } +// +// snap, err := a.srv.fsm.State().Snapshot() +// if err != nil { +// return nil, err +// } +// +// return snap.ACLTokenBySecretID(nil, secretID) +// } + +// // GetPolicies is used to get a set of policies +// func (a *ACL) GetPolicies(args *structs.ACLPolicySetRequest, reply *structs.ACLPolicySetResponse) error { +// if !a.srv.config.ACLEnabled { +// return aclDisabled +// } +// if done, err := a.srv.forward("ACL.GetPolicies", args, args, reply); done { +// return err +// } +// defer metrics.MeasureSince([]string{"nomad", "acl", "get_policies"}, time.Now()) +// +// // For client typed tokens, allow them to query any policies associated with that token. +// // This is used by clients which are resolving the policies to enforce. Any associated +// // policies need to be fetched so that the client can determine what to allow. +// token, err := a.requestACLToken(args.AuthToken) +// if err != nil { +// return err +// } +// +// if token == nil { +// return structs.ErrTokenNotFound +// } +// if token.Type != structs.ACLManagementToken && !token.PolicySubset(args.Names) { +// return structs.ErrPermissionDenied +// } +// +// // Setup the blocking query +// opts := blockingOptions{ +// queryOpts: &args.QueryOptions, +// queryMeta: &reply.QueryMeta, +// run: func(ws memdb.WatchSet, state *state.StateStore) error { +// // Setup the output +// reply.Policies = make(map[string]*structs.ACLPolicy, len(args.Names)) +// +// // Look for the policy +// for _, policyName := range args.Names { +// out, err := state.ACLPolicyByName(ws, policyName) +// if err != nil { +// return err +// } +// if out != nil { +// reply.Policies[policyName] = out +// } +// } +// +// // Use the last index that affected the policy table +// index, err := state.Index("acl_policy") +// if err != nil { +// return err +// } +// reply.Index = index +// return nil +// }} +// return a.srv.blockingRPC(&opts) +// } diff --git a/nomad/scaling_endpoint_test.go b/nomad/scaling_endpoint_test.go new file mode 100644 index 00000000000..e62d780d397 --- /dev/null +++ b/nomad/scaling_endpoint_test.go @@ -0,0 +1,344 @@ +package nomad + +import ( + "testing" + "time" + + msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" +) + +// func TestACLEndpoint_GetPolicy(t *testing.T) { +// t.Parallel() +// +// s1, root, cleanupS1 := TestACLServer(t, nil) +// defer cleanupS1() +// codec := rpcClient(t, s1) +// testutil.WaitForLeader(t, s1.RPC) +// +// // Create the register request +// policy := mock.ACLPolicy() +// s1.fsm.State().UpsertACLPolicies(1000, []*structs.ACLPolicy{policy}) +// +// anonymousPolicy := mock.ACLPolicy() +// anonymousPolicy.Name = "anonymous" +// s1.fsm.State().UpsertACLPolicies(1001, []*structs.ACLPolicy{anonymousPolicy}) +// +// // Create a token with one the policy +// token := mock.ACLToken() +// token.Policies = []string{policy.Name} +// s1.fsm.State().UpsertACLTokens(1002, []*structs.ACLToken{token}) +// +// // Lookup the policy +// get := &structs.ACLPolicySpecificRequest{ +// Name: policy.Name, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// AuthToken: root.SecretID, +// }, +// } +// var resp structs.SingleACLPolicyResponse +// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// assert.Equal(t, uint64(1000), resp.Index) +// assert.Equal(t, policy, resp.Policy) +// +// // Lookup non-existing policy +// get.Name = uuid.Generate() +// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// assert.Equal(t, uint64(1001), resp.Index) +// assert.Nil(t, resp.Policy) +// +// // Lookup the policy with the token +// get = &structs.ACLPolicySpecificRequest{ +// Name: policy.Name, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// AuthToken: token.SecretID, +// }, +// } +// var resp2 structs.SingleACLPolicyResponse +// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp2); err != nil { +// t.Fatalf("err: %v", err) +// } +// assert.EqualValues(t, 1000, resp2.Index) +// assert.Equal(t, policy, resp2.Policy) +// +// // Lookup the anonymous policy with no token +// get = &structs.ACLPolicySpecificRequest{ +// Name: anonymousPolicy.Name, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// }, +// } +// var resp3 structs.SingleACLPolicyResponse +// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp3); err != nil { +// require.NoError(t, err) +// } +// assert.EqualValues(t, 1001, resp3.Index) +// assert.Equal(t, anonymousPolicy, resp3.Policy) +// +// // Lookup non-anonoymous policy with no token +// get = &structs.ACLPolicySpecificRequest{ +// Name: policy.Name, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// }, +// } +// var resp4 structs.SingleACLPolicyResponse +// err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp4) +// require.Error(t, err) +// require.Contains(t, err.Error(), structs.ErrPermissionDenied.Error()) +// } + +// func TestACLEndpoint_GetPolicy_Blocking(t *testing.T) { +// t.Parallel() +// +// s1, root, cleanupS1 := TestACLServer(t, nil) +// defer cleanupS1() +// state := s1.fsm.State() +// codec := rpcClient(t, s1) +// testutil.WaitForLeader(t, s1.RPC) +// +// // Create the policies +// p1 := mock.ACLPolicy() +// p2 := mock.ACLPolicy() +// +// // First create an unrelated policy +// time.AfterFunc(100*time.Millisecond, func() { +// err := state.UpsertACLPolicies(100, []*structs.ACLPolicy{p1}) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// }) +// +// // Upsert the policy we are watching later +// time.AfterFunc(200*time.Millisecond, func() { +// err := state.UpsertACLPolicies(200, []*structs.ACLPolicy{p2}) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// }) +// +// // Lookup the policy +// req := &structs.ACLPolicySpecificRequest{ +// Name: p2.Name, +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// MinQueryIndex: 150, +// AuthToken: root.SecretID, +// }, +// } +// var resp structs.SingleACLPolicyResponse +// start := time.Now() +// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", req, &resp); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// if elapsed := time.Since(start); elapsed < 200*time.Millisecond { +// t.Fatalf("should block (returned in %s) %#v", elapsed, resp) +// } +// if resp.Index != 200 { +// t.Fatalf("Bad index: %d %d", resp.Index, 200) +// } +// if resp.Policy == nil || resp.Policy.Name != p2.Name { +// t.Fatalf("bad: %#v", resp.Policy) +// } +// +// // Eval delete triggers watches +// time.AfterFunc(100*time.Millisecond, func() { +// err := state.DeleteACLPolicies(300, []string{p2.Name}) +// if err != nil { +// t.Fatalf("err: %v", err) +// } +// }) +// +// req.QueryOptions.MinQueryIndex = 250 +// var resp2 structs.SingleACLPolicyResponse +// start = time.Now() +// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", req, &resp2); err != nil { +// t.Fatalf("err: %v", err) +// } +// +// if elapsed := time.Since(start); elapsed < 100*time.Millisecond { +// t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) +// } +// if resp2.Index != 300 { +// t.Fatalf("Bad index: %d %d", resp2.Index, 300) +// } +// if resp2.Policy != nil { +// t.Fatalf("bad: %#v", resp2.Policy) +// } +// } + +func TestScalingEndpoint_GetPolicy(t *testing.T) { + t.Parallel() + + require := require.New(t) + + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the register request + p1 := mock.ScalingPolicy() + p2 := mock.ScalingPolicy() + s1.fsm.State().UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) + + // Lookup the policy + get := &structs.ScalingPolicySpecificRequest{ + ID: p1.ID, + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + var resp structs.SingleScalingPolicyResponse + err := msgpackrpc.CallWithCodec(codec, "Scaling.GetPolicy", get, &resp) + require.NoError(err) + require.Equal(uint64(1000), resp.Index) + require.Equal(*p1, *resp.Policy) + + // Lookup non-existing policy + get.ID = uuid.Generate() + resp = structs.SingleScalingPolicyResponse{} + err = msgpackrpc.CallWithCodec(codec, "Scaling.GetPolicy", get, &resp) + require.NoError(err) + require.Equal(uint64(1000), resp.Index) + require.Nil(resp.Policy) +} + +func TestScalingEndpoint_ListPolicies_Blocking(t *testing.T) { + t.Parallel() + + require := require.New(t) + + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + state := s1.fsm.State() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the policies + p1 := mock.ScalingPolicy() + p2 := mock.ScalingPolicy() + + // First create an unrelated policy + time.AfterFunc(100*time.Millisecond, func() { + err := state.UpsertScalingPolicies(100, []*structs.ScalingPolicy{p1}) + require.NoError(err) + }) + + // Upsert the policy we are watching later + time.AfterFunc(200*time.Millisecond, func() { + err := state.UpsertScalingPolicies(200, []*structs.ScalingPolicy{p2}) + require.NoError(err) + }) + + // Lookup the policy + req := &structs.ScalingPolicyListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + MinQueryIndex: 150, + }, + } + var resp structs.ScalingPolicyListResponse + start := time.Now() + err := msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", req, &resp) + require.NoError(err) + + require.True(time.Since(start) > 200*time.Millisecond, "should block: %#v", resp) + require.Equal(uint64(200), resp.Index, "bad index") + require.Len(resp.Policies, 2) + require.ElementsMatch([]string{p1.ID, p2.ID}, []string{resp.Policies[0].ID, resp.Policies[1].ID}) +} + +func TestScalingEndpoint_ListPolicies(t *testing.T) { + assert := assert.New(t) + t.Parallel() + + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the register request + p1 := mock.ScalingPolicy() + p2 := mock.ScalingPolicy() + + s1.fsm.State().UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) + + // Lookup the policies + get := &structs.ScalingPolicyListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + var resp structs.ACLPolicyListResponse + if err := msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp); err != nil { + t.Fatalf("err: %v", err) + } + assert.EqualValues(1000, resp.Index) + assert.Len(resp.Policies, 2) +} + +// TestACLEndpoint_ListPolicies_Unauthenticated asserts that +// unauthenticated ListPolicies returns anonymous policy if one +// exists, otherwise, empty +// func TestACLEndpoint_ListPolicies_Unauthenticated(t *testing.T) { +// t.Parallel() +// +// s1, _, cleanupS1 := TestACLServer(t, nil) +// defer cleanupS1() +// codec := rpcClient(t, s1) +// testutil.WaitForLeader(t, s1.RPC) +// +// listPolicies := func() (*structs.ACLPolicyListResponse, error) { +// // Lookup the policies +// get := &structs.ACLPolicyListRequest{ +// QueryOptions: structs.QueryOptions{ +// Region: "global", +// }, +// } +// +// var resp structs.ACLPolicyListResponse +// err := msgpackrpc.CallWithCodec(codec, "ACL.ListPolicies", get, &resp) +// if err != nil { +// return nil, err +// } +// return &resp, nil +// } +// +// p1 := mock.ACLPolicy() +// p1.Name = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9" +// s1.fsm.State().UpsertACLPolicies(1000, []*structs.ACLPolicy{p1}) +// +// t.Run("no anonymous policy", func(t *testing.T) { +// resp, err := listPolicies() +// require.NoError(t, err) +// require.Empty(t, resp.Policies) +// require.Equal(t, uint64(1000), resp.Index) +// }) +// +// // now try with anonymous policy +// p2 := mock.ACLPolicy() +// p2.Name = "anonymous" +// s1.fsm.State().UpsertACLPolicies(1001, []*structs.ACLPolicy{p2}) +// +// t.Run("with anonymous policy", func(t *testing.T) { +// resp, err := listPolicies() +// require.NoError(t, err) +// require.Len(t, resp.Policies, 1) +// require.Equal(t, "anonymous", resp.Policies[0].Name) +// require.Equal(t, uint64(1001), resp.Index) +// }) +// } diff --git a/nomad/server.go b/nomad/server.go index 921d9901bd4..efb2988cd1a 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -258,6 +258,7 @@ type endpoints struct { System *System Operator *Operator ACL *ACL + Scaling *Scaling Enterprise *EnterpriseEndpoints // Client endpoints @@ -1102,6 +1103,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { s.staticEndpoints.Periodic = &Periodic{srv: s, logger: s.logger.Named("periodic")} s.staticEndpoints.Plan = &Plan{srv: s, logger: s.logger.Named("plan")} s.staticEndpoints.Region = &Region{srv: s, logger: s.logger.Named("region")} + s.staticEndpoints.Scaling = &Scaling{srv: s, logger: s.logger.Named("scaling")} s.staticEndpoints.Status = &Status{srv: s, logger: s.logger.Named("status")} s.staticEndpoints.System = &System{srv: s, logger: s.logger.Named("system")} s.staticEndpoints.Search = &Search{srv: s, logger: s.logger.Named("search")} @@ -1133,6 +1135,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { server.Register(s.staticEndpoints.Periodic) server.Register(s.staticEndpoints.Plan) server.Register(s.staticEndpoints.Region) + server.Register(s.staticEndpoints.Scaling) server.Register(s.staticEndpoints.Status) server.Register(s.staticEndpoints.System) server.Register(s.staticEndpoints.Search) diff --git a/nomad/state/schema.go b/nomad/state/schema.go index 078022f96a5..449fcd6f666 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -736,14 +736,25 @@ func scalingPolicyTableSchema() *memdb.TableSchema { return &memdb.TableSchema{ Name: "scaling_policy", Indexes: map[string]*memdb.IndexSchema{ - // Primary index is used for job management - // and simple direct lookup. Target is required to be - // unique within a namespace. + // Primary index is used for simple direct lookup. "id": { Name: "id", AllowMissing: false, Unique: true, + // Use a compound index so the tuple of (Namespace, Target) is + // uniquely identifying + Indexer: &memdb.StringFieldIndex{ + Field: "ID", + }, + }, + // Target index is used for looking up by target or listing policies in namespace + // A target can only have a single scaling policy, so this is guaranteed to be unique. + "target": { + Name: "target", + AllowMissing: false, + Unique: true, + // Use a compound index so the tuple of (Namespace, Target) is // uniquely identifying Indexer: &memdb.CompoundIndex{ diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 12667e38e3a..0e0316dc511 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -11,6 +11,7 @@ import ( memdb "github.com/hashicorp/go-memdb" multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" "github.com/pkg/errors" ) @@ -4016,10 +4017,10 @@ func (s *StateStore) updateJobScalingPolicies(index uint64, job *structs.Job, tx } oldPolicy := raw.(*structs.ScalingPolicy) if _, ok := newTargets[oldPolicy.Target]; !ok { - deletedPolicies = append(deletedPolicies, oldPolicy.Target) + deletedPolicies = append(deletedPolicies, oldPolicy.ID) } } - err = s.DeleteScalingPoliciesTxn(index, job.Namespace, deletedPolicies, txn) + err = s.DeleteScalingPoliciesTxn(index, deletedPolicies, txn) if err != nil { return fmt.Errorf("DeleteScalingPolicies of removed policies failed: %v", err) } @@ -4721,7 +4722,7 @@ func (s *StateStore) UpsertScalingPoliciesTxn(index uint64, scalingPolicies []*s for _, scalingPolicy := range scalingPolicies { // Check if the scaling policy already exists - existing, err := txn.First("scaling_policy", "id", scalingPolicy.Namespace, scalingPolicy.Target) + existing, err := txn.First("scaling_policy", "target", scalingPolicy.Namespace, scalingPolicy.Target) if err != nil { return fmt.Errorf("scaling policy lookup failed: %v", err) } @@ -4730,9 +4731,13 @@ func (s *StateStore) UpsertScalingPoliciesTxn(index uint64, scalingPolicies []*s if existing != nil { scalingPolicy.CreateIndex = existing.(*structs.ScalingPolicy).CreateIndex scalingPolicy.ModifyIndex = index + scalingPolicy.ID = existing.(*structs.ScalingPolicy).ID } else { scalingPolicy.CreateIndex = index scalingPolicy.ModifyIndex = index + if scalingPolicy.ID == "" { + scalingPolicy.ID = uuid.Generate() + } } // Insert the scaling policy @@ -4749,11 +4754,11 @@ func (s *StateStore) UpsertScalingPoliciesTxn(index uint64, scalingPolicies []*s return nil } -func (s *StateStore) DeleteScalingPolicies(index uint64, namespace string, targets []string) error { +func (s *StateStore) DeleteScalingPolicies(index uint64, ids []string) error { txn := s.db.Txn(true) defer txn.Abort() - err := s.DeleteScalingPoliciesTxn(index, namespace, targets, txn) + err := s.DeleteScalingPoliciesTxn(index, ids, txn) if err == nil { txn.Commit() } @@ -4762,14 +4767,14 @@ func (s *StateStore) DeleteScalingPolicies(index uint64, namespace string, targe } // DeleteScalingPolicies is used to delete a set of scaling policies by ID -func (s *StateStore) DeleteScalingPoliciesTxn(index uint64, namespace string, targets []string, txn *memdb.Txn) error { - if len(targets) == 0 { +func (s *StateStore) DeleteScalingPoliciesTxn(index uint64, ids []string, txn *memdb.Txn) error { + if len(ids) == 0 { return nil } - for _, tgt := range targets { + for _, id := range ids { // Lookup the scaling policy - existing, err := txn.First("scaling_policy", "id", namespace, tgt) + existing, err := txn.First("scaling_policy", "id", id) if err != nil { return fmt.Errorf("scaling policy lookup failed: %v", err) } @@ -4793,7 +4798,7 @@ func (s *StateStore) DeleteScalingPoliciesTxn(index uint64, namespace string, ta func (s *StateStore) ScalingPoliciesByNamespace(ws memdb.WatchSet, namespace string) (memdb.ResultIterator, error) { txn := s.db.Txn(false) - iter, err := txn.Get("scaling_policy", "id_prefix", namespace) + iter, err := txn.Get("scaling_policy", "target_prefix", namespace) if err != nil { return nil, err } @@ -4819,10 +4824,26 @@ func (s *StateStore) ScalingPoliciesByJobTxn(ws memdb.WatchSet, namespace, jobID return iter, nil } +func (s *StateStore) ScalingPolicyByID(ws memdb.WatchSet, id string) (*structs.ScalingPolicy, error) { + txn := s.db.Txn(false) + + watchCh, existing, err := txn.FirstWatch("scaling_policy", "id", id) + if err != nil { + return nil, fmt.Errorf("scaling_policy lookup failed: %v", err) + } + ws.Add(watchCh) + + if existing != nil { + return existing.(*structs.ScalingPolicy), nil + } + + return nil, nil +} + func (s *StateStore) ScalingPolicyByTarget(ws memdb.WatchSet, namespace, target string) (*structs.ScalingPolicy, error) { txn := s.db.Txn(false) - watchCh, existing, err := txn.FirstWatch("scaling_policy", "id", namespace, target) + watchCh, existing, err := txn.FirstWatch("scaling_policy", "target", namespace, target) if err != nil { return nil, fmt.Errorf("scaling_policy lookup failed: %v", err) } diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index fb64f0f840c..991e2e94077 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -7973,16 +7973,7 @@ func TestStateStore_UpsertJob_UpsertScalingPolicies(t *testing.T) { require := require.New(t) state := testStateStore(t) - job := mock.Job() - // create policy and register against task group - policy := &structs.ScalingPolicy{ - Namespace: job.Namespace, - JobID: job.ID, - Policy: map[string]interface{}{}, - Enabled: true, - } - policy.TargetTaskGroup(job, job.TaskGroups[0]) - job.TaskGroups[0].Scaling = policy + job, policy := mock.JobWithScalingPolicy() // Create a watchset so we can test that upsert fires the watch ws := memdb.NewWatchSet() @@ -8025,7 +8016,7 @@ func TestStateStore_DeleteScalingPolicies(t *testing.T) { require.NoError(err) // Delete the policy - err = state.DeleteScalingPolicies(1001, policy.Namespace, []string{policy.Target, policy2.Target}) + err = state.DeleteScalingPolicies(1001, []string{policy.ID, policy2.ID}) require.NoError(err) // Ensure watching triggered diff --git a/nomad/structs/funcs.go b/nomad/structs/funcs.go index 889a7ce4dd7..11c48de0404 100644 --- a/nomad/structs/funcs.go +++ b/nomad/structs/funcs.go @@ -258,11 +258,14 @@ func CopyScalingPolicy(p *ScalingPolicy) *ScalingPolicy { } c := ScalingPolicy{ - Namespace: p.Namespace, - Target: p.Target, - JobID: p.JobID, - Policy: p.Policy, - Enabled: p.Enabled, + ID: p.ID, + Namespace: p.Namespace, + Target: p.Target, + JobID: p.JobID, + Policy: p.Policy, + Enabled: p.Enabled, + CreateIndex: p.CreateIndex, + ModifyIndex: p.ModifyIndex, } return &c } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 3f9e2d32c67..d03e9c403da 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -25,8 +25,12 @@ import ( "github.com/gorhill/cronexpr" hcodec "github.com/hashicorp/go-msgpack/codec" - multierror "github.com/hashicorp/go-multierror" - version "github.com/hashicorp/go-version" + "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-version" + "github.com/mitchellh/copystructure" + "github.com/ugorji/go/codec" + "golang.org/x/crypto/blake2b" + "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/command/agent/pprof" "github.com/hashicorp/nomad/helper" @@ -35,9 +39,6 @@ import ( "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/lib/kheap" psstructs "github.com/hashicorp/nomad/plugins/shared/structs" - "github.com/mitchellh/copystructure" - "github.com/ugorji/go/codec" - "golang.org/x/crypto/blake2b" ) var ( @@ -1045,6 +1046,29 @@ type DeploymentFailRequest struct { WriteRequest } +// ScalingPolicySpecificRequest is used when we just need to specify a target scaling policy +type ScalingPolicySpecificRequest struct { + ID string + QueryOptions +} + +// SingleScalingPolicyResponse is used to return a single job +type SingleScalingPolicyResponse struct { + Policy *ScalingPolicy + QueryMeta +} + +// ScalingPolicyListRequest is used to parameterize a scaling policy list request +type ScalingPolicyListRequest struct { + QueryOptions +} + +// ScalingPolicyListResponse is used for a list request +type ScalingPolicyListResponse struct { + Policies []*ScalingPolicyListStub + QueryMeta +} + // SingleDeploymentResponse is used to respond with a single deployment type SingleDeploymentResponse struct { Deployment *Deployment @@ -4580,6 +4604,9 @@ const ( // ScalingPolicy specifies the scaling policy for a scaling target type ScalingPolicy struct { + // ID is a generated UUID used for looking up the scaling policy + ID string + // Namespace is the namespace for the containing job Namespace string @@ -4600,10 +4627,20 @@ type ScalingPolicy struct { } func (p *ScalingPolicy) TargetTaskGroup(job *Job, tg *TaskGroup) *ScalingPolicy { - p.Target = fmt.Sprintf("%s/%s", job.ID, tg.Name) + p.Target = fmt.Sprintf("/v1/job/%s/%s/scale", job.ID, tg.Name) return p } +func (p *ScalingPolicy) Stub() *ScalingPolicyListStub { + return &ScalingPolicyListStub{ + ID: p.ID, + JobID: p.JobID, + Target: p.Target, + CreateIndex: p.CreateIndex, + ModifyIndex: p.ModifyIndex, + } +} + // GetScalingPolicies returns a slice of all scaling scaling policies for this job func (j *Job) GetScalingPolicies() []*ScalingPolicy { ret := make([]*ScalingPolicy, 0) @@ -4617,6 +4654,16 @@ func (j *Job) GetScalingPolicies() []*ScalingPolicy { return ret } +// ScalingPolicyListStub is used to return a subset of scaling policy information +// for the scaling policy list +type ScalingPolicyListStub struct { + ID string + JobID string + Target string + CreateIndex uint64 + ModifyIndex uint64 +} + // RestartPolicy configures how Tasks are restarted when they crash or fail. type RestartPolicy struct { // Attempts is the number of restart that will occur in an interval. From 810284968303df2d9535e16b7e5f098dd96462f7 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Thu, 16 Jan 2020 22:14:00 +0000 Subject: [PATCH 06/30] wip: working on job group scaling endpoint --- api/scaling.go | 22 +++++++++++ api/tasks.go | 12 ------ command/agent/job_endpoint.go | 46 +++++++++++++++++++++++ command/agent/job_endpoint_test.go | 60 ++++++++++++++++++++++++++++++ nomad/structs/structs.go | 12 ++++++ 5 files changed, 140 insertions(+), 12 deletions(-) create mode 100644 api/scaling.go diff --git a/api/scaling.go b/api/scaling.go new file mode 100644 index 00000000000..efa042ec7a5 --- /dev/null +++ b/api/scaling.go @@ -0,0 +1,22 @@ +package api + +// ScalingPolicy is the user-specified API object for an autoscaling policy +type ScalingPolicy struct { + Policy map[string]interface{} + Enabled *bool +} + +func (p *ScalingPolicy) Canonicalize() { + if p.Enabled == nil { + p.Enabled = boolToPtr(true) + } +} + +// ScalingRequeset is the payload for a generic scaling action +type ScalingRequest struct { + JobID string + Value interface{} + Reason string + WriteRequest + PolicyOverride bool +} diff --git a/api/tasks.go b/api/tasks.go index d4856ee2eda..f7ee42a78b9 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -409,18 +409,6 @@ func (vm *VolumeMount) Canonicalize() { } } -// ScalingPolicy is the user-specified API object for an autoscaling policy -type ScalingPolicy struct { - Policy map[string]interface{} - Enabled *bool -} - -func (p *ScalingPolicy) Canonicalize() { - if p.Enabled == nil { - p.Enabled = boolToPtr(true) - } -} - // TaskGroup is the unit of scheduling. type TaskGroup struct { Name *string diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 83a020bc939..4db5f5b9377 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -3,6 +3,7 @@ package agent import ( "fmt" "net/http" + "regexp" "strconv" "strings" @@ -82,6 +83,8 @@ func (s *HTTPServer) JobSpecificRequest(resp http.ResponseWriter, req *http.Requ case strings.HasSuffix(path, "/stable"): jobName := strings.TrimSuffix(path, "/stable") return s.jobStable(resp, req, jobName) + case strings.HasSuffix(path, "/scale"): + return s.jobScale(resp, req, path) default: return s.jobCRUD(resp, req, path) } @@ -454,6 +457,49 @@ func (s *HTTPServer) jobDelete(resp http.ResponseWriter, req *http.Request, return out, nil } +func (s *HTTPServer) jobScale(resp http.ResponseWriter, req *http.Request, + jobAndTarget string) (interface{}, error) { + if req.Method != "PUT" && req.Method != "POST" { + return nil, CodedError(405, ErrInvalidMethod) + } + + var args api.ScalingRequest + if err := decodeBody(req, &args); err != nil { + return nil, CodedError(400, err.Error()) + } + + if args.JobID == "" { + return nil, CodedError(400, "Job ID must be specified") + } + if !strings.HasPrefix(jobAndTarget, args.JobID) { + return nil, CodedError(400, "Job ID does not match") + } + subTarget := strings.TrimPrefix(jobAndTarget, args.JobID) + groupScale := regexp.MustCompile(`/[^/]+/scale`) + groupName := groupScale.FindString(subTarget) + if groupName == "" { + return nil, CodedError(400, "Invalid scaling target") + } + + scaleReq := structs.JobScaleRequest{ + JobID: args.JobID, + GroupName: groupName, + Value: args.Value, + PolicyOverride: args.PolicyOverride, + Reason: args.Reason, + } + // parseWriteRequest overrides Namespace, Region and AuthToken + // based on values from the original http request + s.parseWriteRequest(req, &scaleReq.WriteRequest) + + var out structs.JobRegisterResponse + if err := s.agent.RPC("Job.Scale", &scaleReq, &out); err != nil { + return nil, err + } + setIndex(resp, out.Index) + return out, nil +} + func (s *HTTPServer) jobVersions(resp http.ResponseWriter, req *http.Request, jobName string) (interface{}, error) { diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 807a5837517..eebb1fb97ef 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -666,6 +666,66 @@ func TestHTTP_JobDelete(t *testing.T) { }) } +func TestHTTP_Job_GroupScale(t *testing.T) { + t.Parallel() + + require := require.New(t) + + httpTest(t, nil, func(s *TestAgent) { + // Create the job + job, policy := mock.JobWithScalingPolicy() + args := structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: structs.DefaultNamespace, + }, + } + var resp structs.JobRegisterResponse + if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { + t.Fatalf("err: %v", err) + } + + newCount := job.TaskGroups[0].Count + 1 + scaleReq := &api.ScalingRequest{ + JobID: job.ID, + Value: newCount, + Reason: "testing", + } + buf := encodeReq(scaleReq) + + // Make the HTTP request to scale the job group + req, err := http.NewRequest("POST", policy.Target, buf) + require.NoError(err) + respW := httptest.NewRecorder() + + // Make the request + obj, err := s.Server.JobSpecificRequest(respW, req) + require.NoError(err) + + // Check the response + resp = obj.(structs.JobRegisterResponse) + require.NotEmpty(resp.EvalID) + + // Check for the index + require.NotEmpty(respW.Header().Get("X-Nomad-Index")) + + // Check that the group count was changed + getReq := structs.JobSpecificRequest{ + JobID: job.ID, + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: structs.DefaultNamespace, + }, + } + var getResp structs.SingleJobResponse + err = s.Agent.RPC("Job.GetJob", &getReq, &getResp) + require.NoError(err) + require.NotNil(getResp.Job) + require.Equal(newCount, getResp.Job.TaskGroups[0].Count) + }) +} + func TestHTTP_JobForceEvaluate(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index d03e9c403da..f772bff0cf5 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -618,6 +618,18 @@ type JobPlanRequest struct { WriteRequest } +// JobScaleRequest is used for the Job.Scale endpoint to scale one of the +// scaling targets in a job +type JobScaleRequest struct { + JobID string + GroupName string + Value interface{} + Reason string + // PolicyOverride is set when the user is attempting to override any policies + PolicyOverride bool + WriteRequest +} + // JobSummaryRequest is used when we just need to get a specific job summary type JobSummaryRequest struct { JobID string From 1c9bac9087b8869cf4092fad02d97f6adda58dd9 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Fri, 17 Jan 2020 16:51:35 +0000 Subject: [PATCH 07/30] wip: added job.scale rpc endpoint, needs explicit test (tested via http now) --- acl/policy.go | 1 + command/agent/job_endpoint.go | 7 +- command/agent/job_endpoint_test.go | 2 + nomad/job_endpoint.go | 107 +++++++++++++++++++++++++++++ nomad/structs/structs.go | 1 + scheduler/generic_sched.go | 3 +- scheduler/system_sched.go | 2 +- 7 files changed, 119 insertions(+), 4 deletions(-) diff --git a/acl/policy.go b/acl/policy.go index b4925577e3a..cce0d9c47ee 100644 --- a/acl/policy.go +++ b/acl/policy.go @@ -26,6 +26,7 @@ const ( NamespaceCapabilityDeny = "deny" NamespaceCapabilityListJobs = "list-jobs" NamespaceCapabilityReadJob = "read-job" + NamespaceCapabilityScaleJob = "scale-job" NamespaceCapabilitySubmitJob = "submit-job" NamespaceCapabilityDispatchJob = "dispatch-job" NamespaceCapabilityReadLogs = "read-logs" diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 4db5f5b9377..66e117ea2a0 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -475,8 +475,11 @@ func (s *HTTPServer) jobScale(resp http.ResponseWriter, req *http.Request, return nil, CodedError(400, "Job ID does not match") } subTarget := strings.TrimPrefix(jobAndTarget, args.JobID) - groupScale := regexp.MustCompile(`/[^/]+/scale`) - groupName := groupScale.FindString(subTarget) + groupScale := regexp.MustCompile(`^/([^/]+)/scale$`) + var groupName string + if subMatch := groupScale.FindStringSubmatch(subTarget); subMatch != nil { + groupName = subMatch[1] + } if groupName == "" { return nil, CodedError(400, "Invalid scaling target") } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index eebb1fb97ef..53f7efc7141 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -686,6 +686,8 @@ func TestHTTP_Job_GroupScale(t *testing.T) { t.Fatalf("err: %v", err) } + // FINISH: cgbaker: do something with args.reason + newCount := job.TaskGroups[0].Count + 1 scaleReq := &api.ScalingRequest{ JobID: job.ID, diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index a0b17323add..fd8962ae25d 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -809,6 +809,113 @@ func (j *Job) BatchDeregister(args *structs.JobBatchDeregisterRequest, reply *st return nil } +// Scale is used to modify one of the scaling targest in the job +func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterResponse) error { + if done, err := j.srv.forward("Job.Scale", args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "job", "scale"}, time.Now()) + + // Check for submit-job permissions + if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityScaleJob) { + return structs.ErrPermissionDenied + } + + // Validate the arguments + if args.JobID == "" { + return fmt.Errorf("missing job ID for scaling") + } else if args.GroupName == "" { + return fmt.Errorf("missing task group name for scaling") + } else if args.Value == nil { + return fmt.Errorf("missing new scaling value") + } + newCount, ok := args.Value.(float64) + if !ok { + return fmt.Errorf("scaling value for task group must be int: %t %v", args.Value, args.Value) + } + + // Lookup the job + snap, err := j.srv.fsm.State().Snapshot() + if err != nil { + return err + } + ws := memdb.NewWatchSet() + job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) + if err != nil { + return err + } + + found := false + for _, tg := range job.TaskGroups { + if args.GroupName == tg.Name { + tg.Count = int(newCount) + found = true + break + } + } + if !found { + return fmt.Errorf("task group %q specified for scaling does not exist in job", args.GroupName) + } + registerReq := structs.JobRegisterRequest{ + Job: job, + EnforceIndex: true, + JobModifyIndex: job.ModifyIndex, + PolicyOverride: args.PolicyOverride, + WriteRequest: args.WriteRequest, + } + + // Commit this update via Raft + _, index, err := j.srv.raftApply(structs.JobRegisterRequestType, registerReq) + if err != nil { + j.logger.Error("job register for scale failed", "error", err) + return err + } + + // Populate the reply with job information + reply.JobModifyIndex = index + + // If the job is periodic or parameterized, we don't create an eval. + if job != nil && (job.IsPeriodic() || job.IsParameterized()) { + return nil + } + + // Create a new evaluation + // XXX: The job priority / type is strange for this, since it's not a high + // priority even if the job was. + now := time.Now().UTC().UnixNano() + eval := &structs.Evaluation{ + ID: uuid.Generate(), + Namespace: args.RequestNamespace(), + Priority: structs.JobDefaultPriority, + Type: structs.JobTypeService, + TriggeredBy: structs.EvalTriggerScaling, + JobID: args.JobID, + JobModifyIndex: index, + Status: structs.EvalStatusPending, + CreateTime: now, + ModifyTime: now, + } + update := &structs.EvalUpdateRequest{ + Evals: []*structs.Evaluation{eval}, + WriteRequest: structs.WriteRequest{Region: args.Region}, + } + + // Commit this evaluation via Raft + _, evalIndex, err := j.srv.raftApply(structs.EvalUpdateRequestType, update) + if err != nil { + j.logger.Error("eval create failed", "error", err, "method", "deregister") + return err + } + + // Populate the reply with eval information + reply.EvalID = eval.ID + reply.EvalCreateIndex = evalIndex + reply.Index = evalIndex + return nil +} + // GetJob is used to request information about a specific job func (j *Job) GetJob(args *structs.JobSpecificRequest, reply *structs.SingleJobResponse) error { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index f772bff0cf5..68c19efd96b 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -8775,6 +8775,7 @@ const ( EvalTriggerRetryFailedAlloc = "alloc-failure" EvalTriggerQueuedAllocs = "queued-allocs" EvalTriggerPreemption = "preemption" + EvalTriggerScaling = "job-scaling" ) const ( diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index 95cdb8daef0..f47e02500ff 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -134,7 +134,8 @@ func (s *GenericScheduler) Process(eval *structs.Evaluation) error { structs.EvalTriggerRollingUpdate, structs.EvalTriggerQueuedAllocs, structs.EvalTriggerPeriodicJob, structs.EvalTriggerMaxPlans, structs.EvalTriggerDeploymentWatcher, structs.EvalTriggerRetryFailedAlloc, - structs.EvalTriggerFailedFollowUp, structs.EvalTriggerPreemption: + structs.EvalTriggerFailedFollowUp, structs.EvalTriggerPreemption, + structs.EvalTriggerScaling: default: desc := fmt.Sprintf("scheduler cannot handle '%s' evaluation reason", eval.TriggeredBy) diff --git a/scheduler/system_sched.go b/scheduler/system_sched.go index 542bde45139..2940c9b5af2 100644 --- a/scheduler/system_sched.go +++ b/scheduler/system_sched.go @@ -63,7 +63,7 @@ func (s *SystemScheduler) Process(eval *structs.Evaluation) error { case structs.EvalTriggerJobRegister, structs.EvalTriggerNodeUpdate, structs.EvalTriggerFailedFollowUp, structs.EvalTriggerJobDeregister, structs.EvalTriggerRollingUpdate, structs.EvalTriggerPreemption, structs.EvalTriggerDeploymentWatcher, structs.EvalTriggerNodeDrain, structs.EvalTriggerAllocStop, - structs.EvalTriggerQueuedAllocs: + structs.EvalTriggerQueuedAllocs, structs.EvalTriggerScaling: default: desc := fmt.Sprintf("scheduler cannot handle '%s' evaluation reason", eval.TriggeredBy) From ef7cb0e098090775b9f450e047d29d4eaeca3aec Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Fri, 24 Jan 2020 20:15:49 -0500 Subject: [PATCH 08/30] wip: add job scale endpoint in client --- api/jobs.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/api/jobs.go b/api/jobs.go index c5b85a3164d..00d2431d3b7 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -153,6 +153,22 @@ func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) { return &resp, qm, nil } +// Scale is used to retrieve information about a particular +// job given its unique ID. +func (j *Jobs) Scale(jobID, group string, value interface{}, reason string, q *WriteOptions) (*JobRegisterResponse, *WriteMeta, error) { + req := &ScalingRequest{ + JobID: jobID, + Value: value, + Reason: reason, + } + var resp JobRegisterResponse + qm, err := j.client.write(fmt.Sprintf("/v1/job/%s/%s/scale", url.PathEscape(jobID), url.PathEscape(group)), req, &resp, q) + if err != nil { + return nil, nil, err + } + return &resp, qm, nil +} + // Versions is used to retrieve all versions of a particular job given its // unique ID. func (j *Jobs) Versions(jobID string, diffs bool, q *QueryOptions) ([]*Job, []*JobDiff, *QueryMeta, error) { From 4406668b539dbe66057286f20a81d37262e3bbfd Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sun, 26 Jan 2020 16:13:56 +0000 Subject: [PATCH 09/30] wip: add GET endpoint for job group scaling target --- api/scaling.go | 7 ++++ command/agent/job_endpoint.go | 56 ++++++++++++++++++++++++++++++ command/agent/job_endpoint_test.go | 39 +++++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/api/scaling.go b/api/scaling.go index efa042ec7a5..edb0aa91f24 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -20,3 +20,10 @@ type ScalingRequest struct { WriteRequest PolicyOverride bool } + +// ScaleStatusResponse is the payload for a generic scaling action +type ScaleStatusResponse struct { + JobID string + Value interface{} + JobModifyIndex uint64 +} diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 66e117ea2a0..d08ff872880 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -459,6 +459,62 @@ func (s *HTTPServer) jobDelete(resp http.ResponseWriter, req *http.Request, func (s *HTTPServer) jobScale(resp http.ResponseWriter, req *http.Request, jobAndTarget string) (interface{}, error) { + switch req.Method { + case "GET": + return s.jobScaleStatus(resp, req, jobAndTarget) + case "PUT", "POST": + return s.jobScaleAction(resp, req, jobAndTarget) + default: + return nil, CodedError(405, ErrInvalidMethod) + } +} + +func (s *HTTPServer) jobScaleStatus(resp http.ResponseWriter, req *http.Request, + jobAndTarget string) (interface{}, error) { + + regJobGroup := regexp.MustCompile(`^(.+)/([^/]+)/scale$`) + var jobName, groupName string + if subMatch := regJobGroup.FindStringSubmatch(jobAndTarget); subMatch != nil { + jobName = subMatch[1] + groupName = subMatch[2] + } + if jobName == "" || groupName == "" { + return nil, CodedError(400, "Invalid scaling target") + } + + args := structs.JobSpecificRequest{ + JobID: jobName, + } + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + var out structs.SingleJobResponse + if err := s.agent.RPC("Job.GetJob", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + if out.Job == nil { + return nil, CodedError(404, "job not found") + } + + group := out.Job.LookupTaskGroup(groupName) + if group == nil { + return nil, CodedError(404, "group not found in job") + } + status := &api.ScaleStatusResponse{ + JobID: out.Job.ID, + Value: group.Count, + JobModifyIndex: out.Job.ModifyIndex, + } + + return status, nil +} + +func (s *HTTPServer) jobScaleAction(resp http.ResponseWriter, req *http.Request, + jobAndTarget string) (interface{}, error) { + if req.Method != "PUT" && req.Method != "POST" { return nil, CodedError(405, ErrInvalidMethod) } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 53f7efc7141..6cf3b86d5f0 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -728,6 +728,45 @@ func TestHTTP_Job_GroupScale(t *testing.T) { }) } +func TestHTTP_Job_GroupScaleStatus(t *testing.T) { + t.Parallel() + + require := require.New(t) + + httpTest(t, nil, func(s *TestAgent) { + // Create the job + job, policy := mock.JobWithScalingPolicy() + args := structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: structs.DefaultNamespace, + }, + } + var resp structs.JobRegisterResponse + if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { + t.Fatalf("err: %v", err) + } + + // Make the HTTP request to scale the job group + req, err := http.NewRequest("GET", policy.Target, nil) + require.NoError(err) + respW := httptest.NewRecorder() + + // Make the request + obj, err := s.Server.JobSpecificRequest(respW, req) + require.NoError(err) + + // Check the response + status := obj.(*api.ScaleStatusResponse) + require.NotEmpty(resp.EvalID) + require.Equal(job.TaskGroups[0].Count, status.Value.(int)) + + // Check for the index + require.NotEmpty(respW.Header().Get("X-Nomad-Index")) + }) +} + func TestHTTP_JobForceEvaluate(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { From 94381c0da89e5e34ed0c75e3d518d93ea9e2a023 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Mon, 27 Jan 2020 22:14:28 +0000 Subject: [PATCH 10/30] wip: added tests for client methods around group scaling --- api/jobs.go | 12 +++++ api/jobs_test.go | 86 +++++++++++++++++++++++++++++++++++ api/util_test.go | 9 ++++ command/agent/job_endpoint.go | 42 +++++++---------- nomad/job_endpoint.go | 3 ++ 5 files changed, 127 insertions(+), 25 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index 00d2431d3b7..a205fba1021 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -169,6 +169,18 @@ func (j *Jobs) Scale(jobID, group string, value interface{}, reason string, q *W return &resp, qm, nil } +// ScaleStatus is used to retrieve information about a particular +// job given its unique ID. +func (j *Jobs) ScaleStatus(jobID, group string, q *QueryOptions) (*ScaleStatusResponse, *QueryMeta, error) { + var resp ScaleStatusResponse + qm, err := j.client.query(fmt.Sprintf("/v1/job/%s/%s/scale", url.PathEscape(jobID), url.PathEscape(group)), + &resp, q) + if err != nil { + return nil, nil, err + } + return &resp, qm, nil +} + // Versions is used to retrieve all versions of a particular job given its // unique ID. func (j *Jobs) Versions(jobID string, diffs bool, q *QueryOptions) ([]*Job, []*JobDiff, *QueryMeta, error) { diff --git a/api/jobs_test.go b/api/jobs_test.go index 160a8a1216e..d9bcbec270f 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -1533,3 +1533,89 @@ func TestJobs_AddSpread(t *testing.T) { t.Fatalf("expect: %#v, got: %#v", expect, job.Spreads) } } + +// TestJobs_ScaleAction tests the scale target for task group count +func TestJobs_ScaleAction(t *testing.T) { + t.Parallel() + + require := require.New(t) + + c, s := makeClient(t, nil, nil) + defer s.Stop() + jobs := c.Jobs() + + id := "job-id/with\\troublesome:characters\n?&å­—\000" + job := testJobWithScalingPolicy() + job.ID = &id + groupName := *job.TaskGroups[0].Name + groupCount := *job.TaskGroups[0].Count + + // Trying to scale against a target before it exists returns an error + _, _, err := jobs.Scale(id, "missing", + groupCount+1, "this won't work", nil) + require.Error(err) + require.Contains(err.Error(), "not found") + + // Register the job + _, wm, err := jobs.Register(job, nil) + if err != nil { + t.Fatalf("err: %s", err) + } + assertWriteMeta(t, wm) + + // Perform a scaling action with bad group name, verify error + _, _, err = jobs.Scale(id, "incorrect-group-name", + groupCount+1, "this won't work", nil) + require.Error(err) + require.Contains(err.Error(), "does not exist") + + // Query the scaling endpoint and verify success + resp1, wm, err := jobs.Scale(id, groupName, + groupCount+1, "need more instances", nil) + + require.NoError(err) + require.NotNil(resp1) + require.NotEmpty(resp1.EvalID) + assertWriteMeta(t, wm) +} + +// TestJobs_ScaleStatus tests the /scale status endpoint for task group count +func TestJobs_ScaleStatus(t *testing.T) { + t.Parallel() + + require := require.New(t) + + c, s := makeClient(t, nil, nil) + defer s.Stop() + jobs := c.Jobs() + + // Trying to retrieve a status before it exists returns an error + id := "job-id/with\\troublesome:characters\n?&å­—\000" + _, _, err := jobs.ScaleStatus(id, "missing", nil) + require.Error(err) + require.Contains(err.Error(), "not found") + + // Register the job + job := testJobWithScalingPolicy() + job.ID = &id + groupName := *job.TaskGroups[0].Name + groupCount := *job.TaskGroups[0].Count + _, wm, err := jobs.Register(job, nil) + if err != nil { + t.Fatalf("err: %s", err) + } + assertWriteMeta(t, wm) + + // Query the scaling endpoint with bad group name, verify error + _, _, err = jobs.ScaleStatus(id, "incorrect-group-name", nil) + require.Error(err) + require.Contains(err.Error(), "not found") + + // Query the scaling endpoint and verify success + result, qm, err := jobs.ScaleStatus(id, groupName, nil) + require.NoError(err) + assertQueryMeta(t, qm) + + // Check that the result is what we expect + require.Equal(groupCount, int(result.Value.(float64))) +} diff --git a/api/util_test.go b/api/util_test.go index 2ebf502d7a9..f32d6d30d95 100644 --- a/api/util_test.go +++ b/api/util_test.go @@ -48,6 +48,15 @@ func testJob() *Job { return job } +func testJobWithScalingPolicy() *Job { + job := testJob() + job.TaskGroups[0].Scaling = &ScalingPolicy{ + Policy: map[string]interface{}{}, + Enabled: boolToPtr(true), + } + return job +} + func testPeriodicJob() *Job { job := testJob().AddPeriodicConfig(&PeriodicConfig{ Enabled: boolToPtr(true), diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index d08ff872880..8f39ee708c0 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -3,7 +3,6 @@ package agent import ( "fmt" "net/http" - "regexp" "strconv" "strings" @@ -459,28 +458,29 @@ func (s *HTTPServer) jobDelete(resp http.ResponseWriter, req *http.Request, func (s *HTTPServer) jobScale(resp http.ResponseWriter, req *http.Request, jobAndTarget string) (interface{}, error) { + + jobAndGroup := strings.TrimSuffix(jobAndTarget, "/scale") + var jobName, groupName string + if i := strings.LastIndex(jobAndGroup, "/"); i != -1 { + jobName = jobAndGroup[:i] + groupName = jobAndGroup[i+1:] + } + if jobName == "" || groupName == "" { + return nil, CodedError(400, "Invalid scaling target") + } + switch req.Method { case "GET": - return s.jobScaleStatus(resp, req, jobAndTarget) + return s.jobScaleStatus(resp, req, jobName, groupName) case "PUT", "POST": - return s.jobScaleAction(resp, req, jobAndTarget) + return s.jobScaleAction(resp, req, jobName, groupName) default: return nil, CodedError(405, ErrInvalidMethod) } } func (s *HTTPServer) jobScaleStatus(resp http.ResponseWriter, req *http.Request, - jobAndTarget string) (interface{}, error) { - - regJobGroup := regexp.MustCompile(`^(.+)/([^/]+)/scale$`) - var jobName, groupName string - if subMatch := regJobGroup.FindStringSubmatch(jobAndTarget); subMatch != nil { - jobName = subMatch[1] - groupName = subMatch[2] - } - if jobName == "" || groupName == "" { - return nil, CodedError(400, "Invalid scaling target") - } + jobName, groupName string) (interface{}, error) { args := structs.JobSpecificRequest{ JobID: jobName, @@ -513,7 +513,7 @@ func (s *HTTPServer) jobScaleStatus(resp http.ResponseWriter, req *http.Request, } func (s *HTTPServer) jobScaleAction(resp http.ResponseWriter, req *http.Request, - jobAndTarget string) (interface{}, error) { + jobName, groupName string) (interface{}, error) { if req.Method != "PUT" && req.Method != "POST" { return nil, CodedError(405, ErrInvalidMethod) @@ -527,18 +527,10 @@ func (s *HTTPServer) jobScaleAction(resp http.ResponseWriter, req *http.Request, if args.JobID == "" { return nil, CodedError(400, "Job ID must be specified") } - if !strings.HasPrefix(jobAndTarget, args.JobID) { + + if args.JobID != jobName { return nil, CodedError(400, "Job ID does not match") } - subTarget := strings.TrimPrefix(jobAndTarget, args.JobID) - groupScale := regexp.MustCompile(`^/([^/]+)/scale$`) - var groupName string - if subMatch := groupScale.FindStringSubmatch(subTarget); subMatch != nil { - groupName = subMatch[1] - } - if groupName == "" { - return nil, CodedError(400, "Invalid scaling target") - } scaleReq := structs.JobScaleRequest{ JobID: args.JobID, diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index fd8962ae25d..8a979b0960e 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -846,6 +846,9 @@ func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterRes if err != nil { return err } + if job == nil { + return fmt.Errorf("job %q not found", args.JobID) + } found := false for _, tg := range job.TaskGroups { From 16472c026a856e41d00a90679d85e27e49cdb2bd Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Mon, 3 Feb 2020 20:27:28 +0000 Subject: [PATCH 11/30] wip: remove PolicyOverride from scaling request --- api/scaling.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/scaling.go b/api/scaling.go index edb0aa91f24..a260171a218 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -17,13 +17,15 @@ type ScalingRequest struct { JobID string Value interface{} Reason string + Error string WriteRequest - PolicyOverride bool + // why is PolicyOverride in here? was this a mistake + // PolicyOverride bool } // ScaleStatusResponse is the payload for a generic scaling action type ScaleStatusResponse struct { JobID string - Value interface{} JobModifyIndex uint64 + Value interface{} } From 6b9c0043e417f3c878ba6bd39f40f985c3f715ac Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Wed, 5 Feb 2020 21:19:15 +0000 Subject: [PATCH 12/30] wip: added Enabled to ScalingPolicyListStub, removed JobID from body of scaling request --- api/jobs.go | 1 - api/scaling.go | 7 +++---- command/agent/job_endpoint.go | 10 +--------- command/agent/scaling_endpoint.go | 33 ------------------------------- nomad/structs/structs.go | 2 ++ 5 files changed, 6 insertions(+), 47 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index a205fba1021..05f1e007afd 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -157,7 +157,6 @@ func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) { // job given its unique ID. func (j *Jobs) Scale(jobID, group string, value interface{}, reason string, q *WriteOptions) (*JobRegisterResponse, *WriteMeta, error) { req := &ScalingRequest{ - JobID: jobID, Value: value, Reason: reason, } diff --git a/api/scaling.go b/api/scaling.go index a260171a218..c2c51fd46cb 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -12,15 +12,14 @@ func (p *ScalingPolicy) Canonicalize() { } } -// ScalingRequeset is the payload for a generic scaling action +// ScalingRequest is the payload for a generic scaling action type ScalingRequest struct { - JobID string Value interface{} Reason string Error string WriteRequest - // why is PolicyOverride in here? was this a mistake - // PolicyOverride bool + // this is effectively a job update, so we need the ability to override policy. + PolicyOverride bool } // ScaleStatusResponse is the payload for a generic scaling action diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 8f39ee708c0..ebee8bba226 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -524,16 +524,8 @@ func (s *HTTPServer) jobScaleAction(resp http.ResponseWriter, req *http.Request, return nil, CodedError(400, err.Error()) } - if args.JobID == "" { - return nil, CodedError(400, "Job ID must be specified") - } - - if args.JobID != jobName { - return nil, CodedError(400, "Job ID does not match") - } - scaleReq := structs.JobScaleRequest{ - JobID: args.JobID, + JobID: jobName, GroupName: groupName, Value: args.Value, PolicyOverride: args.PolicyOverride, diff --git a/command/agent/scaling_endpoint.go b/command/agent/scaling_endpoint.go index 54bf220c508..92493fc4198 100644 --- a/command/agent/scaling_endpoint.go +++ b/command/agent/scaling_endpoint.go @@ -43,39 +43,6 @@ func (s *HTTPServer) ScalingPolicySpecificRequest(resp http.ResponseWriter, req } } -// func (s *HTTPServer) ValidateJobRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { -// // Ensure request method is POST or PUT -// if !(req.Method == "POST" || req.Method == "PUT") { -// return nil, CodedError(405, ErrInvalidMethod) -// } -// -// var validateRequest api.JobValidateRequest -// if err := decodeBody(req, &validateRequest); err != nil { -// return nil, CodedError(400, err.Error()) -// } -// if validateRequest.Job == nil { -// return nil, CodedError(400, "Job must be specified") -// } -// -// job := ApiJobToStructJob(validateRequest.Job) -// -// args := structs.JobValidateRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: validateRequest.Region, -// }, -// } -// s.parseWriteRequest(req, &args.WriteRequest) -// args.Namespace = job.Namespace -// -// var out structs.JobValidateResponse -// if err := s.agent.RPC("Job.Validate", &args, &out); err != nil { -// return nil, err -// } -// -// return out, nil -// } - func (s *HTTPServer) scalingPolicyCRUD(resp http.ResponseWriter, req *http.Request, policyID string) (interface{}, error) { switch req.Method { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 68c19efd96b..8ec97290e8d 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -4648,6 +4648,7 @@ func (p *ScalingPolicy) Stub() *ScalingPolicyListStub { ID: p.ID, JobID: p.JobID, Target: p.Target, + Enabled: p.Enabled, CreateIndex: p.CreateIndex, ModifyIndex: p.ModifyIndex, } @@ -4672,6 +4673,7 @@ type ScalingPolicyListStub struct { ID string JobID string Target string + Enabled bool CreateIndex uint64 ModifyIndex uint64 } From 7544aaac65ff44e5c4b6418c9239f36515d5de6e Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Wed, 29 Jan 2020 19:26:31 -0500 Subject: [PATCH 13/30] wip: add scaling policies methods to the client --- api/scaling.go | 52 +++++++++++++++++++++-- api/scaling_test.go | 101 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 api/scaling_test.go diff --git a/api/scaling.go b/api/scaling.go index c2c51fd46cb..579802eaddb 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -1,9 +1,31 @@ package api -// ScalingPolicy is the user-specified API object for an autoscaling policy -type ScalingPolicy struct { - Policy map[string]interface{} - Enabled *bool +// Scaling is used to query scaling-related API endpoints +type Scaling struct { + client *Client +} + +// Scaling returns a handle on the scaling endpoints. +func (c *Client) Scaling() *Scaling { + return &Scaling{client: c} +} + +func (s *Scaling) ListPolicies(q *QueryOptions) ([]*ScalingPolicyListStub, *QueryMeta, error) { + var resp []*ScalingPolicyListStub + qm, err := s.client.query("/v1/scaling/policies", &resp, q) + if err != nil { + return nil, nil, err + } + return resp, qm, nil +} + +func (s *Scaling) GetPolicy(ID string, q *QueryOptions) (*ScalingPolicy, *QueryMeta, error) { + var policy ScalingPolicy + qm, err := s.client.query("/v1/scaling/policy/"+ID, &policy, q) + if err != nil { + return nil, nil, err + } + return &policy, qm, nil } func (p *ScalingPolicy) Canonicalize() { @@ -22,6 +44,28 @@ type ScalingRequest struct { PolicyOverride bool } +// ScalingPolicy is the user-specified API object for an autoscaling policy +type ScalingPolicy struct { + ID string + Namespace string + Target string + JobID string + Policy map[string]interface{} + Enabled *bool + CreateIndex uint64 + ModifyIndex uint64 +} + +// ScalingPolicyListStub is used to return a subset of scaling policy information +// for the scaling policy list +type ScalingPolicyListStub struct { + ID string + JobID string + Target string + CreateIndex uint64 + ModifyIndex uint64 +} + // ScaleStatusResponse is the payload for a generic scaling action type ScaleStatusResponse struct { JobID string diff --git a/api/scaling_test.go b/api/scaling_test.go new file mode 100644 index 00000000000..8b8e205267b --- /dev/null +++ b/api/scaling_test.go @@ -0,0 +1,101 @@ +package api + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestScalingPolicies_List(t *testing.T) { + t.Parallel() + require := require.New(t) + + c, s := makeClient(t, nil, nil) + defer s.Stop() + scaling := c.Scaling() + jobs := c.Jobs() + + // Check that we don't have any scaling policies before registering a job that has one + policies, _, err := scaling.ListPolicies(nil) + require.NoError(err) + require.Empty(policies, "expected 0 scaling policies, got: %d", len(policies)) + + // Register a job with a scaling policy + job := testJob() + job.TaskGroups[0].Scaling = &ScalingPolicy{} + _, _, err = jobs.Register(job, nil) + require.NoError(err) + + // Check that we have a scaling policy now + policies, _, err = scaling.ListPolicies(nil) + require.NoError(err) + if len(policies) != 1 { + t.Fatalf("expected 1 scaling policy, got: %d", len(policies)) + } + + policy := policies[0] + + // Check that the scaling policy references the right job + require.Equalf(policy.JobID, *job.ID, "expected JobID=%s, got: %s", *job.ID, policy.JobID) + + // Check that the scaling policy references the right target + expectedTarget := fmt.Sprintf("/v1/job/%s/%s/scale", *job.ID, *job.TaskGroups[0].Name) + require.Equalf(expectedTarget, policy.Target, "expected Target=%s, got: %s", expectedTarget, policy.Target) +} + +func TestScalingPolicies_GetPolicy(t *testing.T) { + t.Parallel() + require := require.New(t) + + c, s := makeClient(t, nil, nil) + defer s.Stop() + scaling := c.Scaling() + jobs := c.Jobs() + + // Empty ID should return 404 + _, _, err := scaling.GetPolicy("", nil) + require.Error(err) + require.Containsf(err.Error(), "404", "expected 404 error, got: %s", err.Error()) + + // Inexistent ID should return 404 + _, _, err = scaling.GetPolicy("i-dont-exist", nil) + require.Error(err) + require.Containsf(err.Error(), "404", "expected 404 error, got: %s", err.Error()) + + // Register a job with a scaling policy + job := testJob() + policy := &ScalingPolicy{ + Enabled: boolToPtr(true), + Policy: map[string]interface{}{ + "key": "value", + }, + } + job.TaskGroups[0].Scaling = policy + _, _, err = jobs.Register(job, nil) + require.NoError(err) + + // Find newly created scaling policy ID + var policyID string + policies, _, err := scaling.ListPolicies(nil) + require.NoError(err) + for _, p := range policies { + if p.JobID == *job.ID { + policyID = p.ID + break + } + } + if policyID == "" { + t.Fatalf("unable to find scaling policy for job %s", *job.ID) + } + + // Fetch scaling policy + resp, _, err := scaling.GetPolicy(policyID, nil) + require.NoError(err) + + // Check that the scaling policy fields match + expectedTarget := fmt.Sprintf("/v1/job/%s/%s/scale", *job.ID, *job.TaskGroups[0].Name) + require.Equalf(expectedTarget, resp.Target, "expected Target=%s, got: %s", expectedTarget, policy.Target) + require.Equal(policy.Policy, resp.Policy) + require.Equal(policy.Enabled, resp.Enabled) +} From 836acc11bd808cd405af7300489274f0697fc2dc Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Wed, 29 Jan 2020 19:27:14 -0500 Subject: [PATCH 14/30] wip: add tests for job scale method --- api/jobs_test.go | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/api/jobs_test.go b/api/jobs_test.go index d9bcbec270f..f7dc23808c4 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -909,6 +909,64 @@ func TestJobs_Info(t *testing.T) { } } +func TestJobs_Scale(t *testing.T) { + t.Parallel() + c, s := makeClient(t, nil, nil) + defer s.Stop() + jobs := c.Jobs() + + // Check if invalid inputs fail + tests := []struct { + jobID string + group string + value interface{} + want string + }{ + {"", "", 1, "404"}, + {"i-dont-exist", "", 1, "400"}, + {"", "i-dont-exist", 1, "400"}, + {"i-dont-exist", "me-neither", 1, "EOF"}, // TODO: this should be a 404 + {"id", "group", nil, "500"}, + {"id", "group", "not-int", "500"}, + } + for _, test := range tests { + _, _, err := jobs.Scale(test.jobID, test.group, test.value, "", nil) + if err == nil { + t.Errorf("expected jobs.Scale(%s, %s) to fail", test.jobID, test.group) + } + if !strings.Contains(err.Error(), test.want) { + t.Errorf("jobs.Scale(%s, %s) error doesn't contain %s, got: %s", test.jobID, test.group, test.want, err) + } + } + + // Register test job + job := testJob() + job.ID = stringToPtr("TestJobs_Scale") + _, wm, err := jobs.Register(job, nil) + if err != nil { + t.Fatalf("err: %s", err) + } + assertWriteMeta(t, wm) + + // Scale job task group + value := 2 + _, wm, err = jobs.Scale(*job.ID, *job.TaskGroups[0].Name, value, "reason", nil) + if err != nil { + t.Fatalf("err: %s", err) + } + assertWriteMeta(t, wm) + + // Query the job again + resp, _, err := jobs.Info(*job.ID, nil) + if err != nil { + t.Fatalf("err: %s", err) + } + if *resp.TaskGroups[0].Count != value { + t.Errorf("expected task group count to be %d, got: %d", value, *resp.TaskGroups[0].Count) + } + // TODO: check if reason is stored +} + func TestJobs_Versions(t *testing.T) { t.Parallel() c, s := makeClient(t, nil, nil) From 9756c64a615a3d50c467ad698855e7185b359e7d Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Tue, 4 Feb 2020 12:33:55 -0500 Subject: [PATCH 15/30] wip: use testify in job scaling tests --- api/jobs_test.go | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/api/jobs_test.go b/api/jobs_test.go index f7dc23808c4..662a9f1a366 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -911,6 +911,8 @@ func TestJobs_Info(t *testing.T) { func TestJobs_Scale(t *testing.T) { t.Parallel() + require := require.New(t) + c, s := makeClient(t, nil, nil) defer s.Stop() jobs := c.Jobs() @@ -931,39 +933,28 @@ func TestJobs_Scale(t *testing.T) { } for _, test := range tests { _, _, err := jobs.Scale(test.jobID, test.group, test.value, "", nil) - if err == nil { - t.Errorf("expected jobs.Scale(%s, %s) to fail", test.jobID, test.group) - } - if !strings.Contains(err.Error(), test.want) { - t.Errorf("jobs.Scale(%s, %s) error doesn't contain %s, got: %s", test.jobID, test.group, test.want, err) - } + require.Errorf(err, "expected jobs.Scale(%s, %s) to fail", test.jobID, test.group) + require.Containsf(err.Error(), test.want, "jobs.Scale(%s, %s) error doesn't contain %s, got: %s", test.jobID, test.group, test.want, err) } // Register test job job := testJob() job.ID = stringToPtr("TestJobs_Scale") _, wm, err := jobs.Register(job, nil) - if err != nil { - t.Fatalf("err: %s", err) - } + require.NoError(err) assertWriteMeta(t, wm) // Scale job task group value := 2 _, wm, err = jobs.Scale(*job.ID, *job.TaskGroups[0].Name, value, "reason", nil) - if err != nil { - t.Fatalf("err: %s", err) - } + require.NoError(err) assertWriteMeta(t, wm) // Query the job again resp, _, err := jobs.Info(*job.ID, nil) - if err != nil { - t.Fatalf("err: %s", err) - } - if *resp.TaskGroups[0].Count != value { - t.Errorf("expected task group count to be %d, got: %d", value, *resp.TaskGroups[0].Count) - } + require.NoError(err) + require.Equal(*resp.TaskGroups[0].Count, value) + // TODO: check if reason is stored } From c095f4111126cacd7d0ed3e18fb8ba3faaf1ddfe Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Wed, 5 Feb 2020 21:47:26 +0000 Subject: [PATCH 16/30] wip: removed some commented junk from scaling poc --- nomad/scaling_endpoint.go | 68 ---------- nomad/scaling_endpoint_test.go | 218 --------------------------------- 2 files changed, 286 deletions(-) diff --git a/nomad/scaling_endpoint.go b/nomad/scaling_endpoint.go index d9ed4849dfe..44be4989588 100644 --- a/nomad/scaling_endpoint.go +++ b/nomad/scaling_endpoint.go @@ -158,71 +158,3 @@ func (a *Scaling) GetPolicy(args *structs.ScalingPolicySpecificRequest, }} return a.srv.blockingRPC(&opts) } - -// func (a *ACL) requestACLToken(secretID string) (*structs.ACLToken, error) { -// if secretID == "" { -// return structs.AnonymousACLToken, nil -// } -// -// snap, err := a.srv.fsm.State().Snapshot() -// if err != nil { -// return nil, err -// } -// -// return snap.ACLTokenBySecretID(nil, secretID) -// } - -// // GetPolicies is used to get a set of policies -// func (a *ACL) GetPolicies(args *structs.ACLPolicySetRequest, reply *structs.ACLPolicySetResponse) error { -// if !a.srv.config.ACLEnabled { -// return aclDisabled -// } -// if done, err := a.srv.forward("ACL.GetPolicies", args, args, reply); done { -// return err -// } -// defer metrics.MeasureSince([]string{"nomad", "acl", "get_policies"}, time.Now()) -// -// // For client typed tokens, allow them to query any policies associated with that token. -// // This is used by clients which are resolving the policies to enforce. Any associated -// // policies need to be fetched so that the client can determine what to allow. -// token, err := a.requestACLToken(args.AuthToken) -// if err != nil { -// return err -// } -// -// if token == nil { -// return structs.ErrTokenNotFound -// } -// if token.Type != structs.ACLManagementToken && !token.PolicySubset(args.Names) { -// return structs.ErrPermissionDenied -// } -// -// // Setup the blocking query -// opts := blockingOptions{ -// queryOpts: &args.QueryOptions, -// queryMeta: &reply.QueryMeta, -// run: func(ws memdb.WatchSet, state *state.StateStore) error { -// // Setup the output -// reply.Policies = make(map[string]*structs.ACLPolicy, len(args.Names)) -// -// // Look for the policy -// for _, policyName := range args.Names { -// out, err := state.ACLPolicyByName(ws, policyName) -// if err != nil { -// return err -// } -// if out != nil { -// reply.Policies[policyName] = out -// } -// } -// -// // Use the last index that affected the policy table -// index, err := state.Index("acl_policy") -// if err != nil { -// return err -// } -// reply.Index = index -// return nil -// }} -// return a.srv.blockingRPC(&opts) -// } diff --git a/nomad/scaling_endpoint_test.go b/nomad/scaling_endpoint_test.go index e62d780d397..c6d06e22b6e 100644 --- a/nomad/scaling_endpoint_test.go +++ b/nomad/scaling_endpoint_test.go @@ -14,172 +14,6 @@ import ( "github.com/hashicorp/nomad/testutil" ) -// func TestACLEndpoint_GetPolicy(t *testing.T) { -// t.Parallel() -// -// s1, root, cleanupS1 := TestACLServer(t, nil) -// defer cleanupS1() -// codec := rpcClient(t, s1) -// testutil.WaitForLeader(t, s1.RPC) -// -// // Create the register request -// policy := mock.ACLPolicy() -// s1.fsm.State().UpsertACLPolicies(1000, []*structs.ACLPolicy{policy}) -// -// anonymousPolicy := mock.ACLPolicy() -// anonymousPolicy.Name = "anonymous" -// s1.fsm.State().UpsertACLPolicies(1001, []*structs.ACLPolicy{anonymousPolicy}) -// -// // Create a token with one the policy -// token := mock.ACLToken() -// token.Policies = []string{policy.Name} -// s1.fsm.State().UpsertACLTokens(1002, []*structs.ACLToken{token}) -// -// // Lookup the policy -// get := &structs.ACLPolicySpecificRequest{ -// Name: policy.Name, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// AuthToken: root.SecretID, -// }, -// } -// var resp structs.SingleACLPolicyResponse -// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// assert.Equal(t, uint64(1000), resp.Index) -// assert.Equal(t, policy, resp.Policy) -// -// // Lookup non-existing policy -// get.Name = uuid.Generate() -// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// assert.Equal(t, uint64(1001), resp.Index) -// assert.Nil(t, resp.Policy) -// -// // Lookup the policy with the token -// get = &structs.ACLPolicySpecificRequest{ -// Name: policy.Name, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// AuthToken: token.SecretID, -// }, -// } -// var resp2 structs.SingleACLPolicyResponse -// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp2); err != nil { -// t.Fatalf("err: %v", err) -// } -// assert.EqualValues(t, 1000, resp2.Index) -// assert.Equal(t, policy, resp2.Policy) -// -// // Lookup the anonymous policy with no token -// get = &structs.ACLPolicySpecificRequest{ -// Name: anonymousPolicy.Name, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// }, -// } -// var resp3 structs.SingleACLPolicyResponse -// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp3); err != nil { -// require.NoError(t, err) -// } -// assert.EqualValues(t, 1001, resp3.Index) -// assert.Equal(t, anonymousPolicy, resp3.Policy) -// -// // Lookup non-anonoymous policy with no token -// get = &structs.ACLPolicySpecificRequest{ -// Name: policy.Name, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// }, -// } -// var resp4 structs.SingleACLPolicyResponse -// err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp4) -// require.Error(t, err) -// require.Contains(t, err.Error(), structs.ErrPermissionDenied.Error()) -// } - -// func TestACLEndpoint_GetPolicy_Blocking(t *testing.T) { -// t.Parallel() -// -// s1, root, cleanupS1 := TestACLServer(t, nil) -// defer cleanupS1() -// state := s1.fsm.State() -// codec := rpcClient(t, s1) -// testutil.WaitForLeader(t, s1.RPC) -// -// // Create the policies -// p1 := mock.ACLPolicy() -// p2 := mock.ACLPolicy() -// -// // First create an unrelated policy -// time.AfterFunc(100*time.Millisecond, func() { -// err := state.UpsertACLPolicies(100, []*structs.ACLPolicy{p1}) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// }) -// -// // Upsert the policy we are watching later -// time.AfterFunc(200*time.Millisecond, func() { -// err := state.UpsertACLPolicies(200, []*structs.ACLPolicy{p2}) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// }) -// -// // Lookup the policy -// req := &structs.ACLPolicySpecificRequest{ -// Name: p2.Name, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// MinQueryIndex: 150, -// AuthToken: root.SecretID, -// }, -// } -// var resp structs.SingleACLPolicyResponse -// start := time.Now() -// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", req, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// if elapsed := time.Since(start); elapsed < 200*time.Millisecond { -// t.Fatalf("should block (returned in %s) %#v", elapsed, resp) -// } -// if resp.Index != 200 { -// t.Fatalf("Bad index: %d %d", resp.Index, 200) -// } -// if resp.Policy == nil || resp.Policy.Name != p2.Name { -// t.Fatalf("bad: %#v", resp.Policy) -// } -// -// // Eval delete triggers watches -// time.AfterFunc(100*time.Millisecond, func() { -// err := state.DeleteACLPolicies(300, []string{p2.Name}) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// }) -// -// req.QueryOptions.MinQueryIndex = 250 -// var resp2 structs.SingleACLPolicyResponse -// start = time.Now() -// if err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", req, &resp2); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// if elapsed := time.Since(start); elapsed < 100*time.Millisecond { -// t.Fatalf("should block (returned in %s) %#v", elapsed, resp2) -// } -// if resp2.Index != 300 { -// t.Fatalf("Bad index: %d %d", resp2.Index, 300) -// } -// if resp2.Policy != nil { -// t.Fatalf("bad: %#v", resp2.Policy) -// } -// } - func TestScalingEndpoint_GetPolicy(t *testing.T) { t.Parallel() @@ -290,55 +124,3 @@ func TestScalingEndpoint_ListPolicies(t *testing.T) { assert.EqualValues(1000, resp.Index) assert.Len(resp.Policies, 2) } - -// TestACLEndpoint_ListPolicies_Unauthenticated asserts that -// unauthenticated ListPolicies returns anonymous policy if one -// exists, otherwise, empty -// func TestACLEndpoint_ListPolicies_Unauthenticated(t *testing.T) { -// t.Parallel() -// -// s1, _, cleanupS1 := TestACLServer(t, nil) -// defer cleanupS1() -// codec := rpcClient(t, s1) -// testutil.WaitForLeader(t, s1.RPC) -// -// listPolicies := func() (*structs.ACLPolicyListResponse, error) { -// // Lookup the policies -// get := &structs.ACLPolicyListRequest{ -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// }, -// } -// -// var resp structs.ACLPolicyListResponse -// err := msgpackrpc.CallWithCodec(codec, "ACL.ListPolicies", get, &resp) -// if err != nil { -// return nil, err -// } -// return &resp, nil -// } -// -// p1 := mock.ACLPolicy() -// p1.Name = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9" -// s1.fsm.State().UpsertACLPolicies(1000, []*structs.ACLPolicy{p1}) -// -// t.Run("no anonymous policy", func(t *testing.T) { -// resp, err := listPolicies() -// require.NoError(t, err) -// require.Empty(t, resp.Policies) -// require.Equal(t, uint64(1000), resp.Index) -// }) -// -// // now try with anonymous policy -// p2 := mock.ACLPolicy() -// p2.Name = "anonymous" -// s1.fsm.State().UpsertACLPolicies(1001, []*structs.ACLPolicy{p2}) -// -// t.Run("with anonymous policy", func(t *testing.T) { -// resp, err := listPolicies() -// require.NoError(t, err) -// require.Len(t, resp.Policies, 1) -// require.Equal(t, "anonymous", resp.Policies[0].Name) -// require.Equal(t, uint64(1001), resp.Index) -// }) -// } From 3b4a1aecd94f5a2929e01fae07c6bfffb50cb968 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Fri, 21 Feb 2020 21:23:30 +0000 Subject: [PATCH 17/30] finished refactoring state store, schema, etc --- api/scaling.go | 10 +- api/scaling_test.go | 31 ++++-- command/agent/job_endpoint.go | 2 +- command/agent/scaling_endpoint.go | 11 +- nomad/job_endpoint.go | 33 +++++- nomad/mock/mock.go | 18 ++-- nomad/state/schema.go | 93 +++++++++++----- nomad/state/state_store.go | 67 +++++++----- nomad/state/state_store_test.go | 171 ++++++++++++++++++++++++++---- nomad/structs/funcs.go | 15 ++- nomad/structs/structs.go | 49 ++++++--- 11 files changed, 381 insertions(+), 119 deletions(-) diff --git a/api/scaling.go b/api/scaling.go index 579802eaddb..5107fdecf30 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -39,6 +39,7 @@ type ScalingRequest struct { Value interface{} Reason string Error string + Meta map[string]interface{} WriteRequest // this is effectively a job update, so we need the ability to override policy. PolicyOverride bool @@ -48,8 +49,9 @@ type ScalingRequest struct { type ScalingPolicy struct { ID string Namespace string - Target string - JobID string + Target map[string]string + Min int64 + Max int64 Policy map[string]interface{} Enabled *bool CreateIndex uint64 @@ -60,8 +62,8 @@ type ScalingPolicy struct { // for the scaling policy list type ScalingPolicyListStub struct { ID string - JobID string - Target string + Enabled bool + Target map[string]string CreateIndex uint64 ModifyIndex uint64 } diff --git a/api/scaling_test.go b/api/scaling_test.go index 8b8e205267b..6b79624141e 100644 --- a/api/scaling_test.go +++ b/api/scaling_test.go @@ -1,13 +1,12 @@ package api import ( - "fmt" "testing" "github.com/stretchr/testify/require" ) -func TestScalingPolicies_List(t *testing.T) { +func TestScalingPolicies_ListPolicies(t *testing.T) { t.Parallel() require := require.New(t) @@ -36,12 +35,18 @@ func TestScalingPolicies_List(t *testing.T) { policy := policies[0] + // Check that the scaling policy references the right namespace + namespace := DefaultNamespace + if job.Namespace != nil && *job.Namespace != "" { + namespace = *job.Namespace + } + require.Equal(policy.Target["Namespace"], namespace) + // Check that the scaling policy references the right job - require.Equalf(policy.JobID, *job.ID, "expected JobID=%s, got: %s", *job.ID, policy.JobID) + require.Equal(policy.Target["Job"], *job.ID) - // Check that the scaling policy references the right target - expectedTarget := fmt.Sprintf("/v1/job/%s/%s/scale", *job.ID, *job.TaskGroups[0].Name) - require.Equalf(expectedTarget, policy.Target, "expected Target=%s, got: %s", expectedTarget, policy.Target) + // Check that the scaling policy references the right group + require.Equal(policy.Target["Group"], *job.TaskGroups[0].Name) } func TestScalingPolicies_GetPolicy(t *testing.T) { @@ -80,7 +85,7 @@ func TestScalingPolicies_GetPolicy(t *testing.T) { policies, _, err := scaling.ListPolicies(nil) require.NoError(err) for _, p := range policies { - if p.JobID == *job.ID { + if p.Target["Job"] == *job.ID { policyID = p.ID break } @@ -94,8 +99,16 @@ func TestScalingPolicies_GetPolicy(t *testing.T) { require.NoError(err) // Check that the scaling policy fields match - expectedTarget := fmt.Sprintf("/v1/job/%s/%s/scale", *job.ID, *job.TaskGroups[0].Name) - require.Equalf(expectedTarget, resp.Target, "expected Target=%s, got: %s", expectedTarget, policy.Target) + namespace := DefaultNamespace + if job.Namespace != nil && *job.Namespace != "" { + namespace = *job.Namespace + } + expectedTarget := map[string]string{ + "Namespace": namespace, + "Job": *job.ID, + "Group": *job.TaskGroups[0].Name, + } + require.Equal(expectedTarget, resp.Target) require.Equal(policy.Policy, resp.Policy) require.Equal(policy.Enabled, resp.Enabled) } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index ebee8bba226..b8d250ddc5d 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -824,7 +824,7 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta } if taskGroup.Scaling != nil { - tg.Scaling = ApiScalingPolicyToStructs(job, taskGroup.Scaling).TargetTaskGroup(job, tg) + tg.Scaling = ApiScalingPolicyToStructs(taskGroup.Scaling).TargetTaskGroup(job, tg) } tg.EphemeralDisk = &structs.EphemeralDisk{ diff --git a/command/agent/scaling_endpoint.go b/command/agent/scaling_endpoint.go index 92493fc4198..04a80670060 100644 --- a/command/agent/scaling_endpoint.go +++ b/command/agent/scaling_endpoint.go @@ -75,11 +75,12 @@ func (s *HTTPServer) scalingPolicyQuery(resp http.ResponseWriter, req *http.Requ return out.Policy, nil } -func ApiScalingPolicyToStructs(job *structs.Job, a1 *api.ScalingPolicy) *structs.ScalingPolicy { +func ApiScalingPolicyToStructs(ap *api.ScalingPolicy) *structs.ScalingPolicy { return &structs.ScalingPolicy{ - Namespace: job.Namespace, - JobID: job.ID, - Enabled: *a1.Enabled, - Policy: a1.Policy, + Enabled: *ap.Enabled, + Min: ap.Min, + Max: ap.Max, + Policy: ap.Policy, + Target: map[string]string{}, } } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 8a979b0960e..185ead74f14 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -14,13 +14,14 @@ import ( "github.com/golang/snappy" "github.com/hashicorp/consul/lib" + "github.com/pkg/errors" + "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/scheduler" - "github.com/pkg/errors" ) const ( @@ -189,6 +190,11 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis return err } + // Ensure that all scaling policies have an appropriate ID + if err := propagateScalingPolicyIDs(existingJob, args.Job); err != nil { + return err + } + // Ensure that the job has permissions for the requested Vault tokens policies := args.Job.VaultPolicies() if len(policies) != 0 { @@ -332,6 +338,31 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis return nil } +// propagateScalingPolicyIDs propagates scaling policy IDs from existing job +// to updated job, or generates random IDs in new job +func propagateScalingPolicyIDs(old, new *structs.Job) error { + + oldIDs := make(map[string]string) + if old != nil { + // jobs currently only have scaling policies on task groups, so we can + // find correspondences using task group names + for _, p := range old.GetScalingPolicies() { + oldIDs[p.Target[structs.ScalingTargetGroup]] = p.ID + } + } + + // ignore any existing ID in the policy, they should be empty + for _, p := range new.GetScalingPolicies() { + if id, ok := oldIDs[p.Target[structs.ScalingTargetGroup]]; ok { + p.ID = id + } else { + p.ID = uuid.Generate() + } + } + + return nil +} + // getSignalConstraint builds a suitable constraint based on the required // signals func getSignalConstraint(signals []string) *structs.Constraint { diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index fb5b24f3d1c..a52838ff99a 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -1256,10 +1256,12 @@ func ACLManagementToken() *structs.ACLToken { func ScalingPolicy() *structs.ScalingPolicy { return &structs.ScalingPolicy{ - ID: uuid.Generate(), - Namespace: structs.DefaultNamespace, - Target: uuid.Generate(), - JobID: uuid.Generate(), + ID: uuid.Generate(), + Target: map[string]string{ + structs.ScalingTargetNamespace: structs.DefaultNamespace, + structs.ScalingTargetJob: uuid.Generate(), + structs.ScalingTargetGroup: uuid.Generate(), + }, Policy: map[string]interface{}{ "a": "b", }, @@ -1272,11 +1274,9 @@ func ScalingPolicy() *structs.ScalingPolicy { func JobWithScalingPolicy() (*structs.Job, *structs.ScalingPolicy) { job := Job() policy := &structs.ScalingPolicy{ - ID: uuid.Generate(), - Namespace: job.Namespace, - JobID: job.ID, - Policy: map[string]interface{}{}, - Enabled: true, + ID: uuid.Generate(), + Policy: map[string]interface{}{}, + Enabled: true, } policy.TargetTaskGroup(job, job.TaskGroups[0]) job.TaskGroups[0].Scaling = policy diff --git a/nomad/state/schema.go b/nomad/state/schema.go index 449fcd6f666..a87738c0344 100644 --- a/nomad/state/schema.go +++ b/nomad/state/schema.go @@ -5,6 +5,7 @@ import ( "sync" memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/nomad/structs" ) @@ -730,6 +731,63 @@ func csiPluginTableSchema() *memdb.TableSchema { } } +// StringFieldIndex is used to extract a field from an object +// using reflection and builds an index on that field. +type ScalingPolicyTargetFieldIndex struct { + Field string +} + +// FromObject is used to extract an index value from an +// object or to indicate that the index value is missing. +func (s *ScalingPolicyTargetFieldIndex) FromObject(obj interface{}) (bool, []byte, error) { + policy, ok := obj.(*structs.ScalingPolicy) + if !ok { + return false, nil, fmt.Errorf("object %#v is not a ScalingPolicy", obj) + } + + if policy.Target == nil { + return false, nil, nil + } + + val, ok := policy.Target[s.Field] + if !ok { + return false, nil, nil + } + + // Add the null character as a terminator + val += "\x00" + return true, []byte(val), nil +} + +// FromArgs is used to build an exact index lookup based on arguments +func (s *ScalingPolicyTargetFieldIndex) FromArgs(args ...interface{}) ([]byte, error) { + if len(args) != 1 { + return nil, fmt.Errorf("must provide only a single argument") + } + arg, ok := args[0].(string) + if !ok { + return nil, fmt.Errorf("argument must be a string: %#v", args[0]) + } + // Add the null character as a terminator + arg += "\x00" + return []byte(arg), nil +} + +// PrefixFromArgs returns a prefix that should be used for scanning based on the arguments +func (s *ScalingPolicyTargetFieldIndex) PrefixFromArgs(args ...interface{}) ([]byte, error) { + val, err := s.FromArgs(args...) + if err != nil { + return nil, err + } + + // Strip the null terminator, the rest is a prefix + n := len(val) + if n > 0 { + return val[:n-1], nil + } + return val, nil +} + // scalingPolicyTableSchema returns the MemDB schema for the policy table. // This table is used to store the policies which are referenced by tokens func scalingPolicyTableSchema() *memdb.TableSchema { @@ -742,49 +800,32 @@ func scalingPolicyTableSchema() *memdb.TableSchema { AllowMissing: false, Unique: true, - // Use a compound index so the tuple of (Namespace, Target) is - // uniquely identifying + // UUID is uniquely identifying Indexer: &memdb.StringFieldIndex{ Field: "ID", }, }, - // Target index is used for looking up by target or listing policies in namespace - // A target can only have a single scaling policy, so this is guaranteed to be unique. + // Target index is used for listing by namespace or job, or looking up a specific target. + // A given task group can have only a single scaling policies, so this is guaranteed to be unique. "target": { Name: "target", AllowMissing: false, Unique: true, - // Use a compound index so the tuple of (Namespace, Target) is + // Use a compound index so the tuple of (Namespace, Job, Group) is // uniquely identifying Indexer: &memdb.CompoundIndex{ Indexes: []memdb.Indexer{ - &memdb.StringFieldIndex{ + &ScalingPolicyTargetFieldIndex{ Field: "Namespace", }, - &memdb.StringFieldIndex{ - Field: "Target", + &ScalingPolicyTargetFieldIndex{ + Field: "Job", }, - }, - }, - }, - // Job index is used to lookup scaling policies by job - "job": { - Name: "job", - AllowMissing: false, - Unique: false, - // Use a compound index so the tuple of (Namespace, JobID) is - // uniquely identifying - Indexer: &memdb.CompoundIndex{ - Indexes: []memdb.Indexer{ - &memdb.StringFieldIndex{ - Field: "Namespace", - }, - - &memdb.StringFieldIndex{ - Field: "JobID", + &ScalingPolicyTargetFieldIndex{ + Field: "Group", }, }, }, diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index 0e0316dc511..26d5dde6d4d 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -10,10 +10,10 @@ import ( log "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" multierror "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/hashicorp/nomad/helper" - "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/structs" - "github.com/pkg/errors" ) // Txn is a transaction against a state store. @@ -1284,12 +1284,15 @@ func (s *StateStore) DeleteJobTxn(index uint64, namespace, jobID string, txn Txn return fmt.Errorf("index update failed: %v", err) } - // Delete the job scaling policies - if _, err := txn.DeleteAll("scaling_policy", "job", namespace, jobID); err != nil { + // Delete any job scaling policies + numDeletedScalingPolicies, err := txn.DeleteAll("scaling_policy", "target_prefix", namespace, jobID) + if err != nil { return fmt.Errorf("deleting job scaling policies failed: %v", err) } - if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { - return fmt.Errorf("index update failed: %v", err) + if numDeletedScalingPolicies > 0 { + if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } } return nil @@ -4002,7 +4005,7 @@ func (s *StateStore) updateJobScalingPolicies(index uint64, job *structs.Job, tx scalingPolicies := job.GetScalingPolicies() newTargets := map[string]struct{}{} for _, p := range scalingPolicies { - newTargets[p.Target] = struct{}{} + newTargets[p.Target[structs.ScalingTargetGroup]] = struct{}{} } // find existing policies that need to be deleted deletedPolicies := []string{} @@ -4016,7 +4019,7 @@ func (s *StateStore) updateJobScalingPolicies(index uint64, job *structs.Job, tx break } oldPolicy := raw.(*structs.ScalingPolicy) - if _, ok := newTargets[oldPolicy.Target]; !ok { + if _, ok := newTargets[oldPolicy.Target[structs.ScalingTargetGroup]]; !ok { deletedPolicies = append(deletedPolicies, oldPolicy.ID) } } @@ -4720,35 +4723,45 @@ func (s *StateStore) UpsertScalingPolicies(index uint64, scalingPolicies []*stru func (s *StateStore) UpsertScalingPoliciesTxn(index uint64, scalingPolicies []*structs.ScalingPolicy, txn *memdb.Txn) error { - for _, scalingPolicy := range scalingPolicies { + hadUpdates := false + + for _, policy := range scalingPolicies { // Check if the scaling policy already exists - existing, err := txn.First("scaling_policy", "target", scalingPolicy.Namespace, scalingPolicy.Target) + existing, err := txn.First("scaling_policy", "target", + policy.Target[structs.ScalingTargetNamespace], + policy.Target[structs.ScalingTargetJob], + policy.Target[structs.ScalingTargetGroup]) if err != nil { return fmt.Errorf("scaling policy lookup failed: %v", err) } // Setup the indexes correctly if existing != nil { - scalingPolicy.CreateIndex = existing.(*structs.ScalingPolicy).CreateIndex - scalingPolicy.ModifyIndex = index - scalingPolicy.ID = existing.(*structs.ScalingPolicy).ID - } else { - scalingPolicy.CreateIndex = index - scalingPolicy.ModifyIndex = index - if scalingPolicy.ID == "" { - scalingPolicy.ID = uuid.Generate() + p := existing.(*structs.ScalingPolicy) + if !p.Diff(policy) { + continue } + policy.ID = p.ID + policy.CreateIndex = p.CreateIndex + policy.ModifyIndex = index + } else { + // policy.ID must have been set already in Job.Register before log apply + policy.CreateIndex = index + policy.ModifyIndex = index } // Insert the scaling policy - if err := txn.Insert("scaling_policy", scalingPolicy); err != nil { + hadUpdates = true + if err := txn.Insert("scaling_policy", policy); err != nil { return err } } // Update the indexes table for scaling policy - if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { - return fmt.Errorf("index update failed: %v", err) + if hadUpdates { + if err := txn.Insert("index", &IndexEntry{"scaling_policy", index}); err != nil { + return fmt.Errorf("index update failed: %v", err) + } } return nil @@ -4815,7 +4828,7 @@ func (s *StateStore) ScalingPoliciesByJob(ws memdb.WatchSet, namespace, jobID st func (s *StateStore) ScalingPoliciesByJobTxn(ws memdb.WatchSet, namespace, jobID string, txn *memdb.Txn) (memdb.ResultIterator, error) { - iter, err := txn.Get("scaling_policy", "job", namespace, jobID) + iter, err := txn.Get("scaling_policy", "target_prefix", namespace, jobID) if err != nil { return nil, err } @@ -4840,10 +4853,16 @@ func (s *StateStore) ScalingPolicyByID(ws memdb.WatchSet, id string) (*structs.S return nil, nil } -func (s *StateStore) ScalingPolicyByTarget(ws memdb.WatchSet, namespace, target string) (*structs.ScalingPolicy, error) { +func (s *StateStore) ScalingPolicyByTarget(ws memdb.WatchSet, target map[string]string) (*structs.ScalingPolicy, + error) { txn := s.db.Txn(false) - watchCh, existing, err := txn.FirstWatch("scaling_policy", "target", namespace, target) + // currently, only scaling policy type is against a task group + namespace := target[structs.ScalingTargetNamespace] + job := target[structs.ScalingTargetJob] + group := target[structs.ScalingTargetGroup] + + watchCh, existing, err := txn.FirstWatch("scaling_policy", "target", namespace, job, group) if err != nil { return nil, fmt.Errorf("scaling_policy lookup failed: %v", err) } diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 991e2e94077..52dc41e76d1 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -10,12 +10,13 @@ import ( "time" "github.com/hashicorp/go-memdb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func testStateStore(t *testing.T) *StateStore { @@ -7928,10 +7929,10 @@ func TestStateStore_UpsertScalingPolicy(t *testing.T) { policy2 := mock.ScalingPolicy() ws := memdb.NewWatchSet() - _, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + _, err := state.ScalingPolicyByTarget(ws, policy.Target) require.NoError(err) - _, err = state.ScalingPolicyByTarget(ws, policy.Namespace, policy2.Target) + _, err = state.ScalingPolicyByTarget(ws, policy2.Target) require.NoError(err) err = state.UpsertScalingPolicies(1000, []*structs.ScalingPolicy{policy, policy2}) @@ -7939,15 +7940,15 @@ func TestStateStore_UpsertScalingPolicy(t *testing.T) { require.True(watchFired(ws)) ws = memdb.NewWatchSet() - out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + out, err := state.ScalingPolicyByTarget(ws, policy.Target) require.NoError(err) require.Equal(policy, out) - out, err = state.ScalingPolicyByTarget(ws, policy2.Namespace, policy2.Target) + out, err = state.ScalingPolicyByTarget(ws, policy2.Target) require.NoError(err) require.Equal(policy2, out) - iter, err := state.ScalingPoliciesByNamespace(ws, policy.Namespace) + iter, err := state.ScalingPoliciesByNamespace(ws, policy.Target[structs.ScalingTargetNamespace]) require.NoError(err) // Ensure we see both policies @@ -7977,7 +7978,7 @@ func TestStateStore_UpsertJob_UpsertScalingPolicies(t *testing.T) { // Create a watchset so we can test that upsert fires the watch ws := memdb.NewWatchSet() - out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + out, err := state.ScalingPolicyByTarget(ws, policy.Target) require.NoError(err) require.Nil(out) @@ -7987,7 +7988,7 @@ func TestStateStore_UpsertJob_UpsertScalingPolicies(t *testing.T) { require.True(watchFired(ws), "watch did not fire") ws = memdb.NewWatchSet() - out, err = state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + out, err = state.ScalingPolicyByTarget(ws, policy.Target) require.NoError(err) require.NotNil(out) require.Equal(newIndex, out.CreateIndex) @@ -7997,6 +7998,94 @@ func TestStateStore_UpsertJob_UpsertScalingPolicies(t *testing.T) { require.Equal(newIndex, index) } +// Scaling Policy IDs are generated randomly during Job.Register +// Subsequent updates of the job should preserve the ID for the scaling policy +// associated with a given target. +func TestStateStore_UpsertJob_PreserveScalingPolicyIDsAndIndex(t *testing.T) { + t.Parallel() + + require := require.New(t) + + state := testStateStore(t) + job, policy := mock.JobWithScalingPolicy() + + var newIndex uint64 = 1000 + err := state.UpsertJob(newIndex, job) + require.NoError(err) + + ws := memdb.NewWatchSet() + p1, err := state.ScalingPolicyByTarget(ws, policy.Target) + require.NoError(err) + require.NotNil(p1) + require.Equal(newIndex, p1.CreateIndex) + require.Equal(newIndex, p1.ModifyIndex) + + index, err := state.Index("scaling_policy") + require.Equal(newIndex, index) + require.NotEmpty(p1.ID) + + // update the job + job.Meta["new-meta"] = "new-value" + newIndex += 100 + err = state.UpsertJob(newIndex, job) + require.NoError(err) + require.False(watchFired(ws), "watch should not have fired") + + p2, err := state.ScalingPolicyByTarget(nil, policy.Target) + require.NoError(err) + require.NotNil(p2) + require.Equal(p1.ID, p2.ID, "ID should not have changed") + require.Equal(p1.CreateIndex, p2.CreateIndex) + require.Equal(p1.ModifyIndex, p2.ModifyIndex) + + index, err = state.Index("scaling_policy") + require.Equal(index, p1.CreateIndex, "table index should not have changed") +} + +// Updating the scaling policy for a job should update the index table and fire the watch. +// This test is the converse of TestStateStore_UpsertJob_PreserveScalingPolicyIDsAndIndex +func TestStateStore_UpsertJob_UpdateScalingPolicy(t *testing.T) { + t.Parallel() + + require := require.New(t) + + state := testStateStore(t) + job, policy := mock.JobWithScalingPolicy() + + var oldIndex uint64 = 1000 + require.NoError(state.UpsertJob(oldIndex, job)) + + ws := memdb.NewWatchSet() + p1, err := state.ScalingPolicyByTarget(ws, policy.Target) + require.NoError(err) + require.NotNil(p1) + require.Equal(oldIndex, p1.CreateIndex) + require.Equal(oldIndex, p1.ModifyIndex) + prevId := p1.ID + + index, err := state.Index("scaling_policy") + require.Equal(oldIndex, index) + require.NotEmpty(p1.ID) + + // update the job with the updated scaling policy; make sure to use a different object + newPolicy := structs.CopyScalingPolicy(p1) + newPolicy.Policy["new-field"] = "new-value" + job.TaskGroups[0].Scaling = newPolicy + require.NoError(state.UpsertJob(oldIndex+100, job)) + require.True(watchFired(ws), "watch should have fired") + + p2, err := state.ScalingPolicyByTarget(nil, policy.Target) + require.NoError(err) + require.NotNil(p2) + require.Equal(p2.Policy["new-field"], "new-value") + require.Equal(prevId, p2.ID, "ID should not have changed") + require.Equal(oldIndex, p2.CreateIndex) + require.Greater(p2.ModifyIndex, oldIndex, "ModifyIndex should have advanced") + + index, err = state.Index("scaling_policy") + require.Greater(index, oldIndex, "table index should have advanced") +} + func TestStateStore_DeleteScalingPolicies(t *testing.T) { t.Parallel() @@ -8012,7 +8101,7 @@ func TestStateStore_DeleteScalingPolicies(t *testing.T) { // Create a watcher ws := memdb.NewWatchSet() - _, err = state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + _, err = state.ScalingPolicyByTarget(ws, policy.Target) require.NoError(err) // Delete the policy @@ -8024,17 +8113,17 @@ func TestStateStore_DeleteScalingPolicies(t *testing.T) { // Ensure we don't get the objects back ws = memdb.NewWatchSet() - out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + out, err := state.ScalingPolicyByTarget(ws, policy.Target) require.NoError(err) require.Nil(out) ws = memdb.NewWatchSet() - out, err = state.ScalingPolicyByTarget(ws, policy2.Namespace, policy2.Target) + out, err = state.ScalingPolicyByTarget(ws, policy2.Target) require.NoError(err) require.Nil(out) // Ensure we see both policies - iter, err := state.ScalingPoliciesByNamespace(ws, policy.Namespace) + iter, err := state.ScalingPoliciesByNamespace(ws, policy.Target[structs.ScalingTargetNamespace]) require.NoError(err) count := 0 for { @@ -8065,7 +8154,7 @@ func TestStateStore_DeleteJob_ChildScalingPolicies(t *testing.T) { require.NoError(err) policy := mock.ScalingPolicy() - policy.JobID = job.ID + policy.Target[structs.ScalingTargetJob] = job.ID err = state.UpsertScalingPolicies(1001, []*structs.ScalingPolicy{policy}) require.NoError(err) @@ -8075,13 +8164,44 @@ func TestStateStore_DeleteJob_ChildScalingPolicies(t *testing.T) { // Ensure the scaling policy was deleted ws := memdb.NewWatchSet() - out, err := state.ScalingPolicyByTarget(ws, policy.Namespace, policy.Target) + out, err := state.ScalingPolicyByTarget(ws, policy.Target) require.NoError(err) require.Nil(out) index, err := state.Index("scaling_policy") require.True(index > 1001) } +// This test ensures that deleting a job that doesn't have any scaling policies +// will not cause the scaling_policy table index to increase, on either job +// registration or deletion. +func TestStateStore_DeleteJob_ScalingPolicyIndexNoop(t *testing.T) { + t.Parallel() + + require := require.New(t) + + state := testStateStore(t) + + job := mock.Job() + + prevIndex, err := state.Index("scaling_policy") + require.NoError(err) + + err = state.UpsertJob(1000, job) + require.NoError(err) + + newIndex, err := state.Index("scaling_policy") + require.NoError(err) + require.Equal(prevIndex, newIndex) + + // Delete the job + err = state.DeleteJob(1002, job.Namespace, job.ID) + require.NoError(err) + + newIndex, err = state.Index("scaling_policy") + require.NoError(err) + require.Equal(prevIndex, newIndex) +} + func TestStateStore_ScalingPoliciesByJob(t *testing.T) { t.Parallel() @@ -8091,14 +8211,16 @@ func TestStateStore_ScalingPoliciesByJob(t *testing.T) { policyA := mock.ScalingPolicy() policyB1 := mock.ScalingPolicy() policyB2 := mock.ScalingPolicy() - policyB1.JobID = policyB2.JobID + policyB1.Target[structs.ScalingTargetJob] = policyB2.Target[structs.ScalingTargetJob] // Create the policies var baseIndex uint64 = 1000 err := state.UpsertScalingPolicies(baseIndex, []*structs.ScalingPolicy{policyA, policyB1, policyB2}) require.NoError(err) - iter, err := state.ScalingPoliciesByJob(nil, policyA.Namespace, policyA.JobID) + iter, err := state.ScalingPoliciesByJob(nil, + policyA.Target[structs.ScalingTargetNamespace], + policyA.Target[structs.ScalingTargetJob]) require.NoError(err) // Ensure we see expected policies @@ -8110,15 +8232,17 @@ func TestStateStore_ScalingPoliciesByJob(t *testing.T) { break } count++ - found = append(found, raw.(*structs.ScalingPolicy).Target) + found = append(found, raw.(*structs.ScalingPolicy).Target[structs.ScalingTargetGroup]) } require.Equal(1, count) sort.Strings(found) - expect := []string{policyA.Target} + expect := []string{policyA.Target[structs.ScalingTargetGroup]} sort.Strings(expect) require.Equal(expect, found) - iter, err = state.ScalingPoliciesByJob(nil, policyB1.Namespace, policyB1.JobID) + iter, err = state.ScalingPoliciesByJob(nil, + policyB1.Target[structs.ScalingTargetNamespace], + policyB1.Target[structs.ScalingTargetJob]) require.NoError(err) // Ensure we see expected policies @@ -8130,11 +8254,14 @@ func TestStateStore_ScalingPoliciesByJob(t *testing.T) { break } count++ - found = append(found, raw.(*structs.ScalingPolicy).Target) + found = append(found, raw.(*structs.ScalingPolicy).Target[structs.ScalingTargetGroup]) } require.Equal(2, count) sort.Strings(found) - expect = []string{policyB1.Target, policyB2.Target} + expect = []string{ + policyB1.Target[structs.ScalingTargetGroup], + policyB2.Target[structs.ScalingTargetGroup], + } sort.Strings(expect) require.Equal(expect, found) } diff --git a/nomad/structs/funcs.go b/nomad/structs/funcs.go index 11c48de0404..8e1abc6e213 100644 --- a/nomad/structs/funcs.go +++ b/nomad/structs/funcs.go @@ -13,6 +13,7 @@ import ( multierror "github.com/hashicorp/go-multierror" lru "github.com/hashicorp/golang-lru" "github.com/hashicorp/nomad/acl" + "github.com/mitchellh/copystructure" "golang.org/x/crypto/blake2b" ) @@ -257,16 +258,22 @@ func CopyScalingPolicy(p *ScalingPolicy) *ScalingPolicy { return nil } + opaquePolicyConfig, err := copystructure.Copy(p.Policy) + if err != nil { + panic(err.Error()) + } + c := ScalingPolicy{ ID: p.ID, - Namespace: p.Namespace, - Target: p.Target, - JobID: p.JobID, - Policy: p.Policy, + Policy: opaquePolicyConfig.(map[string]interface{}), Enabled: p.Enabled, CreateIndex: p.CreateIndex, ModifyIndex: p.ModifyIndex, } + c.Target = make(map[string]string, len(p.Target)) + for k, v := range p.Target { + c.Target[k] = v + } return &c } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 8ec97290e8d..f0e86d80fd4 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -4619,18 +4619,18 @@ type ScalingPolicy struct { // ID is a generated UUID used for looking up the scaling policy ID string - // Namespace is the namespace for the containing job - Namespace string - - // Target is the scaling target; there can be only one policy per scaling target - Target string - - // JobID is the ID of the parent job; there can be multiple policies per job - JobID string + // Target contains information about the target of the scaling policy, like job and group + Target map[string]string // Policy is an opaque description of the scaling policy, passed to the autoscaler Policy map[string]interface{} + // Min is the minimum allowable scaling count for this target + Min int64 + + // Max is the maximum allowable scaling count for this target + Max int64 + // Enabled indicates whether this policy has been enabled/disabled Enabled bool @@ -4638,20 +4638,42 @@ type ScalingPolicy struct { ModifyIndex uint64 } +const ( + ScalingTargetNamespace = "Namespace" + ScalingTargetJob = "Job" + ScalingTargetGroup = "Group" +) + +// Diff indicates whether the specification for a given scaling policy has changed +func (p *ScalingPolicy) Diff(p2 *ScalingPolicy) bool { + copy := *p2 + copy.ID = p.ID + copy.CreateIndex = p.CreateIndex + copy.ModifyIndex = p.ModifyIndex + return !reflect.DeepEqual(*p, copy) +} + func (p *ScalingPolicy) TargetTaskGroup(job *Job, tg *TaskGroup) *ScalingPolicy { - p.Target = fmt.Sprintf("/v1/job/%s/%s/scale", job.ID, tg.Name) + p.Target = map[string]string{ + ScalingTargetNamespace: job.Namespace, + ScalingTargetJob: job.ID, + ScalingTargetGroup: tg.Name, + } return p } func (p *ScalingPolicy) Stub() *ScalingPolicyListStub { - return &ScalingPolicyListStub{ + stub := &ScalingPolicyListStub{ ID: p.ID, - JobID: p.JobID, - Target: p.Target, + Target: make(map[string]string), Enabled: p.Enabled, CreateIndex: p.CreateIndex, ModifyIndex: p.ModifyIndex, } + for k, v := range p.Target { + stub.Target[k] = v + } + return stub } // GetScalingPolicies returns a slice of all scaling scaling policies for this job @@ -4671,9 +4693,8 @@ func (j *Job) GetScalingPolicies() []*ScalingPolicy { // for the scaling policy list type ScalingPolicyListStub struct { ID string - JobID string - Target string Enabled bool + Target map[string]string CreateIndex uint64 ModifyIndex uint64 } From 66bf8dd48dedcf95ac9bfaeab7261527f5125a70 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Wed, 18 Mar 2020 14:32:59 +0000 Subject: [PATCH 18/30] wip: some tests still failing updating job scaling endpoints to match RFC, cleaning up the API object as well --- api/jobs.go | 18 ++++-- api/jobs_test.go | 66 ++++++++------------- api/scaling.go | 27 +++++++-- command/agent/job_endpoint.go | 44 +++++++------- jobspec/test-fixtures/tg-scaling-policy.hcl | 5 +- nomad/job_endpoint.go | 48 ++++++++------- nomad/structs/structs.go | 9 ++- 7 files changed, 117 insertions(+), 100 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index 05f1e007afd..c8df51be342 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -155,13 +155,20 @@ func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) { // Scale is used to retrieve information about a particular // job given its unique ID. -func (j *Jobs) Scale(jobID, group string, value interface{}, reason string, q *WriteOptions) (*JobRegisterResponse, *WriteMeta, error) { +func (j *Jobs) Scale(jobID, group string, count int, + reason, error *string, meta map[string]interface{}, q *WriteOptions) (*JobRegisterResponse, *WriteMeta, error) { req := &ScalingRequest{ - Value: value, + Count: int64(count), + Target: map[string]string{ + "Job": jobID, + "Group": group, + }, + Error: error, Reason: reason, + Meta: meta, } var resp JobRegisterResponse - qm, err := j.client.write(fmt.Sprintf("/v1/job/%s/%s/scale", url.PathEscape(jobID), url.PathEscape(group)), req, &resp, q) + qm, err := j.client.write(fmt.Sprintf("/v1/job/%s/scale", url.PathEscape(jobID)), req, &resp, q) if err != nil { return nil, nil, err } @@ -170,10 +177,9 @@ func (j *Jobs) Scale(jobID, group string, value interface{}, reason string, q *W // ScaleStatus is used to retrieve information about a particular // job given its unique ID. -func (j *Jobs) ScaleStatus(jobID, group string, q *QueryOptions) (*ScaleStatusResponse, *QueryMeta, error) { +func (j *Jobs) ScaleStatus(jobID string, q *QueryOptions) (*ScaleStatusResponse, *QueryMeta, error) { var resp ScaleStatusResponse - qm, err := j.client.query(fmt.Sprintf("/v1/job/%s/%s/scale", url.PathEscape(jobID), url.PathEscape(group)), - &resp, q) + qm, err := j.client.query(fmt.Sprintf("/v1/job/%s/scale", url.PathEscape(jobID)), &resp, q) if err != nil { return nil, nil, err } diff --git a/api/jobs_test.go b/api/jobs_test.go index 662a9f1a366..dc2d3a858e2 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -909,7 +909,7 @@ func TestJobs_Info(t *testing.T) { } } -func TestJobs_Scale(t *testing.T) { +func TestJobs_ScaleInvalidAction(t *testing.T) { t.Parallel() require := require.New(t) @@ -921,18 +921,16 @@ func TestJobs_Scale(t *testing.T) { tests := []struct { jobID string group string - value interface{} + value int want string }{ {"", "", 1, "404"}, {"i-dont-exist", "", 1, "400"}, - {"", "i-dont-exist", 1, "400"}, - {"i-dont-exist", "me-neither", 1, "EOF"}, // TODO: this should be a 404 - {"id", "group", nil, "500"}, - {"id", "group", "not-int", "500"}, + {"", "i-dont-exist", 1, "404"}, + {"i-dont-exist", "me-neither", 1, "404"}, } for _, test := range tests { - _, _, err := jobs.Scale(test.jobID, test.group, test.value, "", nil) + _, _, err := jobs.Scale(test.jobID, test.group, test.value, stringToPtr("reason"), nil, nil, nil) require.Errorf(err, "expected jobs.Scale(%s, %s) to fail", test.jobID, test.group) require.Containsf(err.Error(), test.want, "jobs.Scale(%s, %s) error doesn't contain %s, got: %s", test.jobID, test.group, test.want, err) } @@ -944,18 +942,11 @@ func TestJobs_Scale(t *testing.T) { require.NoError(err) assertWriteMeta(t, wm) - // Scale job task group - value := 2 - _, wm, err = jobs.Scale(*job.ID, *job.TaskGroups[0].Name, value, "reason", nil) - require.NoError(err) - assertWriteMeta(t, wm) - - // Query the job again - resp, _, err := jobs.Info(*job.ID, nil) - require.NoError(err) - require.Equal(*resp.TaskGroups[0].Count, value) - - // TODO: check if reason is stored + // Perform a scaling action with bad group name, verify error + _, _, err = jobs.Scale(*job.ID, "incorrect-group-name", 2, + stringToPtr("because"), nil, nil, nil) + require.Error(err) + require.Contains(err.Error(), "does not exist") } func TestJobs_Versions(t *testing.T) { @@ -1586,7 +1577,6 @@ func TestJobs_AddSpread(t *testing.T) { // TestJobs_ScaleAction tests the scale target for task group count func TestJobs_ScaleAction(t *testing.T) { t.Parallel() - require := require.New(t) c, s := makeClient(t, nil, nil) @@ -1600,32 +1590,31 @@ func TestJobs_ScaleAction(t *testing.T) { groupCount := *job.TaskGroups[0].Count // Trying to scale against a target before it exists returns an error - _, _, err := jobs.Scale(id, "missing", - groupCount+1, "this won't work", nil) + _, _, err := jobs.Scale(id, "missing", groupCount+1, stringToPtr("this won't work"), nil, nil, nil) require.Error(err) require.Contains(err.Error(), "not found") // Register the job _, wm, err := jobs.Register(job, nil) - if err != nil { - t.Fatalf("err: %s", err) - } + require.NoError(err) assertWriteMeta(t, wm) - // Perform a scaling action with bad group name, verify error - _, _, err = jobs.Scale(id, "incorrect-group-name", - groupCount+1, "this won't work", nil) - require.Error(err) - require.Contains(err.Error(), "does not exist") - - // Query the scaling endpoint and verify success + // Perform scaling action + newCount := groupCount + 1 resp1, wm, err := jobs.Scale(id, groupName, - groupCount+1, "need more instances", nil) + newCount, stringToPtr("need more instances"), nil, nil, nil) require.NoError(err) require.NotNil(resp1) require.NotEmpty(resp1.EvalID) assertWriteMeta(t, wm) + + // Query the job again + resp, _, err := jobs.Info(*job.ID, nil) + require.NoError(err) + require.Equal(*resp.TaskGroups[0].Count, newCount) + + // TODO: check if reason is stored } // TestJobs_ScaleStatus tests the /scale status endpoint for task group count @@ -1640,7 +1629,7 @@ func TestJobs_ScaleStatus(t *testing.T) { // Trying to retrieve a status before it exists returns an error id := "job-id/with\\troublesome:characters\n?&å­—\000" - _, _, err := jobs.ScaleStatus(id, "missing", nil) + _, _, err := jobs.ScaleStatus(id, nil) require.Error(err) require.Contains(err.Error(), "not found") @@ -1655,16 +1644,11 @@ func TestJobs_ScaleStatus(t *testing.T) { } assertWriteMeta(t, wm) - // Query the scaling endpoint with bad group name, verify error - _, _, err = jobs.ScaleStatus(id, "incorrect-group-name", nil) - require.Error(err) - require.Contains(err.Error(), "not found") - // Query the scaling endpoint and verify success - result, qm, err := jobs.ScaleStatus(id, groupName, nil) + result, qm, err := jobs.ScaleStatus(id, nil) require.NoError(err) assertQueryMeta(t, qm) // Check that the result is what we expect - require.Equal(groupCount, int(result.Value.(float64))) + require.Equal(groupCount, result.TaskGroups[groupName].Desired) } diff --git a/api/scaling.go b/api/scaling.go index 5107fdecf30..2eae4abf5af 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -36,9 +36,10 @@ func (p *ScalingPolicy) Canonicalize() { // ScalingRequest is the payload for a generic scaling action type ScalingRequest struct { - Value interface{} - Reason string - Error string + Count int64 + Target map[string]string + Reason *string + Error *string Meta map[string]interface{} WriteRequest // this is effectively a job update, so we need the ability to override policy. @@ -71,6 +72,24 @@ type ScalingPolicyListStub struct { // ScaleStatusResponse is the payload for a generic scaling action type ScaleStatusResponse struct { JobID string + JobCreateIndex uint64 JobModifyIndex uint64 - Value interface{} + Stopped bool + TaskGroups map[string]TaskGroupScaleStatus +} + +type TaskGroupScaleStatus struct { + Desired uint64 + Running uint64 + Started uint64 + Healthy uint64 + Events []ScalingEvent +} + +type ScalingEvent struct { + Reason *string + Error *string + Meta map[string]interface{} + Time uint64 + EvalID *string } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index b8d250ddc5d..2cfcb026bf5 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -83,7 +83,8 @@ func (s *HTTPServer) JobSpecificRequest(resp http.ResponseWriter, req *http.Requ jobName := strings.TrimSuffix(path, "/stable") return s.jobStable(resp, req, jobName) case strings.HasSuffix(path, "/scale"): - return s.jobScale(resp, req, path) + jobName := strings.TrimSuffix(path, "/scale") + return s.jobScale(resp, req, jobName) default: return s.jobCRUD(resp, req, path) } @@ -457,30 +458,20 @@ func (s *HTTPServer) jobDelete(resp http.ResponseWriter, req *http.Request, } func (s *HTTPServer) jobScale(resp http.ResponseWriter, req *http.Request, - jobAndTarget string) (interface{}, error) { - - jobAndGroup := strings.TrimSuffix(jobAndTarget, "/scale") - var jobName, groupName string - if i := strings.LastIndex(jobAndGroup, "/"); i != -1 { - jobName = jobAndGroup[:i] - groupName = jobAndGroup[i+1:] - } - if jobName == "" || groupName == "" { - return nil, CodedError(400, "Invalid scaling target") - } + jobName string) (interface{}, error) { switch req.Method { case "GET": - return s.jobScaleStatus(resp, req, jobName, groupName) + return s.jobScaleStatus(resp, req, jobName) case "PUT", "POST": - return s.jobScaleAction(resp, req, jobName, groupName) + return s.jobScaleAction(resp, req, jobName) default: return nil, CodedError(405, ErrInvalidMethod) } } func (s *HTTPServer) jobScaleStatus(resp http.ResponseWriter, req *http.Request, - jobName, groupName string) (interface{}, error) { + jobName string) (interface{}, error) { args := structs.JobSpecificRequest{ JobID: jobName, @@ -499,21 +490,19 @@ func (s *HTTPServer) jobScaleStatus(resp http.ResponseWriter, req *http.Request, return nil, CodedError(404, "job not found") } - group := out.Job.LookupTaskGroup(groupName) - if group == nil { - return nil, CodedError(404, "group not found in job") - } status := &api.ScaleStatusResponse{ JobID: out.Job.ID, - Value: group.Count, + JobCreateIndex: out.Job.CreateIndex, JobModifyIndex: out.Job.ModifyIndex, + Stopped: out.Job.Stop, + TaskGroups: nil, // TODO } return status, nil } func (s *HTTPServer) jobScaleAction(resp http.ResponseWriter, req *http.Request, - jobName, groupName string) (interface{}, error) { + jobName string) (interface{}, error) { if req.Method != "PUT" && req.Method != "POST" { return nil, CodedError(405, ErrInvalidMethod) @@ -524,12 +513,21 @@ func (s *HTTPServer) jobScaleAction(resp http.ResponseWriter, req *http.Request, return nil, CodedError(400, err.Error()) } + namespace := args.Target[structs.ScalingTargetNamespace] + targetJob := args.Target[structs.ScalingTargetJob] + if targetJob != "" && targetJob != jobName { + return nil, CodedError(400, "job ID in payload did not match URL") + } + scaleReq := structs.JobScaleRequest{ JobID: jobName, - GroupName: groupName, - Value: args.Value, + Namespace: namespace, + Target: args.Target, + Count: args.Count, PolicyOverride: args.PolicyOverride, Reason: args.Reason, + Error: args.Error, + Meta: args.Meta, } // parseWriteRequest overrides Namespace, Region and AuthToken // based on values from the original http request diff --git a/jobspec/test-fixtures/tg-scaling-policy.hcl b/jobspec/test-fixtures/tg-scaling-policy.hcl index 2875892b3f1..483e62a6432 100644 --- a/jobspec/test-fixtures/tg-scaling-policy.hcl +++ b/jobspec/test-fixtures/tg-scaling-policy.hcl @@ -2,11 +2,12 @@ job "elastic" { group "group" { scaling { enabled = false + policy { foo = "bar" - b = true + b = true val = 5 - f = .1 + f = .1 } } } diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 185ead74f14..e1c29b26d83 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -847,50 +847,54 @@ func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterRes } defer metrics.MeasureSince([]string{"nomad", "job", "scale"}, time.Now()) + // Validate the arguments + namespace := args.Target[structs.ScalingTargetNamespace] + jobID := args.Target[structs.ScalingTargetJob] + groupName := args.Target[structs.ScalingTargetGroup] + if namespace != "" && namespace != args.RequestNamespace() { + return structs.NewErrRPCCoded(400, "namespace in payload did not match header") + } else if namespace == "" { + namespace = args.RequestNamespace() + } + if jobID != "" && jobID != args.JobID { + return fmt.Errorf("job ID in payload did not match URL") + } + if groupName == "" { + return structs.NewErrRPCCoded(400, "missing task group name for scaling action") + } + // Check for submit-job permissions if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { return err - } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityScaleJob) { + } else if aclObj != nil && !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityScaleJob) { return structs.ErrPermissionDenied } - // Validate the arguments - if args.JobID == "" { - return fmt.Errorf("missing job ID for scaling") - } else if args.GroupName == "" { - return fmt.Errorf("missing task group name for scaling") - } else if args.Value == nil { - return fmt.Errorf("missing new scaling value") - } - newCount, ok := args.Value.(float64) - if !ok { - return fmt.Errorf("scaling value for task group must be int: %t %v", args.Value, args.Value) - } - // Lookup the job snap, err := j.srv.fsm.State().Snapshot() if err != nil { return err } ws := memdb.NewWatchSet() - job, err := snap.JobByID(ws, args.RequestNamespace(), args.JobID) + job, err := snap.JobByID(ws, namespace, args.JobID) if err != nil { return err } if job == nil { - return fmt.Errorf("job %q not found", args.JobID) + return structs.NewErrRPCCoded(404, fmt.Sprintf("job %q not found", args.JobID)) } found := false for _, tg := range job.TaskGroups { - if args.GroupName == tg.Name { - tg.Count = int(newCount) + if groupName == tg.Name { + tg.Count = int(args.Count) // TODO: not safe, check this above found = true break } } if !found { - return fmt.Errorf("task group %q specified for scaling does not exist in job", args.GroupName) + return structs.NewErrRPCCoded(400, + fmt.Sprintf("task group %q specified for scaling does not exist in job", groupName)) } registerReq := structs.JobRegisterRequest{ Job: job, @@ -907,6 +911,9 @@ func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterRes return err } + // FINISH: + // register the scaling event to the scaling_event table, once that exists + // Populate the reply with job information reply.JobModifyIndex = index @@ -916,8 +923,7 @@ func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterRes } // Create a new evaluation - // XXX: The job priority / type is strange for this, since it's not a high - // priority even if the job was. + // FINISH: only do this if args.Error == nil || "" now := time.Now().UTC().UnixNano() eval := &structs.Evaluation{ ID: uuid.Generate(), diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index f0e86d80fd4..ee9c0fa56db 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -621,10 +621,13 @@ type JobPlanRequest struct { // JobScaleRequest is used for the Job.Scale endpoint to scale one of the // scaling targets in a job type JobScaleRequest struct { + Namespace string JobID string - GroupName string - Value interface{} - Reason string + Target map[string]string + Count int64 + Reason *string + Error *string + Meta map[string]interface{} // PolicyOverride is set when the user is attempting to override any policies PolicyOverride bool WriteRequest From 9243718899865bc39ea79f68fc69f36c5ae9682d Mon Sep 17 00:00:00 2001 From: James Rasell Date: Thu, 19 Mar 2020 15:30:14 +0100 Subject: [PATCH 19/30] scaling: ensure min and max int64s are in toplevel of block. --- jobspec/parse_group.go | 2 ++ nomad/structs/funcs.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/jobspec/parse_group.go b/jobspec/parse_group.go index 44b25185e5e..7285551b1cb 100644 --- a/jobspec/parse_group.go +++ b/jobspec/parse_group.go @@ -337,6 +337,8 @@ func parseScalingPolicy(out **api.ScalingPolicy, list *ast.ObjectList) error { } valid := []string{ + "min", + "max", "policy", "enabled", } diff --git a/nomad/structs/funcs.go b/nomad/structs/funcs.go index 8e1abc6e213..da449a1eca6 100644 --- a/nomad/structs/funcs.go +++ b/nomad/structs/funcs.go @@ -267,6 +267,8 @@ func CopyScalingPolicy(p *ScalingPolicy) *ScalingPolicy { ID: p.ID, Policy: opaquePolicyConfig.(map[string]interface{}), Enabled: p.Enabled, + Min: p.Min, + Max: p.Max, CreateIndex: p.CreateIndex, ModifyIndex: p.ModifyIndex, } From 03eb96aba23d7745c1c82d9b460dc2f5a4afd63a Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Fri, 20 Mar 2020 22:00:31 +0000 Subject: [PATCH 20/30] wip: scaling status return, almost done --- api/jobs.go | 4 +- api/scaling.go | 6 +-- command/agent/job_endpoint.go | 2 +- command/agent/job_endpoint_test.go | 2 +- nomad/job_endpoint.go | 70 +++++++++++++++++++++++++++++- nomad/job_endpoint_test.go | 58 +++++++++++++++++++++++++ nomad/structs/structs.go | 35 +++++++++++++++ 7 files changed, 169 insertions(+), 8 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index c8df51be342..843d85a9779 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -177,8 +177,8 @@ func (j *Jobs) Scale(jobID, group string, count int, // ScaleStatus is used to retrieve information about a particular // job given its unique ID. -func (j *Jobs) ScaleStatus(jobID string, q *QueryOptions) (*ScaleStatusResponse, *QueryMeta, error) { - var resp ScaleStatusResponse +func (j *Jobs) ScaleStatus(jobID string, q *QueryOptions) (*JobScaleStatusResponse, *QueryMeta, error) { + var resp JobScaleStatusResponse qm, err := j.client.query(fmt.Sprintf("/v1/job/%s/scale", url.PathEscape(jobID)), &resp, q) if err != nil { return nil, nil, err diff --git a/api/scaling.go b/api/scaling.go index 2eae4abf5af..3948b992177 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -69,12 +69,12 @@ type ScalingPolicyListStub struct { ModifyIndex uint64 } -// ScaleStatusResponse is the payload for a generic scaling action -type ScaleStatusResponse struct { +// JobScaleStatusResponse is the payload for a generic scaling action +type JobScaleStatusResponse struct { JobID string JobCreateIndex uint64 JobModifyIndex uint64 - Stopped bool + JobStopped bool TaskGroups map[string]TaskGroupScaleStatus } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 2cfcb026bf5..aafc52839b1 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -490,7 +490,7 @@ func (s *HTTPServer) jobScaleStatus(resp http.ResponseWriter, req *http.Request, return nil, CodedError(404, "job not found") } - status := &api.ScaleStatusResponse{ + status := &api.JobScaleStatusResponse{ JobID: out.Job.ID, JobCreateIndex: out.Job.CreateIndex, JobModifyIndex: out.Job.ModifyIndex, diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 6cf3b86d5f0..afbf0f6074e 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -758,7 +758,7 @@ func TestHTTP_Job_GroupScaleStatus(t *testing.T) { require.NoError(err) // Check the response - status := obj.(*api.ScaleStatusResponse) + status := obj.(*api.JobScaleStatusResponse) require.NotEmpty(resp.EvalID) require.Equal(job.TaskGroups[0].Count, status.Value.(int)) diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index e1c29b26d83..7f5ef41c032 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -840,7 +840,7 @@ func (j *Job) BatchDeregister(args *structs.JobBatchDeregisterRequest, reply *st return nil } -// Scale is used to modify one of the scaling targest in the job +// Scale is used to modify one of the scaling targets in the job func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterResponse) error { if done, err := j.srv.forward("Job.Scale", args, args, reply); done { return err @@ -1688,3 +1688,71 @@ func validateDispatchRequest(req *structs.JobDispatchRequest, job *structs.Job) return nil } + +// ScaleStatus retrieves the scaling status for a job +func (j *Job) ScaleStatus(args *structs.JobScaleStatusRequest, + reply *structs.JobScaleStatusResponse) error { + + if done, err := j.srv.forward("Job.ScaleStatus", args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "job", "scale_status"}, time.Now()) + + // FINISH + // Check for job-autoscaler permissions + // if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { + // return err + // } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { + // return structs.ErrPermissionDenied + // } + + // Setup the blocking query + opts := blockingOptions{ + queryOpts: &args.QueryOptions, + queryMeta: &reply.QueryMeta, + run: func(ws memdb.WatchSet, state *state.StateStore) error { + + // We need the job and the job summary + job, err := state.JobByID(ws, args.RequestNamespace(), args.JobID) + if err != nil { + return err + } + if job == nil { + return structs.NewErrRPCCoded(404, "job does not exist") + } + deployment, err := state.LatestDeploymentByJobID(ws, args.RequestNamespace(), args.JobID) + if err != nil { + return err + } + + // Setup the output + reply.JobModifyIndex = job.ModifyIndex + reply.JobCreateIndex = job.CreateIndex + reply.JobID = job.ID + reply.JobStopped = job.Stop + + for _, tg := range job.TaskGroups { + tgScale := &structs.TaskGroupScaleStatus{ + Desired: tg.Count, + } + if deployment != nil { + if ds, ok := deployment.TaskGroups[tg.Name]; ok { + tgScale.Placed = ds.PlacedAllocs + tgScale.Healthy = ds.HealthyAllocs + tgScale.Unhealthy = ds.UnhealthyAllocs + } + } + } + + if deployment != nil && deployment.ModifyIndex > job.ModifyIndex { + reply.Index = deployment.ModifyIndex + } else { + reply.Index = job.ModifyIndex + } + + // Set the query response + j.srv.setQueryMeta(&reply.QueryMeta) + return nil + }} + return j.srv.blockingRPC(&opts) +} diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 5954a2bc414..97fdccf1b5f 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -5267,3 +5267,61 @@ func TestJobEndpoint_Dispatch(t *testing.T) { }) } } + +func TestJobEndpoint_GetScaleStatus(t *testing.T) { + t.Parallel() + require := require.New(t) + + s1, cleanupS1 := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the register request + job := mock.Job() + req := &structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + + // Fetch the response + var resp structs.JobRegisterResponse + require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) + job.CreateIndex = resp.JobModifyIndex + job.ModifyIndex = resp.JobModifyIndex + + // Fetch the scaling status + get := &structs.JobScaleStatusRequest{ + JobID: job.ID, + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: job.Namespace, + }, + } + var resp2 structs.JobScaleStatusResponse + require.NoError(msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp2)) + + expectedStatus := structs.JobScaleStatusResponse{ + JobID: job.ID, + JobCreateIndex: job.CreateIndex, + JobModifyIndex: job.ModifyIndex, + JobStopped: job.Stop, + TaskGroups: map[string]structs.TaskGroupScaleStatus{ + job.TaskGroups[0].Name: { + Desired: 1, + Placed: 1, + Running: 1, + Healthy: 0, + Unhealthy: 0, + Events: nil, + }, + }, + } + + require.True(reflect.DeepEqual(resp2, expectedStatus)) +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index ee9c0fa56db..65f80cfe872 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -639,6 +639,12 @@ type JobSummaryRequest struct { QueryOptions } +// JobScaleStatusRequest is used to get the scale status for a job +type JobScaleStatusRequest struct { + JobID string + QueryOptions +} + // JobDispatchRequest is used to dispatch a job based on a parameterized job type JobDispatchRequest struct { JobID string @@ -1228,6 +1234,35 @@ type JobSummaryResponse struct { QueryMeta } +// JobScaleStatusResponse is used to return the scale status for a job +type JobScaleStatusResponse struct { + JobID string + JobCreateIndex uint64 + JobModifyIndex uint64 + JobStopped bool + TaskGroups map[string]TaskGroupScaleStatus + QueryMeta +} + +// TaskGroupScaleStatus is used to return the scale status for a given task group +type TaskGroupScaleStatus struct { + Desired int + Placed int + Running int + Healthy int + Unhealthy int + Events []ScalingEvent +} + +// ScalingEvent represents a specific scaling event +type ScalingEvent struct { + Reason *string + Error *string + Meta map[string]interface{} + Time uint64 + EvalID *string +} + type JobDispatchResponse struct { DispatchedJobID string EvalID string From 4759bc311080e3401bce9196c3b47cfe79df8687 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sat, 21 Mar 2020 14:18:43 +0000 Subject: [PATCH 21/30] finished Job.ScaleStatus RPC, need to work on http endpoint --- nomad/job_endpoint.go | 15 ++++++++++----- nomad/job_endpoint_test.go | 39 ++++++++++++++++++++++---------------- nomad/structs/structs.go | 8 ++++++-- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 7f5ef41c032..e5f45b96b77 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1718,7 +1718,8 @@ func (j *Job) ScaleStatus(args *structs.JobScaleStatusRequest, return err } if job == nil { - return structs.NewErrRPCCoded(404, "job does not exist") + reply.JobScaleStatus = nil + return nil } deployment, err := state.LatestDeploymentByJobID(ws, args.RequestNamespace(), args.JobID) if err != nil { @@ -1726,10 +1727,13 @@ func (j *Job) ScaleStatus(args *structs.JobScaleStatusRequest, } // Setup the output - reply.JobModifyIndex = job.ModifyIndex - reply.JobCreateIndex = job.CreateIndex - reply.JobID = job.ID - reply.JobStopped = job.Stop + reply.JobScaleStatus = &structs.JobScaleStatus{ + JobID: job.ID, + JobCreateIndex: job.CreateIndex, + JobModifyIndex: job.ModifyIndex, + JobStopped: job.Stop, + TaskGroups: make(map[string]*structs.TaskGroupScaleStatus), + } for _, tg := range job.TaskGroups { tgScale := &structs.TaskGroupScaleStatus{ @@ -1742,6 +1746,7 @@ func (j *Job) ScaleStatus(args *structs.JobScaleStatusRequest, tgScale.Unhealthy = ds.UnhealthyAllocs } } + reply.JobScaleStatus.TaskGroups[tg.Name] = tgScale } if deployment != nil && deployment.ModifyIndex > job.ModifyIndex { diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 97fdccf1b5f..e32271ce064 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -5279,8 +5279,22 @@ func TestJobEndpoint_GetScaleStatus(t *testing.T) { codec := rpcClient(t, s1) testutil.WaitForLeader(t, s1.RPC) - // Create the register request job := mock.Job() + + // check before job registration + // Fetch the scaling status + get := &structs.JobScaleStatusRequest{ + JobID: job.ID, + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: job.Namespace, + }, + } + var resp2 structs.JobScaleStatusResponse + require.NoError(msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp2)) + require.Nil(resp2.JobScaleStatus) + + // Create the register request req := &structs.JobRegisterRequest{ Job: job, WriteRequest: structs.WriteRequest{ @@ -5295,27 +5309,20 @@ func TestJobEndpoint_GetScaleStatus(t *testing.T) { job.CreateIndex = resp.JobModifyIndex job.ModifyIndex = resp.JobModifyIndex - // Fetch the scaling status - get := &structs.JobScaleStatusRequest{ - JobID: job.ID, - QueryOptions: structs.QueryOptions{ - Region: "global", - Namespace: job.Namespace, - }, - } - var resp2 structs.JobScaleStatusResponse + // check after job registration require.NoError(msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp2)) + require.NotNil(resp2.JobScaleStatus) - expectedStatus := structs.JobScaleStatusResponse{ + expectedStatus := structs.JobScaleStatus{ JobID: job.ID, JobCreateIndex: job.CreateIndex, JobModifyIndex: job.ModifyIndex, JobStopped: job.Stop, - TaskGroups: map[string]structs.TaskGroupScaleStatus{ + TaskGroups: map[string]*structs.TaskGroupScaleStatus{ job.TaskGroups[0].Name: { - Desired: 1, - Placed: 1, - Running: 1, + Desired: job.TaskGroups[0].Count, + Placed: 0, + Running: 0, Healthy: 0, Unhealthy: 0, Events: nil, @@ -5323,5 +5330,5 @@ func TestJobEndpoint_GetScaleStatus(t *testing.T) { }, } - require.True(reflect.DeepEqual(resp2, expectedStatus)) + require.True(reflect.DeepEqual(*resp2.JobScaleStatus, expectedStatus)) } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 65f80cfe872..9fff0958138 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1236,12 +1236,16 @@ type JobSummaryResponse struct { // JobScaleStatusResponse is used to return the scale status for a job type JobScaleStatusResponse struct { + JobScaleStatus *JobScaleStatus + QueryMeta +} + +type JobScaleStatus struct { JobID string JobCreateIndex uint64 JobModifyIndex uint64 JobStopped bool - TaskGroups map[string]TaskGroupScaleStatus - QueryMeta + TaskGroups map[string]*TaskGroupScaleStatus } // TaskGroupScaleStatus is used to return the scale status for a given task group From 2d88d57e52b498c5aec440dd2cb75d95ebee7675 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sat, 21 Mar 2020 15:03:22 +0000 Subject: [PATCH 22/30] fixed http endpoints for job.register and job.scalestatus --- api/scaling.go | 2 +- command/agent/job_endpoint.go | 18 +- command/agent/job_endpoint_test.go | 31 +- command/agent/scaling_endpoint_test.go | 2068 ------------------------ 4 files changed, 21 insertions(+), 2098 deletions(-) diff --git a/api/scaling.go b/api/scaling.go index 3948b992177..e2323edfa51 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -69,7 +69,7 @@ type ScalingPolicyListStub struct { ModifyIndex uint64 } -// JobScaleStatusResponse is the payload for a generic scaling action +// JobScaleStatusResponse is used to return information about job scaling status type JobScaleStatusResponse struct { JobID string JobCreateIndex uint64 diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index aafc52839b1..65196a6edd1 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -473,32 +473,24 @@ func (s *HTTPServer) jobScale(resp http.ResponseWriter, req *http.Request, func (s *HTTPServer) jobScaleStatus(resp http.ResponseWriter, req *http.Request, jobName string) (interface{}, error) { - args := structs.JobSpecificRequest{ + args := structs.JobScaleStatusRequest{ JobID: jobName, } if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil } - var out structs.SingleJobResponse - if err := s.agent.RPC("Job.GetJob", &args, &out); err != nil { + var out structs.JobScaleStatusResponse + if err := s.agent.RPC("Job.ScaleStatus", &args, &out); err != nil { return nil, err } setMeta(resp, &out.QueryMeta) - if out.Job == nil { + if out.JobScaleStatus == nil { return nil, CodedError(404, "job not found") } - status := &api.JobScaleStatusResponse{ - JobID: out.Job.ID, - JobCreateIndex: out.Job.CreateIndex, - JobModifyIndex: out.Job.ModifyIndex, - Stopped: out.Job.Stop, - TaskGroups: nil, // TODO - } - - return status, nil + return out.JobScaleStatus, nil } func (s *HTTPServer) jobScaleAction(resp http.ResponseWriter, req *http.Request, diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index afbf0f6074e..e7934681f9b 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -666,14 +666,14 @@ func TestHTTP_JobDelete(t *testing.T) { }) } -func TestHTTP_Job_GroupScale(t *testing.T) { +func TestHTTP_Job_ScaleTaskGroup(t *testing.T) { t.Parallel() require := require.New(t) httpTest(t, nil, func(s *TestAgent) { // Create the job - job, policy := mock.JobWithScalingPolicy() + job := mock.Job() args := structs.JobRegisterRequest{ Job: job, WriteRequest: structs.WriteRequest{ @@ -682,22 +682,21 @@ func TestHTTP_Job_GroupScale(t *testing.T) { }, } var resp structs.JobRegisterResponse - if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { - t.Fatalf("err: %v", err) - } - - // FINISH: cgbaker: do something with args.reason + require.NoError(s.Agent.RPC("Job.Register", &args, &resp)) newCount := job.TaskGroups[0].Count + 1 scaleReq := &api.ScalingRequest{ - JobID: job.ID, - Value: newCount, - Reason: "testing", + Count: int64(newCount), + Reason: helper.StringToPtr("testing"), + Target: map[string]string{ + "Job": job.ID, + "Group": job.TaskGroups[0].Name, + }, } buf := encodeReq(scaleReq) // Make the HTTP request to scale the job group - req, err := http.NewRequest("POST", policy.Target, buf) + req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/scale", buf) require.NoError(err) respW := httptest.NewRecorder() @@ -728,14 +727,14 @@ func TestHTTP_Job_GroupScale(t *testing.T) { }) } -func TestHTTP_Job_GroupScaleStatus(t *testing.T) { +func TestHTTP_Job_ScaleStatus(t *testing.T) { t.Parallel() require := require.New(t) httpTest(t, nil, func(s *TestAgent) { // Create the job - job, policy := mock.JobWithScalingPolicy() + job := mock.Job() args := structs.JobRegisterRequest{ Job: job, WriteRequest: structs.WriteRequest{ @@ -749,7 +748,7 @@ func TestHTTP_Job_GroupScaleStatus(t *testing.T) { } // Make the HTTP request to scale the job group - req, err := http.NewRequest("GET", policy.Target, nil) + req, err := http.NewRequest("GET", "/v1/job/"+job.ID+"/scale", nil) require.NoError(err) respW := httptest.NewRecorder() @@ -758,9 +757,9 @@ func TestHTTP_Job_GroupScaleStatus(t *testing.T) { require.NoError(err) // Check the response - status := obj.(*api.JobScaleStatusResponse) + status := obj.(*structs.JobScaleStatus) require.NotEmpty(resp.EvalID) - require.Equal(job.TaskGroups[0].Count, status.Value.(int)) + require.Equal(job.TaskGroups[0].Count, status.TaskGroups[job.TaskGroups[0].Name].Desired) // Check for the index require.NotEmpty(respW.Header().Get("X-Nomad-Index")) diff --git a/command/agent/scaling_endpoint_test.go b/command/agent/scaling_endpoint_test.go index c0d44737ced..10c3c418611 100644 --- a/command/agent/scaling_endpoint_test.go +++ b/command/agent/scaling_endpoint_test.go @@ -104,2071 +104,3 @@ func TestHTTP_ScalingPolicyGet(t *testing.T) { require.Equal(p.ID, obj.(*structs.ScalingPolicy).ID) }) } - -// func TestHTTP_JobQuery_Payload(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := mock.Job() -// -// // Insert Payload compressed -// expected := []byte("hello world") -// compressed := snappy.Encode(nil, expected) -// job.Payload = compressed -// -// // Directly manipulate the state -// state := s.Agent.server.State() -// if err := state.UpsertJob(1000, job); err != nil { -// t.Fatalf("Failed to upsert job: %v", err) -// } -// -// // Make the HTTP request -// req, err := http.NewRequest("GET", "/v1/job/"+job.ID, nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { -// t.Fatalf("missing known leader") -// } -// if respW.Header().Get("X-Nomad-LastContact") == "" { -// t.Fatalf("missing last contact") -// } -// -// // Check the job -// j := obj.(*structs.Job) -// if j.ID != job.ID { -// t.Fatalf("bad: %#v", j) -// } -// -// // Check the payload is decompressed -// if !reflect.DeepEqual(j.Payload, expected) { -// t.Fatalf("Payload not decompressed properly; got %#v; want %#v", j.Payload, expected) -// } -// }) -// } -// -// func TestHTTP_JobUpdate(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := MockJob() -// args := api.JobRegisterRequest{ -// Job: job, -// WriteRequest: api.WriteRequest{ -// Region: "global", -// Namespace: api.DefaultNamespace, -// }, -// } -// buf := encodeReq(args) -// -// // Make the HTTP request -// req, err := http.NewRequest("PUT", "/v1/job/"+*job.ID, buf) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// dereg := obj.(structs.JobRegisterResponse) -// if dereg.EvalID == "" { -// t.Fatalf("bad: %v", dereg) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// -// // Check the job is registered -// getReq := structs.JobSpecificRequest{ -// JobID: *job.ID, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var getResp structs.SingleJobResponse -// if err := s.Agent.RPC("Job.GetJob", &getReq, &getResp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// if getResp.Job == nil { -// t.Fatalf("job does not exist") -// } -// }) -// } -// -// func TestHTTP_JobUpdateRegion(t *testing.T) { -// t.Parallel() -// -// cases := []struct { -// Name string -// ConfigRegion string -// APIRegion string -// ExpectedRegion string -// }{ -// { -// Name: "api region takes precedence", -// ConfigRegion: "not-global", -// APIRegion: "north-america", -// ExpectedRegion: "north-america", -// }, -// { -// Name: "config region is set", -// ConfigRegion: "north-america", -// APIRegion: "", -// ExpectedRegion: "north-america", -// }, -// { -// Name: "api region is set", -// ConfigRegion: "", -// APIRegion: "north-america", -// ExpectedRegion: "north-america", -// }, -// { -// Name: "defaults to node region global if no region is provided", -// ConfigRegion: "", -// APIRegion: "", -// ExpectedRegion: "global", -// }, -// { -// Name: "defaults to node region not-global if no region is provided", -// ConfigRegion: "", -// APIRegion: "", -// ExpectedRegion: "not-global", -// }, -// } -// -// for _, tc := range cases { -// t.Run(tc.Name, func(t *testing.T) { -// httpTest(t, func(c *Config) { c.Region = tc.ExpectedRegion }, func(s *TestAgent) { -// // Create the job -// job := MockRegionalJob() -// -// if tc.ConfigRegion == "" { -// job.Region = nil -// } else { -// job.Region = &tc.ConfigRegion -// } -// -// args := api.JobRegisterRequest{ -// Job: job, -// WriteRequest: api.WriteRequest{ -// Namespace: api.DefaultNamespace, -// Region: tc.APIRegion, -// }, -// } -// -// buf := encodeReq(args) -// -// // Make the HTTP request -// url := "/v1/job/" + *job.ID -// -// req, err := http.NewRequest("PUT", url, buf) -// require.NoError(t, err) -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// require.NoError(t, err) -// -// // Check the response -// dereg := obj.(structs.JobRegisterResponse) -// require.NotEmpty(t, dereg.EvalID) -// -// // Check for the index -// require.NotEmpty(t, respW.Header().Get("X-Nomad-Index"), "missing index") -// -// // Check the job is registered -// getReq := structs.JobSpecificRequest{ -// JobID: *job.ID, -// QueryOptions: structs.QueryOptions{ -// Region: tc.ExpectedRegion, -// Namespace: structs.DefaultNamespace, -// }, -// } -// var getResp structs.SingleJobResponse -// err = s.Agent.RPC("Job.GetJob", &getReq, &getResp) -// require.NoError(t, err) -// require.NotNil(t, getResp.Job, "job does not exist") -// require.Equal(t, tc.ExpectedRegion, getResp.Job.Region) -// }) -// }) -// } -// } -// -// func TestHTTP_JobDelete(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := mock.Job() -// args := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Make the HTTP request to do a soft delete -// req, err := http.NewRequest("DELETE", "/v1/job/"+job.ID, nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// dereg := obj.(structs.JobDeregisterResponse) -// if dereg.EvalID == "" { -// t.Fatalf("bad: %v", dereg) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// -// // Check the job is still queryable -// getReq1 := structs.JobSpecificRequest{ -// JobID: job.ID, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var getResp1 structs.SingleJobResponse -// if err := s.Agent.RPC("Job.GetJob", &getReq1, &getResp1); err != nil { -// t.Fatalf("err: %v", err) -// } -// if getResp1.Job == nil { -// t.Fatalf("job doesn't exists") -// } -// if !getResp1.Job.Stop { -// t.Fatalf("job should be marked as stop") -// } -// -// // Make the HTTP request to do a purge delete -// req2, err := http.NewRequest("DELETE", "/v1/job/"+job.ID+"?purge=true", nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW.Flush() -// -// // Make the request -// obj, err = s.Server.JobSpecificRequest(respW, req2) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// dereg = obj.(structs.JobDeregisterResponse) -// if dereg.EvalID == "" { -// t.Fatalf("bad: %v", dereg) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// -// // Check the job is gone -// getReq2 := structs.JobSpecificRequest{ -// JobID: job.ID, -// QueryOptions: structs.QueryOptions{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var getResp2 structs.SingleJobResponse -// if err := s.Agent.RPC("Job.GetJob", &getReq2, &getResp2); err != nil { -// t.Fatalf("err: %v", err) -// } -// if getResp2.Job != nil { -// t.Fatalf("job still exists") -// } -// }) -// } -// -// func TestHTTP_JobForceEvaluate(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := mock.Job() -// args := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Make the HTTP request -// req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/evaluate", nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// reg := obj.(structs.JobRegisterResponse) -// if reg.EvalID == "" { -// t.Fatalf("bad: %v", reg) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// }) -// } -// -// func TestHTTP_JobEvaluate_ForceReschedule(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := mock.Job() -// args := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// jobEvalReq := api.JobEvaluateRequest{ -// JobID: job.ID, -// EvalOptions: api.EvalOptions{ -// ForceReschedule: true, -// }, -// } -// -// buf := encodeReq(jobEvalReq) -// -// // Make the HTTP request -// req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/evaluate", buf) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// reg := obj.(structs.JobRegisterResponse) -// if reg.EvalID == "" { -// t.Fatalf("bad: %v", reg) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// }) -// } -// -// func TestHTTP_JobEvaluations(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := mock.Job() -// args := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Make the HTTP request -// req, err := http.NewRequest("GET", "/v1/job/"+job.ID+"/evaluations", nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// evals := obj.([]*structs.Evaluation) -// // Can be multiple evals, use the last one, since they are in order -// idx := len(evals) - 1 -// if len(evals) < 0 || evals[idx].ID != resp.EvalID { -// t.Fatalf("bad: %v", evals) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { -// t.Fatalf("missing known leader") -// } -// if respW.Header().Get("X-Nomad-LastContact") == "" { -// t.Fatalf("missing last contact") -// } -// }) -// } -// -// func TestHTTP_JobAllocations(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// alloc1 := mock.Alloc() -// args := structs.JobRegisterRequest{ -// Job: alloc1.Job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Directly manipulate the state -// expectedDisplayMsg := "test message" -// testEvent := structs.NewTaskEvent("test event").SetMessage(expectedDisplayMsg) -// var events []*structs.TaskEvent -// events = append(events, testEvent) -// taskState := &structs.TaskState{Events: events} -// alloc1.TaskStates = make(map[string]*structs.TaskState) -// alloc1.TaskStates["test"] = taskState -// state := s.Agent.server.State() -// err := state.UpsertAllocs(1000, []*structs.Allocation{alloc1}) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Make the HTTP request -// req, err := http.NewRequest("GET", "/v1/job/"+alloc1.Job.ID+"/allocations?all=true", nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// allocs := obj.([]*structs.AllocListStub) -// if len(allocs) != 1 && allocs[0].ID != alloc1.ID { -// t.Fatalf("bad: %v", allocs) -// } -// displayMsg := allocs[0].TaskStates["test"].Events[0].DisplayMessage -// assert.Equal(t, expectedDisplayMsg, displayMsg) -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { -// t.Fatalf("missing known leader") -// } -// if respW.Header().Get("X-Nomad-LastContact") == "" { -// t.Fatalf("missing last contact") -// } -// }) -// } -// -// func TestHTTP_JobDeployments(t *testing.T) { -// assert := assert.New(t) -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// j := mock.Job() -// args := structs.JobRegisterRequest{ -// Job: j, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// assert.Nil(s.Agent.RPC("Job.Register", &args, &resp), "JobRegister") -// -// // Directly manipulate the state -// state := s.Agent.server.State() -// d := mock.Deployment() -// d.JobID = j.ID -// d.JobCreateIndex = resp.JobModifyIndex -// -// assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") -// -// // Make the HTTP request -// req, err := http.NewRequest("GET", "/v1/job/"+j.ID+"/deployments", nil) -// assert.Nil(err, "HTTP") -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// assert.Nil(err, "JobSpecificRequest") -// -// // Check the response -// deploys := obj.([]*structs.Deployment) -// assert.Len(deploys, 1, "deployments") -// assert.Equal(d.ID, deploys[0].ID, "deployment id") -// -// assert.NotZero(respW.Header().Get("X-Nomad-Index"), "missing index") -// assert.Equal("true", respW.Header().Get("X-Nomad-KnownLeader"), "missing known leader") -// assert.NotZero(respW.Header().Get("X-Nomad-LastContact"), "missing last contact") -// }) -// } -// -// func TestHTTP_JobDeployment(t *testing.T) { -// assert := assert.New(t) -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// j := mock.Job() -// args := structs.JobRegisterRequest{ -// Job: j, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// assert.Nil(s.Agent.RPC("Job.Register", &args, &resp), "JobRegister") -// -// // Directly manipulate the state -// state := s.Agent.server.State() -// d := mock.Deployment() -// d.JobID = j.ID -// d.JobCreateIndex = resp.JobModifyIndex -// assert.Nil(state.UpsertDeployment(1000, d), "UpsertDeployment") -// -// // Make the HTTP request -// req, err := http.NewRequest("GET", "/v1/job/"+j.ID+"/deployment", nil) -// assert.Nil(err, "HTTP") -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// assert.Nil(err, "JobSpecificRequest") -// -// // Check the response -// out := obj.(*structs.Deployment) -// assert.NotNil(out, "deployment") -// assert.Equal(d.ID, out.ID, "deployment id") -// -// assert.NotZero(respW.Header().Get("X-Nomad-Index"), "missing index") -// assert.Equal("true", respW.Header().Get("X-Nomad-KnownLeader"), "missing known leader") -// assert.NotZero(respW.Header().Get("X-Nomad-LastContact"), "missing last contact") -// }) -// } -// -// func TestHTTP_JobVersions(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := mock.Job() -// args := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// job2 := mock.Job() -// job2.ID = job.ID -// job2.Priority = 100 -// -// args2 := structs.JobRegisterRequest{ -// Job: job2, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp2 structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args2, &resp2); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Make the HTTP request -// req, err := http.NewRequest("GET", "/v1/job/"+job.ID+"/versions?diffs=true", nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// vResp := obj.(structs.JobVersionsResponse) -// versions := vResp.Versions -// if len(versions) != 2 { -// t.Fatalf("got %d versions; want 2", len(versions)) -// } -// -// if v := versions[0]; v.Version != 1 || v.Priority != 100 { -// t.Fatalf("bad %v", v) -// } -// -// if v := versions[1]; v.Version != 0 { -// t.Fatalf("bad %v", v) -// } -// -// if len(vResp.Diffs) != 1 { -// t.Fatalf("bad %v", vResp) -// } -// -// // Check for the index -// if respW.Header().Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// if respW.Header().Get("X-Nomad-KnownLeader") != "true" { -// t.Fatalf("missing known leader") -// } -// if respW.Header().Get("X-Nomad-LastContact") == "" { -// t.Fatalf("missing last contact") -// } -// }) -// } -// -// func TestHTTP_PeriodicForce(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create and register a periodic job. -// job := mock.PeriodicJob() -// args := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Make the HTTP request -// req, err := http.NewRequest("POST", "/v1/job/"+job.ID+"/periodic/force", nil) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check for the index -// if respW.HeaderMap.Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// -// // Check the response -// r := obj.(structs.PeriodicForceResponse) -// if r.EvalID == "" { -// t.Fatalf("bad: %#v", r) -// } -// }) -// } -// -// func TestHTTP_JobPlan(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := MockJob() -// args := api.JobPlanRequest{ -// Job: job, -// Diff: true, -// WriteRequest: api.WriteRequest{ -// Region: "global", -// Namespace: api.DefaultNamespace, -// }, -// } -// buf := encodeReq(args) -// -// // Make the HTTP request -// req, err := http.NewRequest("PUT", "/v1/job/"+*job.ID+"/plan", buf) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// plan := obj.(structs.JobPlanResponse) -// if plan.Annotations == nil { -// t.Fatalf("bad: %v", plan) -// } -// -// if plan.Diff == nil { -// t.Fatalf("bad: %v", plan) -// } -// }) -// } -// -// func TestHTTP_JobPlanRegion(t *testing.T) { -// t.Parallel() -// -// cases := []struct { -// Name string -// ConfigRegion string -// APIRegion string -// ExpectedRegion string -// }{ -// { -// Name: "api region takes precedence", -// ConfigRegion: "not-global", -// APIRegion: "north-america", -// ExpectedRegion: "north-america", -// }, -// { -// Name: "config region is set", -// ConfigRegion: "north-america", -// APIRegion: "", -// ExpectedRegion: "north-america", -// }, -// { -// Name: "api region is set", -// ConfigRegion: "", -// APIRegion: "north-america", -// ExpectedRegion: "north-america", -// }, -// { -// Name: "falls back to default if no region is provided", -// ConfigRegion: "", -// APIRegion: "", -// ExpectedRegion: "global", -// }, -// } -// -// for _, tc := range cases { -// t.Run(tc.Name, func(t *testing.T) { -// httpTest(t, func(c *Config) { c.Region = tc.ExpectedRegion }, func(s *TestAgent) { -// // Create the job -// job := MockRegionalJob() -// -// if tc.ConfigRegion == "" { -// job.Region = nil -// } else { -// job.Region = &tc.ConfigRegion -// } -// -// args := api.JobPlanRequest{ -// Job: job, -// Diff: true, -// WriteRequest: api.WriteRequest{ -// Region: tc.APIRegion, -// Namespace: api.DefaultNamespace, -// }, -// } -// buf := encodeReq(args) -// -// // Make the HTTP request -// req, err := http.NewRequest("PUT", "/v1/job/"+*job.ID+"/plan", buf) -// require.NoError(t, err) -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// require.NoError(t, err) -// -// // Check the response -// plan := obj.(structs.JobPlanResponse) -// require.NotNil(t, plan.Annotations) -// require.NotNil(t, plan.Diff) -// }) -// }) -// } -// } -// -// func TestHTTP_JobDispatch(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the parameterized job -// job := mock.BatchJob() -// job.ParameterizedJob = &structs.ParameterizedJobConfig{} -// -// args := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var resp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", &args, &resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Make the request -// respW := httptest.NewRecorder() -// args2 := structs.JobDispatchRequest{ -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// buf := encodeReq(args2) -// -// // Make the HTTP request -// req2, err := http.NewRequest("PUT", "/v1/job/"+job.ID+"/dispatch", buf) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW.Flush() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req2) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// dispatch := obj.(structs.JobDispatchResponse) -// if dispatch.EvalID == "" { -// t.Fatalf("bad: %v", dispatch) -// } -// -// if dispatch.DispatchedJobID == "" { -// t.Fatalf("bad: %v", dispatch) -// } -// }) -// } -// -// func TestHTTP_JobRevert(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job and register it twice -// job := mock.Job() -// regReq := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var regResp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Change the job to get a new version -// job.Datacenters = append(job.Datacenters, "foo") -// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// args := structs.JobRevertRequest{ -// JobID: job.ID, -// JobVersion: 0, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// buf := encodeReq(args) -// -// // Make the HTTP request -// req, err := http.NewRequest("PUT", "/v1/job/"+job.ID+"/revert", buf) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// revertResp := obj.(structs.JobRegisterResponse) -// if revertResp.EvalID == "" { -// t.Fatalf("bad: %v", revertResp) -// } -// -// // Check for the index -// if respW.HeaderMap.Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// }) -// } -// -// func TestHTTP_JobStable(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job and register it twice -// job := mock.Job() -// regReq := structs.JobRegisterRequest{ -// Job: job, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// var regResp structs.JobRegisterResponse -// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// if err := s.Agent.RPC("Job.Register", ®Req, ®Resp); err != nil { -// t.Fatalf("err: %v", err) -// } -// -// args := structs.JobStabilityRequest{ -// JobID: job.ID, -// JobVersion: 0, -// Stable: true, -// WriteRequest: structs.WriteRequest{ -// Region: "global", -// Namespace: structs.DefaultNamespace, -// }, -// } -// buf := encodeReq(args) -// -// // Make the HTTP request -// req, err := http.NewRequest("PUT", "/v1/job/"+job.ID+"/stable", buf) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.JobSpecificRequest(respW, req) -// if err != nil { -// t.Fatalf("err: %v", err) -// } -// -// // Check the response -// stableResp := obj.(structs.JobStabilityResponse) -// if stableResp.Index == 0 { -// t.Fatalf("bad: %v", stableResp) -// } -// -// // Check for the index -// if respW.HeaderMap.Get("X-Nomad-Index") == "" { -// t.Fatalf("missing index") -// } -// }) -// } -// -// func TestJobs_ApiJobToStructsJob(t *testing.T) { -// apiJob := &api.Job{ -// Stop: helper.BoolToPtr(true), -// Region: helper.StringToPtr("global"), -// Namespace: helper.StringToPtr("foo"), -// ID: helper.StringToPtr("foo"), -// ParentID: helper.StringToPtr("lol"), -// Name: helper.StringToPtr("name"), -// Type: helper.StringToPtr("service"), -// Priority: helper.IntToPtr(50), -// AllAtOnce: helper.BoolToPtr(true), -// Datacenters: []string{"dc1", "dc2"}, -// Constraints: []*api.Constraint{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// }, -// }, -// Affinities: []*api.Affinity{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// Weight: helper.Int8ToPtr(50), -// }, -// }, -// Update: &api.UpdateStrategy{ -// Stagger: helper.TimeToPtr(1 * time.Second), -// MaxParallel: helper.IntToPtr(5), -// HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual), -// MinHealthyTime: helper.TimeToPtr(1 * time.Minute), -// HealthyDeadline: helper.TimeToPtr(3 * time.Minute), -// ProgressDeadline: helper.TimeToPtr(3 * time.Minute), -// AutoRevert: helper.BoolToPtr(false), -// Canary: helper.IntToPtr(1), -// }, -// Spreads: []*api.Spread{ -// { -// Attribute: "${meta.rack}", -// Weight: helper.Int8ToPtr(100), -// SpreadTarget: []*api.SpreadTarget{ -// { -// Value: "r1", -// Percent: 50, -// }, -// }, -// }, -// }, -// Periodic: &api.PeriodicConfig{ -// Enabled: helper.BoolToPtr(true), -// Spec: helper.StringToPtr("spec"), -// SpecType: helper.StringToPtr("cron"), -// ProhibitOverlap: helper.BoolToPtr(true), -// TimeZone: helper.StringToPtr("test zone"), -// }, -// ParameterizedJob: &api.ParameterizedJobConfig{ -// Payload: "payload", -// MetaRequired: []string{"a", "b"}, -// MetaOptional: []string{"c", "d"}, -// }, -// Payload: []byte("payload"), -// Meta: map[string]string{ -// "foo": "bar", -// }, -// TaskGroups: []*api.TaskGroup{ -// { -// Name: helper.StringToPtr("group1"), -// Count: helper.IntToPtr(5), -// Constraints: []*api.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Affinities: []*api.Affinity{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// Weight: helper.Int8ToPtr(100), -// }, -// }, -// RestartPolicy: &api.RestartPolicy{ -// Interval: helper.TimeToPtr(1 * time.Second), -// Attempts: helper.IntToPtr(5), -// Delay: helper.TimeToPtr(10 * time.Second), -// Mode: helper.StringToPtr("delay"), -// }, -// ReschedulePolicy: &api.ReschedulePolicy{ -// Interval: helper.TimeToPtr(12 * time.Hour), -// Attempts: helper.IntToPtr(5), -// DelayFunction: helper.StringToPtr("constant"), -// Delay: helper.TimeToPtr(30 * time.Second), -// Unlimited: helper.BoolToPtr(true), -// MaxDelay: helper.TimeToPtr(20 * time.Minute), -// }, -// Migrate: &api.MigrateStrategy{ -// MaxParallel: helper.IntToPtr(12), -// HealthCheck: helper.StringToPtr("task_events"), -// MinHealthyTime: helper.TimeToPtr(12 * time.Hour), -// HealthyDeadline: helper.TimeToPtr(12 * time.Hour), -// }, -// Spreads: []*api.Spread{ -// { -// Attribute: "${node.datacenter}", -// Weight: helper.Int8ToPtr(100), -// SpreadTarget: []*api.SpreadTarget{ -// { -// Value: "dc1", -// Percent: 100, -// }, -// }, -// }, -// }, -// EphemeralDisk: &api.EphemeralDisk{ -// SizeMB: helper.IntToPtr(100), -// Sticky: helper.BoolToPtr(true), -// Migrate: helper.BoolToPtr(true), -// }, -// Update: &api.UpdateStrategy{ -// HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Checks), -// MinHealthyTime: helper.TimeToPtr(2 * time.Minute), -// HealthyDeadline: helper.TimeToPtr(5 * time.Minute), -// ProgressDeadline: helper.TimeToPtr(5 * time.Minute), -// AutoRevert: helper.BoolToPtr(true), -// }, -// Meta: map[string]string{ -// "key": "value", -// }, -// Services: []*api.Service{ -// { -// Name: "groupserviceA", -// Tags: []string{"a", "b"}, -// CanaryTags: []string{"d", "e"}, -// PortLabel: "1234", -// Meta: map[string]string{ -// "servicemeta": "foobar", -// }, -// CheckRestart: &api.CheckRestart{ -// Limit: 4, -// Grace: helper.TimeToPtr(11 * time.Second), -// }, -// Checks: []api.ServiceCheck{ -// { -// Id: "hello", -// Name: "bar", -// Type: "http", -// Command: "foo", -// Args: []string{"a", "b"}, -// Path: "/check", -// Protocol: "http", -// PortLabel: "foo", -// AddressMode: "driver", -// GRPCService: "foo.Bar", -// GRPCUseTLS: true, -// Interval: 4 * time.Second, -// Timeout: 2 * time.Second, -// InitialStatus: "ok", -// CheckRestart: &api.CheckRestart{ -// Limit: 3, -// IgnoreWarnings: true, -// }, -// TaskName: "task1", -// }, -// }, -// Connect: &api.ConsulConnect{ -// Native: false, -// SidecarService: &api.ConsulSidecarService{ -// Tags: []string{"f", "g"}, -// Port: "9000", -// }, -// }, -// }, -// }, -// Tasks: []*api.Task{ -// { -// Name: "task1", -// Leader: true, -// Driver: "docker", -// User: "mary", -// Config: map[string]interface{}{ -// "lol": "code", -// }, -// Env: map[string]string{ -// "hello": "world", -// }, -// Constraints: []*api.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Affinities: []*api.Affinity{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// Weight: helper.Int8ToPtr(50), -// }, -// }, -// -// Services: []*api.Service{ -// { -// Id: "id", -// Name: "serviceA", -// Tags: []string{"1", "2"}, -// CanaryTags: []string{"3", "4"}, -// PortLabel: "foo", -// Meta: map[string]string{ -// "servicemeta": "foobar", -// }, -// CheckRestart: &api.CheckRestart{ -// Limit: 4, -// Grace: helper.TimeToPtr(11 * time.Second), -// }, -// Checks: []api.ServiceCheck{ -// { -// Id: "hello", -// Name: "bar", -// Type: "http", -// Command: "foo", -// Args: []string{"a", "b"}, -// Path: "/check", -// Protocol: "http", -// PortLabel: "foo", -// AddressMode: "driver", -// GRPCService: "foo.Bar", -// GRPCUseTLS: true, -// Interval: 4 * time.Second, -// Timeout: 2 * time.Second, -// InitialStatus: "ok", -// CheckRestart: &api.CheckRestart{ -// Limit: 3, -// IgnoreWarnings: true, -// }, -// }, -// { -// Id: "check2id", -// Name: "check2", -// Type: "tcp", -// PortLabel: "foo", -// Interval: 4 * time.Second, -// Timeout: 2 * time.Second, -// }, -// }, -// }, -// }, -// Resources: &api.Resources{ -// CPU: helper.IntToPtr(100), -// MemoryMB: helper.IntToPtr(10), -// Networks: []*api.NetworkResource{ -// { -// IP: "10.10.11.1", -// MBits: helper.IntToPtr(10), -// ReservedPorts: []api.Port{ -// { -// Label: "http", -// Value: 80, -// }, -// }, -// DynamicPorts: []api.Port{ -// { -// Label: "ssh", -// Value: 2000, -// }, -// }, -// }, -// }, -// Devices: []*api.RequestedDevice{ -// { -// Name: "nvidia/gpu", -// Count: helper.Uint64ToPtr(4), -// Constraints: []*api.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Affinities: []*api.Affinity{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// Weight: helper.Int8ToPtr(50), -// }, -// }, -// }, -// { -// Name: "gpu", -// Count: nil, -// }, -// }, -// }, -// Meta: map[string]string{ -// "lol": "code", -// }, -// KillTimeout: helper.TimeToPtr(10 * time.Second), -// KillSignal: "SIGQUIT", -// LogConfig: &api.LogConfig{ -// MaxFiles: helper.IntToPtr(10), -// MaxFileSizeMB: helper.IntToPtr(100), -// }, -// Artifacts: []*api.TaskArtifact{ -// { -// GetterSource: helper.StringToPtr("source"), -// GetterOptions: map[string]string{ -// "a": "b", -// }, -// GetterMode: helper.StringToPtr("dir"), -// RelativeDest: helper.StringToPtr("dest"), -// }, -// }, -// Vault: &api.Vault{ -// Policies: []string{"a", "b", "c"}, -// Env: helper.BoolToPtr(true), -// ChangeMode: helper.StringToPtr("c"), -// ChangeSignal: helper.StringToPtr("sighup"), -// }, -// Templates: []*api.Template{ -// { -// SourcePath: helper.StringToPtr("source"), -// DestPath: helper.StringToPtr("dest"), -// EmbeddedTmpl: helper.StringToPtr("embedded"), -// ChangeMode: helper.StringToPtr("change"), -// ChangeSignal: helper.StringToPtr("signal"), -// Splay: helper.TimeToPtr(1 * time.Minute), -// Perms: helper.StringToPtr("666"), -// LeftDelim: helper.StringToPtr("abc"), -// RightDelim: helper.StringToPtr("def"), -// Envvars: helper.BoolToPtr(true), -// VaultGrace: helper.TimeToPtr(3 * time.Second), -// }, -// }, -// DispatchPayload: &api.DispatchPayloadConfig{ -// File: "fileA", -// }, -// }, -// }, -// }, -// }, -// VaultToken: helper.StringToPtr("token"), -// Status: helper.StringToPtr("status"), -// StatusDescription: helper.StringToPtr("status_desc"), -// Version: helper.Uint64ToPtr(10), -// CreateIndex: helper.Uint64ToPtr(1), -// ModifyIndex: helper.Uint64ToPtr(3), -// JobModifyIndex: helper.Uint64ToPtr(5), -// } -// -// expected := &structs.Job{ -// Stop: true, -// Region: "global", -// Namespace: "foo", -// ID: "foo", -// ParentID: "lol", -// Name: "name", -// Type: "service", -// Priority: 50, -// AllAtOnce: true, -// Datacenters: []string{"dc1", "dc2"}, -// Constraints: []*structs.Constraint{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// }, -// }, -// Affinities: []*structs.Affinity{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// Weight: 50, -// }, -// }, -// Spreads: []*structs.Spread{ -// { -// Attribute: "${meta.rack}", -// Weight: 100, -// SpreadTarget: []*structs.SpreadTarget{ -// { -// Value: "r1", -// Percent: 50, -// }, -// }, -// }, -// }, -// Update: structs.UpdateStrategy{ -// Stagger: 1 * time.Second, -// MaxParallel: 5, -// }, -// Periodic: &structs.PeriodicConfig{ -// Enabled: true, -// Spec: "spec", -// SpecType: "cron", -// ProhibitOverlap: true, -// TimeZone: "test zone", -// }, -// ParameterizedJob: &structs.ParameterizedJobConfig{ -// Payload: "payload", -// MetaRequired: []string{"a", "b"}, -// MetaOptional: []string{"c", "d"}, -// }, -// Payload: []byte("payload"), -// Meta: map[string]string{ -// "foo": "bar", -// }, -// TaskGroups: []*structs.TaskGroup{ -// { -// Name: "group1", -// Count: 5, -// Constraints: []*structs.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Affinities: []*structs.Affinity{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// Weight: 100, -// }, -// }, -// RestartPolicy: &structs.RestartPolicy{ -// Interval: 1 * time.Second, -// Attempts: 5, -// Delay: 10 * time.Second, -// Mode: "delay", -// }, -// Spreads: []*structs.Spread{ -// { -// Attribute: "${node.datacenter}", -// Weight: 100, -// SpreadTarget: []*structs.SpreadTarget{ -// { -// Value: "dc1", -// Percent: 100, -// }, -// }, -// }, -// }, -// ReschedulePolicy: &structs.ReschedulePolicy{ -// Interval: 12 * time.Hour, -// Attempts: 5, -// DelayFunction: "constant", -// Delay: 30 * time.Second, -// Unlimited: true, -// MaxDelay: 20 * time.Minute, -// }, -// Migrate: &structs.MigrateStrategy{ -// MaxParallel: 12, -// HealthCheck: "task_events", -// MinHealthyTime: 12 * time.Hour, -// HealthyDeadline: 12 * time.Hour, -// }, -// EphemeralDisk: &structs.EphemeralDisk{ -// SizeMB: 100, -// Sticky: true, -// Migrate: true, -// }, -// Update: &structs.UpdateStrategy{ -// Stagger: 1 * time.Second, -// MaxParallel: 5, -// HealthCheck: structs.UpdateStrategyHealthCheck_Checks, -// MinHealthyTime: 2 * time.Minute, -// HealthyDeadline: 5 * time.Minute, -// ProgressDeadline: 5 * time.Minute, -// AutoRevert: true, -// AutoPromote: false, -// Canary: 1, -// }, -// Meta: map[string]string{ -// "key": "value", -// }, -// Services: []*structs.Service{ -// { -// Name: "groupserviceA", -// Tags: []string{"a", "b"}, -// CanaryTags: []string{"d", "e"}, -// PortLabel: "1234", -// AddressMode: "auto", -// Meta: map[string]string{ -// "servicemeta": "foobar", -// }, -// Checks: []*structs.ServiceCheck{ -// { -// Name: "bar", -// Type: "http", -// Command: "foo", -// Args: []string{"a", "b"}, -// Path: "/check", -// Protocol: "http", -// PortLabel: "foo", -// AddressMode: "driver", -// GRPCService: "foo.Bar", -// GRPCUseTLS: true, -// Interval: 4 * time.Second, -// Timeout: 2 * time.Second, -// InitialStatus: "ok", -// CheckRestart: &structs.CheckRestart{ -// Grace: 11 * time.Second, -// Limit: 3, -// IgnoreWarnings: true, -// }, -// TaskName: "task1", -// }, -// }, -// Connect: &structs.ConsulConnect{ -// Native: false, -// SidecarService: &structs.ConsulSidecarService{ -// Tags: []string{"f", "g"}, -// Port: "9000", -// }, -// }, -// }, -// }, -// Tasks: []*structs.Task{ -// { -// Name: "task1", -// Driver: "docker", -// Leader: true, -// User: "mary", -// Config: map[string]interface{}{ -// "lol": "code", -// }, -// Constraints: []*structs.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Affinities: []*structs.Affinity{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// Weight: 50, -// }, -// }, -// Env: map[string]string{ -// "hello": "world", -// }, -// Services: []*structs.Service{ -// { -// Name: "serviceA", -// Tags: []string{"1", "2"}, -// CanaryTags: []string{"3", "4"}, -// PortLabel: "foo", -// AddressMode: "auto", -// Meta: map[string]string{ -// "servicemeta": "foobar", -// }, -// Checks: []*structs.ServiceCheck{ -// { -// Name: "bar", -// Type: "http", -// Command: "foo", -// Args: []string{"a", "b"}, -// Path: "/check", -// Protocol: "http", -// PortLabel: "foo", -// AddressMode: "driver", -// Interval: 4 * time.Second, -// Timeout: 2 * time.Second, -// InitialStatus: "ok", -// GRPCService: "foo.Bar", -// GRPCUseTLS: true, -// CheckRestart: &structs.CheckRestart{ -// Limit: 3, -// Grace: 11 * time.Second, -// IgnoreWarnings: true, -// }, -// }, -// { -// Name: "check2", -// Type: "tcp", -// PortLabel: "foo", -// Interval: 4 * time.Second, -// Timeout: 2 * time.Second, -// CheckRestart: &structs.CheckRestart{ -// Limit: 4, -// Grace: 11 * time.Second, -// }, -// }, -// }, -// }, -// }, -// Resources: &structs.Resources{ -// CPU: 100, -// MemoryMB: 10, -// Networks: []*structs.NetworkResource{ -// { -// IP: "10.10.11.1", -// MBits: 10, -// ReservedPorts: []structs.Port{ -// { -// Label: "http", -// Value: 80, -// }, -// }, -// DynamicPorts: []structs.Port{ -// { -// Label: "ssh", -// Value: 2000, -// }, -// }, -// }, -// }, -// Devices: []*structs.RequestedDevice{ -// { -// Name: "nvidia/gpu", -// Count: 4, -// Constraints: []*structs.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Affinities: []*structs.Affinity{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// Weight: 50, -// }, -// }, -// }, -// { -// Name: "gpu", -// Count: 1, -// }, -// }, -// }, -// Meta: map[string]string{ -// "lol": "code", -// }, -// KillTimeout: 10 * time.Second, -// KillSignal: "SIGQUIT", -// LogConfig: &structs.LogConfig{ -// MaxFiles: 10, -// MaxFileSizeMB: 100, -// }, -// Artifacts: []*structs.TaskArtifact{ -// { -// GetterSource: "source", -// GetterOptions: map[string]string{ -// "a": "b", -// }, -// GetterMode: "dir", -// RelativeDest: "dest", -// }, -// }, -// Vault: &structs.Vault{ -// Policies: []string{"a", "b", "c"}, -// Env: true, -// ChangeMode: "c", -// ChangeSignal: "sighup", -// }, -// Templates: []*structs.Template{ -// { -// SourcePath: "source", -// DestPath: "dest", -// EmbeddedTmpl: "embedded", -// ChangeMode: "change", -// ChangeSignal: "SIGNAL", -// Splay: 1 * time.Minute, -// Perms: "666", -// LeftDelim: "abc", -// RightDelim: "def", -// Envvars: true, -// VaultGrace: 3 * time.Second, -// }, -// }, -// DispatchPayload: &structs.DispatchPayloadConfig{ -// File: "fileA", -// }, -// }, -// }, -// }, -// }, -// -// VaultToken: "token", -// } -// -// structsJob := ApiJobToStructJob(apiJob) -// -// if diff := pretty.Diff(expected, structsJob); len(diff) > 0 { -// t.Fatalf("bad:\n%s", strings.Join(diff, "\n")) -// } -// -// systemAPIJob := &api.Job{ -// Stop: helper.BoolToPtr(true), -// Region: helper.StringToPtr("global"), -// Namespace: helper.StringToPtr("foo"), -// ID: helper.StringToPtr("foo"), -// ParentID: helper.StringToPtr("lol"), -// Name: helper.StringToPtr("name"), -// Type: helper.StringToPtr("system"), -// Priority: helper.IntToPtr(50), -// AllAtOnce: helper.BoolToPtr(true), -// Datacenters: []string{"dc1", "dc2"}, -// Constraints: []*api.Constraint{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// }, -// }, -// TaskGroups: []*api.TaskGroup{ -// { -// Name: helper.StringToPtr("group1"), -// Count: helper.IntToPtr(5), -// Constraints: []*api.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// RestartPolicy: &api.RestartPolicy{ -// Interval: helper.TimeToPtr(1 * time.Second), -// Attempts: helper.IntToPtr(5), -// Delay: helper.TimeToPtr(10 * time.Second), -// Mode: helper.StringToPtr("delay"), -// }, -// EphemeralDisk: &api.EphemeralDisk{ -// SizeMB: helper.IntToPtr(100), -// Sticky: helper.BoolToPtr(true), -// Migrate: helper.BoolToPtr(true), -// }, -// Meta: map[string]string{ -// "key": "value", -// }, -// Tasks: []*api.Task{ -// { -// Name: "task1", -// Leader: true, -// Driver: "docker", -// User: "mary", -// Config: map[string]interface{}{ -// "lol": "code", -// }, -// Env: map[string]string{ -// "hello": "world", -// }, -// Constraints: []*api.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Resources: &api.Resources{ -// CPU: helper.IntToPtr(100), -// MemoryMB: helper.IntToPtr(10), -// Networks: []*api.NetworkResource{ -// { -// IP: "10.10.11.1", -// MBits: helper.IntToPtr(10), -// ReservedPorts: []api.Port{ -// { -// Label: "http", -// Value: 80, -// }, -// }, -// DynamicPorts: []api.Port{ -// { -// Label: "ssh", -// Value: 2000, -// }, -// }, -// }, -// }, -// }, -// Meta: map[string]string{ -// "lol": "code", -// }, -// KillTimeout: helper.TimeToPtr(10 * time.Second), -// KillSignal: "SIGQUIT", -// LogConfig: &api.LogConfig{ -// MaxFiles: helper.IntToPtr(10), -// MaxFileSizeMB: helper.IntToPtr(100), -// }, -// Artifacts: []*api.TaskArtifact{ -// { -// GetterSource: helper.StringToPtr("source"), -// GetterOptions: map[string]string{ -// "a": "b", -// }, -// GetterMode: helper.StringToPtr("dir"), -// RelativeDest: helper.StringToPtr("dest"), -// }, -// }, -// DispatchPayload: &api.DispatchPayloadConfig{ -// File: "fileA", -// }, -// }, -// }, -// }, -// }, -// Status: helper.StringToPtr("status"), -// StatusDescription: helper.StringToPtr("status_desc"), -// Version: helper.Uint64ToPtr(10), -// CreateIndex: helper.Uint64ToPtr(1), -// ModifyIndex: helper.Uint64ToPtr(3), -// JobModifyIndex: helper.Uint64ToPtr(5), -// } -// -// expectedSystemJob := &structs.Job{ -// Stop: true, -// Region: "global", -// Namespace: "foo", -// ID: "foo", -// ParentID: "lol", -// Name: "name", -// Type: "system", -// Priority: 50, -// AllAtOnce: true, -// Datacenters: []string{"dc1", "dc2"}, -// Constraints: []*structs.Constraint{ -// { -// LTarget: "a", -// RTarget: "b", -// Operand: "c", -// }, -// }, -// TaskGroups: []*structs.TaskGroup{ -// { -// Name: "group1", -// Count: 5, -// Constraints: []*structs.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// RestartPolicy: &structs.RestartPolicy{ -// Interval: 1 * time.Second, -// Attempts: 5, -// Delay: 10 * time.Second, -// Mode: "delay", -// }, -// EphemeralDisk: &structs.EphemeralDisk{ -// SizeMB: 100, -// Sticky: true, -// Migrate: true, -// }, -// Meta: map[string]string{ -// "key": "value", -// }, -// Tasks: []*structs.Task{ -// { -// Name: "task1", -// Driver: "docker", -// Leader: true, -// User: "mary", -// Config: map[string]interface{}{ -// "lol": "code", -// }, -// Constraints: []*structs.Constraint{ -// { -// LTarget: "x", -// RTarget: "y", -// Operand: "z", -// }, -// }, -// Env: map[string]string{ -// "hello": "world", -// }, -// Resources: &structs.Resources{ -// CPU: 100, -// MemoryMB: 10, -// Networks: []*structs.NetworkResource{ -// { -// IP: "10.10.11.1", -// MBits: 10, -// ReservedPorts: []structs.Port{ -// { -// Label: "http", -// Value: 80, -// }, -// }, -// DynamicPorts: []structs.Port{ -// { -// Label: "ssh", -// Value: 2000, -// }, -// }, -// }, -// }, -// }, -// Meta: map[string]string{ -// "lol": "code", -// }, -// KillTimeout: 10 * time.Second, -// KillSignal: "SIGQUIT", -// LogConfig: &structs.LogConfig{ -// MaxFiles: 10, -// MaxFileSizeMB: 100, -// }, -// Artifacts: []*structs.TaskArtifact{ -// { -// GetterSource: "source", -// GetterOptions: map[string]string{ -// "a": "b", -// }, -// GetterMode: "dir", -// RelativeDest: "dest", -// }, -// }, -// DispatchPayload: &structs.DispatchPayloadConfig{ -// File: "fileA", -// }, -// }, -// }, -// }, -// }, -// } -// -// systemStructsJob := ApiJobToStructJob(systemAPIJob) -// -// if diff := pretty.Diff(expectedSystemJob, systemStructsJob); len(diff) > 0 { -// t.Fatalf("bad:\n%s", strings.Join(diff, "\n")) -// } -// } -// -// func TestJobs_ApiJobToStructsJobUpdate(t *testing.T) { -// apiJob := &api.Job{ -// Update: &api.UpdateStrategy{ -// Stagger: helper.TimeToPtr(1 * time.Second), -// MaxParallel: helper.IntToPtr(5), -// HealthCheck: helper.StringToPtr(structs.UpdateStrategyHealthCheck_Manual), -// MinHealthyTime: helper.TimeToPtr(1 * time.Minute), -// HealthyDeadline: helper.TimeToPtr(3 * time.Minute), -// ProgressDeadline: helper.TimeToPtr(3 * time.Minute), -// AutoRevert: helper.BoolToPtr(false), -// AutoPromote: nil, -// Canary: helper.IntToPtr(1), -// }, -// TaskGroups: []*api.TaskGroup{ -// { -// Update: &api.UpdateStrategy{ -// Canary: helper.IntToPtr(2), -// AutoRevert: helper.BoolToPtr(true), -// }, -// }, { -// Update: &api.UpdateStrategy{ -// Canary: helper.IntToPtr(3), -// AutoPromote: helper.BoolToPtr(true), -// }, -// }, -// }, -// } -// -// structsJob := ApiJobToStructJob(apiJob) -// -// // Update has been moved from job down to the groups -// jobUpdate := structs.UpdateStrategy{ -// Stagger: 1000000000, -// MaxParallel: 5, -// HealthCheck: "", -// MinHealthyTime: 0, -// HealthyDeadline: 0, -// ProgressDeadline: 0, -// AutoRevert: false, -// AutoPromote: false, -// Canary: 0, -// } -// -// // But the groups inherit settings from the job update -// group1 := structs.UpdateStrategy{ -// Stagger: 1000000000, -// MaxParallel: 5, -// HealthCheck: "manual", -// MinHealthyTime: 60000000000, -// HealthyDeadline: 180000000000, -// ProgressDeadline: 180000000000, -// AutoRevert: true, -// AutoPromote: false, -// Canary: 2, -// } -// -// group2 := structs.UpdateStrategy{ -// Stagger: 1000000000, -// MaxParallel: 5, -// HealthCheck: "manual", -// MinHealthyTime: 60000000000, -// HealthyDeadline: 180000000000, -// ProgressDeadline: 180000000000, -// AutoRevert: false, -// AutoPromote: true, -// Canary: 3, -// } -// -// require.Equal(t, jobUpdate, structsJob.Update) -// require.Equal(t, group1, *structsJob.TaskGroups[0].Update) -// require.Equal(t, group2, *structsJob.TaskGroups[1].Update) -// } -// -// // TestHTTP_JobValidate_SystemMigrate asserts that a system job with a migrate -// // stanza fails to validate but does not panic (see #5477). -// func TestHTTP_JobValidate_SystemMigrate(t *testing.T) { -// t.Parallel() -// httpTest(t, nil, func(s *TestAgent) { -// // Create the job -// job := &api.Job{ -// Region: helper.StringToPtr("global"), -// Datacenters: []string{"dc1"}, -// ID: helper.StringToPtr("systemmigrate"), -// Name: helper.StringToPtr("systemmigrate"), -// TaskGroups: []*api.TaskGroup{ -// {Name: helper.StringToPtr("web")}, -// }, -// -// // System job... -// Type: helper.StringToPtr("system"), -// -// // ...with an empty migrate stanza -// Migrate: &api.MigrateStrategy{}, -// } -// -// args := api.JobValidateRequest{ -// Job: job, -// WriteRequest: api.WriteRequest{Region: "global"}, -// } -// buf := encodeReq(args) -// -// // Make the HTTP request -// req, err := http.NewRequest("PUT", "/v1/validate/job", buf) -// require.NoError(t, err) -// respW := httptest.NewRecorder() -// -// // Make the request -// obj, err := s.Server.ValidateJobRequest(respW, req) -// require.NoError(t, err) -// -// // Check the response -// resp := obj.(structs.JobValidateResponse) -// require.Contains(t, resp.Error, `Job type "system" does not allow migrate block`) -// }) -// } From 41560096d0d717881c1146e986a3c650a5ce6792 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sat, 21 Mar 2020 15:20:25 +0000 Subject: [PATCH 23/30] scaling api: put api.* objects in agreement with structs.* objects --- api/scaling.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/api/scaling.go b/api/scaling.go index e2323edfa51..9e71757113b 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -79,11 +79,12 @@ type JobScaleStatusResponse struct { } type TaskGroupScaleStatus struct { - Desired uint64 - Running uint64 - Started uint64 - Healthy uint64 - Events []ScalingEvent + Desired int + Placed int + Running int + Healthy int + Unhealthy int + Events []ScalingEvent } type ScalingEvent struct { From 9292e88f2b931b2ee9680c5717f7124aba3979fc Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sun, 22 Mar 2020 11:54:04 +0000 Subject: [PATCH 24/30] changes to Canonicalize, Validate, and api->struct conversion so that tg.Count, tg.Scaling.Min/Max are well-defined with reasonable defaults. - tg.Count defaults to tg.Scaling.Min if present (falls back on previous default of 1 if Scaling is absent) - Validate() enforces tg.Scaling.Min <= tg.Count <= tg.Scaling.Max modification in ApiScalingPolicyToStructs, api.TaskGroup.Validate so that defaults are handled for TaskGroup.Count and --- api/scaling.go | 14 +++++++-- api/scaling_test.go | 6 +++- api/tasks.go | 9 ++++-- api/tasks_test.go | 48 ++++++++++++++++++++++++++++++ api/util_test.go | 2 ++ command/agent/job_endpoint.go | 2 +- command/agent/scaling_endpoint.go | 9 ++++-- nomad/mock/mock.go | 2 ++ nomad/structs/structs.go | 33 +++++++++++++++++++++ nomad/structs/structs_test.go | 49 +++++++++++++++++++++++++++++++ 10 files changed, 165 insertions(+), 9 deletions(-) diff --git a/api/scaling.go b/api/scaling.go index 9e71757113b..298a651aec6 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -28,10 +28,20 @@ func (s *Scaling) GetPolicy(ID string, q *QueryOptions) (*ScalingPolicy, *QueryM return &policy, qm, nil } -func (p *ScalingPolicy) Canonicalize() { +func (p *ScalingPolicy) Canonicalize(tg *TaskGroup) { if p.Enabled == nil { p.Enabled = boolToPtr(true) } + if p.Min == nil { + var m int64 + if tg.Count != nil { + m = int64(*tg.Count) + } else { + // this should not be at this point, but safeguard here just in case + m = 0 + } + p.Min = &m + } } // ScalingRequest is the payload for a generic scaling action @@ -51,7 +61,7 @@ type ScalingPolicy struct { ID string Namespace string Target map[string]string - Min int64 + Min *int64 Max int64 Policy map[string]interface{} Enabled *bool diff --git a/api/scaling_test.go b/api/scaling_test.go index 6b79624141e..53238af9d21 100644 --- a/api/scaling_test.go +++ b/api/scaling_test.go @@ -22,7 +22,9 @@ func TestScalingPolicies_ListPolicies(t *testing.T) { // Register a job with a scaling policy job := testJob() - job.TaskGroups[0].Scaling = &ScalingPolicy{} + job.TaskGroups[0].Scaling = &ScalingPolicy{ + Max: 100, + } _, _, err = jobs.Register(job, nil) require.NoError(err) @@ -72,6 +74,8 @@ func TestScalingPolicies_GetPolicy(t *testing.T) { job := testJob() policy := &ScalingPolicy{ Enabled: boolToPtr(true), + Min: int64ToPtr(1), + Max: 1, Policy: map[string]interface{}{ "key": "value", }, diff --git a/api/tasks.go b/api/tasks.go index f7ee42a78b9..d18d8a0b1ea 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -444,7 +444,11 @@ func (g *TaskGroup) Canonicalize(job *Job) { g.Name = stringToPtr("") } if g.Count == nil { - g.Count = intToPtr(1) + if g.Scaling != nil && g.Scaling.Min != nil { + g.Count = intToPtr(int(*g.Scaling.Min)) + } else { + g.Count = intToPtr(1) + } } for _, t := range g.Tasks { t.Canonicalize(g, job) @@ -544,8 +548,9 @@ func (g *TaskGroup) Canonicalize(job *Job) { for _, s := range g.Services { s.Canonicalize(nil, g, job) } + if g.Scaling != nil { - g.Scaling.Canonicalize() + g.Scaling.Canonicalize(g) } } diff --git a/api/tasks_test.go b/api/tasks_test.go index 80acd37fe0d..8217aee1fe3 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -432,6 +432,54 @@ func TestTaskGroup_Canonicalize_Update(t *testing.T) { assert.Nil(t, tg.Update) } +func TestTaskGroup_Canonicalize_Scaling(t *testing.T) { + require := require.New(t) + + job := &Job{ + ID: stringToPtr("test"), + } + job.Canonicalize() + tg := &TaskGroup{ + Name: stringToPtr("foo"), + Count: nil, + Scaling: &ScalingPolicy{ + Min: nil, + Max: 10, + Policy: nil, + Enabled: nil, + CreateIndex: 0, + ModifyIndex: 0, + }, + } + job.TaskGroups = []*TaskGroup{tg} + + // both nil => both == 1 + tg.Canonicalize(job) + require.NotNil(tg.Count) + require.NotNil(tg.Scaling.Min) + require.Equal(1, *tg.Count) + require.Equal(int64(*tg.Count), *tg.Scaling.Min) + + tg.Count = nil + tg.Scaling.Min = int64ToPtr(5) + // count == nil => count = Scaling.Min + tg.Canonicalize(job) + require.NotNil(tg.Count) + require.NotNil(tg.Scaling.Min) + require.Equal(5, *tg.Count) + require.Equal(int64(*tg.Count), *tg.Scaling.Min) + + // Scaling.Min == nil => Scaling.Min == count + tg.Count = intToPtr(5) + tg.Scaling.Min = nil + // count == nil => count = Scaling.Min + tg.Canonicalize(job) + require.NotNil(tg.Count) + require.NotNil(tg.Scaling.Min) + require.Equal(int64(5), *tg.Scaling.Min) + require.Equal(*tg.Scaling.Min, int64(*tg.Count)) +} + func TestTaskGroup_Merge_Update(t *testing.T) { job := &Job{ ID: stringToPtr("test"), diff --git a/api/util_test.go b/api/util_test.go index f32d6d30d95..c796766e32c 100644 --- a/api/util_test.go +++ b/api/util_test.go @@ -52,6 +52,8 @@ func testJobWithScalingPolicy() *Job { job := testJob() job.TaskGroups[0].Scaling = &ScalingPolicy{ Policy: map[string]interface{}{}, + Min: int64ToPtr(1), + Max: 1, Enabled: boolToPtr(true), } return job diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 65196a6edd1..e9524e6dcc0 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -814,7 +814,7 @@ func ApiTgToStructsTG(job *structs.Job, taskGroup *api.TaskGroup, tg *structs.Ta } if taskGroup.Scaling != nil { - tg.Scaling = ApiScalingPolicyToStructs(taskGroup.Scaling).TargetTaskGroup(job, tg) + tg.Scaling = ApiScalingPolicyToStructs(tg.Count, taskGroup.Scaling).TargetTaskGroup(job, tg) } tg.EphemeralDisk = &structs.EphemeralDisk{ diff --git a/command/agent/scaling_endpoint.go b/command/agent/scaling_endpoint.go index 04a80670060..fe94d4dbcda 100644 --- a/command/agent/scaling_endpoint.go +++ b/command/agent/scaling_endpoint.go @@ -75,12 +75,15 @@ func (s *HTTPServer) scalingPolicyQuery(resp http.ResponseWriter, req *http.Requ return out.Policy, nil } -func ApiScalingPolicyToStructs(ap *api.ScalingPolicy) *structs.ScalingPolicy { - return &structs.ScalingPolicy{ +func ApiScalingPolicyToStructs(count int, ap *api.ScalingPolicy) *structs.ScalingPolicy { + p := structs.ScalingPolicy{ Enabled: *ap.Enabled, - Min: ap.Min, Max: ap.Max, Policy: ap.Policy, Target: map[string]string{}, } + if ap.Min == nil { + p.Min = int64(count) + } + return &p } diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index a52838ff99a..ccecb6d658e 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -1275,6 +1275,8 @@ func JobWithScalingPolicy() (*structs.Job, *structs.ScalingPolicy) { job := Job() policy := &structs.ScalingPolicy{ ID: uuid.Generate(), + Min: int64(job.TaskGroups[0].Count), + Max: int64(job.TaskGroups[0].Count), Policy: map[string]interface{}{}, Enabled: true, } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 9fff0958138..36fdd5eacd3 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -5373,6 +5373,12 @@ func (tg *TaskGroup) Validate(j *Job) error { mErr.Errors = append(mErr.Errors, outer) } + // Validate the scaling policy + if err := tg.validateScalingPolicy(); err != nil { + outer := fmt.Errorf("Task group scaling policy validation failed: %v", err) + mErr.Errors = append(mErr.Errors, outer) + } + // Validate the tasks for _, task := range tg.Tasks { // Validate the task does not reference undefined volume mounts @@ -5524,6 +5530,33 @@ func (tg *TaskGroup) validateServices() error { return mErr.ErrorOrNil() } +// validateScalingPolicy ensures that the scaling policy has consistent +// min and max, not in conflict with the task group count +func (tg *TaskGroup) validateScalingPolicy() error { + if tg.Scaling == nil { + return nil + } + + var mErr multierror.Error + + if tg.Scaling.Min > tg.Scaling.Max { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("Scaling policy invalid: maximum count must not be less than minimum count")) + } + + if int64(tg.Count) < tg.Scaling.Min { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("Scaling policy invalid: task group count must not be less than minimum count in scaling policy")) + } + + if tg.Scaling.Max < int64(tg.Count) { + mErr.Errors = append(mErr.Errors, + fmt.Errorf("Scaling policy invalid: task group count must not be greater than maximum count in scaling policy")) + } + + return mErr.ErrorOrNil() +} + // Warnings returns a list of warnings that may be from dubious settings or // deprecation warnings. func (tg *TaskGroup) Warnings(j *Job) error { diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index cba53774d4d..a1c999d5e78 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper/uuid" + "github.com/kr/pretty" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -122,6 +123,54 @@ func TestJob_Validate(t *testing.T) { } } +func TestJob_ValidateScaling(t *testing.T) { + require := require.New(t) + + p := &ScalingPolicy{ + Policy: nil, // allowed to be nil + Min: 5, + Max: 5, + Enabled: true, + } + job := testJob() + job.TaskGroups[0].Scaling = p + job.TaskGroups[0].Count = 5 + + require.NoError(job.Validate()) + + // min <= max + p.Max = 0 + p.Min = 10 + err := job.Validate() + require.Error(err) + mErr := err.(*multierror.Error) + require.Len(mErr.Errors, 1) + require.Contains(mErr.Errors[0].Error(), "maximum count must not be less than minimum count") + require.Contains(mErr.Errors[0].Error(), "task group count must not be less than minimum count in scaling policy") + require.Contains(mErr.Errors[0].Error(), "task group count must not be greater than maximum count in scaling policy") + + // count <= max + p.Max = 0 + p.Min = 5 + job.TaskGroups[0].Count = 5 + err = job.Validate() + require.Error(err) + mErr = err.(*multierror.Error) + require.Len(mErr.Errors, 1) + require.Contains(mErr.Errors[0].Error(), "maximum count must not be less than minimum count") + require.Contains(mErr.Errors[0].Error(), "task group count must not be greater than maximum count in scaling policy") + + // min <= count + job.TaskGroups[0].Count = 0 + p.Min = 5 + p.Max = 5 + err = job.Validate() + require.Error(err) + mErr = err.(*multierror.Error) + require.Len(mErr.Errors, 1) + require.Contains(mErr.Errors[0].Error(), "task group count must not be less than minimum count in scaling policy") +} + func TestJob_Warnings(t *testing.T) { cases := []struct { Name string From f704a49962e41336b49acd184320dc7825ee8bff Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sun, 22 Mar 2020 14:21:51 +0000 Subject: [PATCH 25/30] added new ACL capabilities related to autoscaling: - read-job-scaling - scale-job - list-scaling-policies - read-scaling-policy updated the read and right policy dispositions, added the new autoscaler disposition --- acl/policy.go | 62 +++++++++++++++++++++++++++++----------------- acl/policy_test.go | 23 +++++++++++++++++ 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/acl/policy.go b/acl/policy.go index cce0d9c47ee..887ee95a866 100644 --- a/acl/policy.go +++ b/acl/policy.go @@ -11,10 +11,11 @@ const ( // The following levels are the only valid values for the `policy = "read"` stanza. // When policies are merged together, the most privilege is granted, except for deny // which always takes precedence and supercedes. - PolicyDeny = "deny" - PolicyRead = "read" - PolicyList = "list" - PolicyWrite = "write" + PolicyDeny = "deny" + PolicyRead = "read" + PolicyList = "list" + PolicyWrite = "write" + PolicyAutoscaler = "autoscaler" ) const ( @@ -23,23 +24,26 @@ const ( // combined we take the union of all capabilities. If the deny capability is present, it // takes precedence and overwrites all other capabilities. - NamespaceCapabilityDeny = "deny" - NamespaceCapabilityListJobs = "list-jobs" - NamespaceCapabilityReadJob = "read-job" - NamespaceCapabilityScaleJob = "scale-job" - NamespaceCapabilitySubmitJob = "submit-job" - NamespaceCapabilityDispatchJob = "dispatch-job" - NamespaceCapabilityReadLogs = "read-logs" - NamespaceCapabilityReadFS = "read-fs" - NamespaceCapabilityAllocExec = "alloc-exec" - NamespaceCapabilityAllocNodeExec = "alloc-node-exec" - NamespaceCapabilityAllocLifecycle = "alloc-lifecycle" - NamespaceCapabilitySentinelOverride = "sentinel-override" - NamespaceCapabilityCSIRegisterPlugin = "csi-register-plugin" - NamespaceCapabilityCSIWriteVolume = "csi-write-volume" - NamespaceCapabilityCSIReadVolume = "csi-read-volume" - NamespaceCapabilityCSIListVolume = "csi-list-volume" - NamespaceCapabilityCSIMountVolume = "csi-mount-volume" + NamespaceCapabilityDeny = "deny" + NamespaceCapabilityListJobs = "list-jobs" + NamespaceCapabilityReadJob = "read-job" + NamespaceCapabilitySubmitJob = "submit-job" + NamespaceCapabilityDispatchJob = "dispatch-job" + NamespaceCapabilityReadLogs = "read-logs" + NamespaceCapabilityReadFS = "read-fs" + NamespaceCapabilityAllocExec = "alloc-exec" + NamespaceCapabilityAllocNodeExec = "alloc-node-exec" + NamespaceCapabilityAllocLifecycle = "alloc-lifecycle" + NamespaceCapabilitySentinelOverride = "sentinel-override" + NamespaceCapabilityCSIRegisterPlugin = "csi-register-plugin" + NamespaceCapabilityCSIWriteVolume = "csi-write-volume" + NamespaceCapabilityCSIReadVolume = "csi-read-volume" + NamespaceCapabilityCSIListVolume = "csi-list-volume" + NamespaceCapabilityCSIMountVolume = "csi-mount-volume" + NamespaceCapabilityListScalingPolicies = "list-scaling-policies" + NamespaceCapabilityReadScalingPolicy = "read-scaling-policy" + NamespaceCapabilityReadJobScaling = "read-job-scaling" + NamespaceCapabilityScaleJob = "scale-job" ) var ( @@ -122,7 +126,7 @@ type PluginPolicy struct { // isPolicyValid makes sure the given string matches one of the valid policies. func isPolicyValid(policy string) bool { switch policy { - case PolicyDeny, PolicyRead, PolicyWrite: + case PolicyDeny, PolicyRead, PolicyWrite, PolicyAutoscaler: return true default: return false @@ -145,7 +149,8 @@ func isNamespaceCapabilityValid(cap string) bool { NamespaceCapabilitySubmitJob, NamespaceCapabilityDispatchJob, NamespaceCapabilityReadLogs, NamespaceCapabilityReadFS, NamespaceCapabilityAllocLifecycle, NamespaceCapabilityAllocExec, NamespaceCapabilityAllocNodeExec, - NamespaceCapabilityCSIReadVolume, NamespaceCapabilityCSIWriteVolume, NamespaceCapabilityCSIListVolume, NamespaceCapabilityCSIMountVolume, NamespaceCapabilityCSIRegisterPlugin: + NamespaceCapabilityCSIReadVolume, NamespaceCapabilityCSIWriteVolume, NamespaceCapabilityCSIListVolume, NamespaceCapabilityCSIMountVolume, NamespaceCapabilityCSIRegisterPlugin, + NamespaceCapabilityListScalingPolicies, NamespaceCapabilityReadScalingPolicy, NamespaceCapabilityReadJobScaling, NamespaceCapabilityScaleJob: return true // Separate the enterprise-only capabilities case NamespaceCapabilitySentinelOverride: @@ -163,9 +168,13 @@ func expandNamespacePolicy(policy string) []string { NamespaceCapabilityReadJob, NamespaceCapabilityCSIListVolume, NamespaceCapabilityCSIReadVolume, + NamespaceCapabilityReadJobScaling, + NamespaceCapabilityListScalingPolicies, + NamespaceCapabilityReadScalingPolicy, } write := append(read, []string{ + NamespaceCapabilityScaleJob, NamespaceCapabilitySubmitJob, NamespaceCapabilityDispatchJob, NamespaceCapabilityReadLogs, @@ -183,6 +192,13 @@ func expandNamespacePolicy(policy string) []string { return read case PolicyWrite: return write + case PolicyAutoscaler: + return []string{ + NamespaceCapabilityListScalingPolicies, + NamespaceCapabilityReadScalingPolicy, + NamespaceCapabilityReadJobScaling, + NamespaceCapabilityScaleJob, + } default: return nil } diff --git a/acl/policy_test.go b/acl/policy_test.go index d8d21ac8140..aff25356f87 100644 --- a/acl/policy_test.go +++ b/acl/policy_test.go @@ -32,6 +32,9 @@ func TestParse(t *testing.T) { NamespaceCapabilityReadJob, NamespaceCapabilityCSIListVolume, NamespaceCapabilityCSIReadVolume, + NamespaceCapabilityReadJobScaling, + NamespaceCapabilityListScalingPolicies, + NamespaceCapabilityReadScalingPolicy, }, }, }, @@ -48,6 +51,9 @@ func TestParse(t *testing.T) { namespace "secret" { capabilities = ["deny", "read-logs"] } + namespace "autoscaler" { + policy = "autoscaler" + } agent { policy = "read" } @@ -75,6 +81,9 @@ func TestParse(t *testing.T) { NamespaceCapabilityReadJob, NamespaceCapabilityCSIListVolume, NamespaceCapabilityCSIReadVolume, + NamespaceCapabilityReadJobScaling, + NamespaceCapabilityListScalingPolicies, + NamespaceCapabilityReadScalingPolicy, }, }, { @@ -85,6 +94,10 @@ func TestParse(t *testing.T) { NamespaceCapabilityReadJob, NamespaceCapabilityCSIListVolume, NamespaceCapabilityCSIReadVolume, + NamespaceCapabilityReadJobScaling, + NamespaceCapabilityListScalingPolicies, + NamespaceCapabilityReadScalingPolicy, + NamespaceCapabilityScaleJob, NamespaceCapabilitySubmitJob, NamespaceCapabilityDispatchJob, NamespaceCapabilityReadLogs, @@ -102,6 +115,16 @@ func TestParse(t *testing.T) { NamespaceCapabilityReadLogs, }, }, + { + Name: "autoscaler", + Policy: PolicyAutoscaler, + Capabilities: []string{ + NamespaceCapabilityListScalingPolicies, + NamespaceCapabilityReadScalingPolicy, + NamespaceCapabilityReadJobScaling, + NamespaceCapabilityScaleJob, + }, + }, }, Agent: &AgentPolicy{ Policy: PolicyRead, From ac5a166ca36d47985ef1b1adaf49056413c5affb Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sun, 22 Mar 2020 15:40:39 +0000 Subject: [PATCH 26/30] wip: ACL checking for RPC Job.ScaleStatus --- nomad/job_endpoint.go | 17 +++++--- nomad/job_endpoint_test.go | 89 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index e5f45b96b77..d973c913709 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -1698,13 +1698,16 @@ func (j *Job) ScaleStatus(args *structs.JobScaleStatusRequest, } defer metrics.MeasureSince([]string{"nomad", "job", "scale_status"}, time.Now()) - // FINISH - // Check for job-autoscaler permissions - // if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { - // return err - // } else if aclObj != nil && !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) { - // return structs.ErrPermissionDenied - // } + // Check for autoscaler permissions + if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if aclObj != nil { + hasReadJob := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) + hasReadJobScaling := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJobScaling) + if !(hasReadJob || hasReadJobScaling) { + return structs.ErrPermissionDenied + } + } // Setup the blocking query opts := blockingOptions{ diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index e32271ce064..4f7e35ad62f 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -5332,3 +5332,92 @@ func TestJobEndpoint_GetScaleStatus(t *testing.T) { require.True(reflect.DeepEqual(*resp2.JobScaleStatus, expectedStatus)) } + +func TestJobEndpoint_GetScaleStatus_ACL(t *testing.T) { + t.Parallel() + require := require.New(t) + + s1, root, cleanupS1 := TestACLServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + state := s1.fsm.State() + + // Create the job + job := mock.Job() + err := state.UpsertJob(1000, job) + require.Nil(err) + + // Get the job scale status + get := &structs.JobScaleStatusRequest{ + JobID: job.ID, + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: job.Namespace, + }, + } + + // Get without a token should fail + var resp structs.JobScaleStatusResponse + err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp) + require.NotNil(err) + require.Contains(err.Error(), "Permission denied") + + // Expect failure for request with an invalid token + invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) + + get.AuthToken = invalidToken.SecretID + var invalidResp structs.JobScaleStatusResponse + require.NotNil(err) + err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &invalidResp) + require.Contains(err.Error(), "Permission denied") + + type testCase struct { + authToken string + name string + } + cases := []testCase{ + { + name: "mgmt token should succeed", + authToken: root.SecretID, + }, + { + name: "read disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read", + mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)). + SecretID, + }, + { + name: "write disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-write", + mock.NamespacePolicy(structs.DefaultNamespace, "write", nil)). + SecretID, + }, + { + name: "autoscaler disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-autoscaler", + mock.NamespacePolicy(structs.DefaultNamespace, "autoscaler", nil)). + SecretID, + }, + { + name: "read-job capability should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read-job", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJob})).SecretID, + }, + { + name: "read-job-scaling capability should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read-job-scaling", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityReadJobScaling})). + SecretID, + }, + } + + for _, tc := range cases { + get.AuthToken = tc.authToken + var validResp structs.JobScaleStatusResponse + err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &validResp) + require.NoError(err, tc.name) + require.NotNil(validResp.JobScaleStatus) + } +} From d4f967cdd497098f5186f0f36c7fbf853d1ba221 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Sun, 22 Mar 2020 21:49:09 +0000 Subject: [PATCH 27/30] made count optional during job scaling actions added ACL protection in Job.Scale in Job.Scale, only perform a Job.Register if the Count was non-nil --- api/jobs.go | 10 +- api/jobs_test.go | 8 +- api/scaling.go | 2 +- command/agent/job_endpoint_test.go | 2 +- nomad/job_endpoint.go | 70 ++++++---- nomad/job_endpoint_test.go | 210 ++++++++++++++++++++++++++--- nomad/scaling_endpoint_test.go | 58 ++++---- nomad/structs/structs.go | 2 +- 8 files changed, 277 insertions(+), 85 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index 843d85a9779..ab90cf5e8d7 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -8,6 +8,8 @@ import ( "time" "github.com/gorhill/cronexpr" + + "github.com/hashicorp/nomad/helper" ) const ( @@ -155,10 +157,14 @@ func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) { // Scale is used to retrieve information about a particular // job given its unique ID. -func (j *Jobs) Scale(jobID, group string, count int, +func (j *Jobs) Scale(jobID, group string, count *int, reason, error *string, meta map[string]interface{}, q *WriteOptions) (*JobRegisterResponse, *WriteMeta, error) { + var count64 *int64 + if count != nil { + count64 = helper.Int64ToPtr(int64(*count)) + } req := &ScalingRequest{ - Count: int64(count), + Count: count64, Target: map[string]string{ "Job": jobID, "Group": group, diff --git a/api/jobs_test.go b/api/jobs_test.go index dc2d3a858e2..90a4d64348d 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -930,7 +930,7 @@ func TestJobs_ScaleInvalidAction(t *testing.T) { {"i-dont-exist", "me-neither", 1, "404"}, } for _, test := range tests { - _, _, err := jobs.Scale(test.jobID, test.group, test.value, stringToPtr("reason"), nil, nil, nil) + _, _, err := jobs.Scale(test.jobID, test.group, &test.value, stringToPtr("reason"), nil, nil, nil) require.Errorf(err, "expected jobs.Scale(%s, %s) to fail", test.jobID, test.group) require.Containsf(err.Error(), test.want, "jobs.Scale(%s, %s) error doesn't contain %s, got: %s", test.jobID, test.group, test.want, err) } @@ -943,7 +943,7 @@ func TestJobs_ScaleInvalidAction(t *testing.T) { assertWriteMeta(t, wm) // Perform a scaling action with bad group name, verify error - _, _, err = jobs.Scale(*job.ID, "incorrect-group-name", 2, + _, _, err = jobs.Scale(*job.ID, "incorrect-group-name", intToPtr(2), stringToPtr("because"), nil, nil, nil) require.Error(err) require.Contains(err.Error(), "does not exist") @@ -1590,7 +1590,7 @@ func TestJobs_ScaleAction(t *testing.T) { groupCount := *job.TaskGroups[0].Count // Trying to scale against a target before it exists returns an error - _, _, err := jobs.Scale(id, "missing", groupCount+1, stringToPtr("this won't work"), nil, nil, nil) + _, _, err := jobs.Scale(id, "missing", intToPtr(groupCount+1), stringToPtr("this won't work"), nil, nil, nil) require.Error(err) require.Contains(err.Error(), "not found") @@ -1602,7 +1602,7 @@ func TestJobs_ScaleAction(t *testing.T) { // Perform scaling action newCount := groupCount + 1 resp1, wm, err := jobs.Scale(id, groupName, - newCount, stringToPtr("need more instances"), nil, nil, nil) + intToPtr(newCount), stringToPtr("need more instances"), nil, nil, nil) require.NoError(err) require.NotNil(resp1) diff --git a/api/scaling.go b/api/scaling.go index 298a651aec6..7f872cf3666 100644 --- a/api/scaling.go +++ b/api/scaling.go @@ -46,7 +46,7 @@ func (p *ScalingPolicy) Canonicalize(tg *TaskGroup) { // ScalingRequest is the payload for a generic scaling action type ScalingRequest struct { - Count int64 + Count *int64 Target map[string]string Reason *string Error *string diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index e7934681f9b..935622b2016 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -686,7 +686,7 @@ func TestHTTP_Job_ScaleTaskGroup(t *testing.T) { newCount := job.TaskGroups[0].Count + 1 scaleReq := &api.ScalingRequest{ - Count: int64(newCount), + Count: helper.Int64ToPtr(int64(newCount)), Reason: helper.StringToPtr("testing"), Target: map[string]string{ "Job": job.ID, diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index d973c913709..86ad10bcccb 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -862,12 +862,22 @@ func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterRes if groupName == "" { return structs.NewErrRPCCoded(400, "missing task group name for scaling action") } + if args.Error != nil && args.Reason != nil { + return structs.NewErrRPCCoded(400, "scaling action should not contain error and scaling reason") + } + if args.Error != nil && args.Count != nil { + return structs.NewErrRPCCoded(400, "scaling action should not contain error and count") + } // Check for submit-job permissions if aclObj, err := j.srv.ResolveToken(args.AuthToken); err != nil { return err - } else if aclObj != nil && !aclObj.AllowNsOp(namespace, acl.NamespaceCapabilityScaleJob) { - return structs.ErrPermissionDenied + } else if aclObj != nil { + hasScaleJob := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityScaleJob) + hasSubmitJob := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) + if !(hasScaleJob || hasSubmitJob) { + return structs.ErrPermissionDenied + } } // Lookup the job @@ -884,39 +894,43 @@ func (j *Job) Scale(args *structs.JobScaleRequest, reply *structs.JobRegisterRes return structs.NewErrRPCCoded(404, fmt.Sprintf("job %q not found", args.JobID)) } - found := false - for _, tg := range job.TaskGroups { - if groupName == tg.Name { - tg.Count = int(args.Count) // TODO: not safe, check this above - found = true - break + index := job.ModifyIndex + if args.Count != nil { + found := false + for _, tg := range job.TaskGroups { + if groupName == tg.Name { + tg.Count = int(*args.Count) // TODO: not safe, check this above + found = true + break + } + } + if !found { + return structs.NewErrRPCCoded(400, + fmt.Sprintf("task group %q specified for scaling does not exist in job", groupName)) + } + registerReq := structs.JobRegisterRequest{ + Job: job, + EnforceIndex: true, + JobModifyIndex: job.ModifyIndex, + PolicyOverride: args.PolicyOverride, + WriteRequest: args.WriteRequest, } - } - if !found { - return structs.NewErrRPCCoded(400, - fmt.Sprintf("task group %q specified for scaling does not exist in job", groupName)) - } - registerReq := structs.JobRegisterRequest{ - Job: job, - EnforceIndex: true, - JobModifyIndex: job.ModifyIndex, - PolicyOverride: args.PolicyOverride, - WriteRequest: args.WriteRequest, - } - // Commit this update via Raft - _, index, err := j.srv.raftApply(structs.JobRegisterRequestType, registerReq) - if err != nil { - j.logger.Error("job register for scale failed", "error", err) - return err - } + // Commit this update via Raft + _, index, err = j.srv.raftApply(structs.JobRegisterRequestType, registerReq) + if err != nil { + j.logger.Error("job register for scale failed", "error", err) + return err + } - // FINISH: - // register the scaling event to the scaling_event table, once that exists + } // Populate the reply with job information reply.JobModifyIndex = index + // FINISH: + // register the scaling event to the scaling_event table, once that exists + // If the job is periodic or parameterized, we don't create an eval. if job != nil && (job.IsPeriodic() || job.IsParameterized()) { return nil diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 4f7e35ad62f..409d2020464 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -10,6 +10,9 @@ import ( memdb "github.com/hashicorp/go-memdb" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" + "github.com/kr/pretty" + "github.com/stretchr/testify/require" + "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper" @@ -17,8 +20,6 @@ import ( "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/testutil" - "github.com/kr/pretty" - "github.com/stretchr/testify/require" ) func TestJobEndpoint_Register(t *testing.T) { @@ -5268,16 +5269,199 @@ func TestJobEndpoint_Dispatch(t *testing.T) { } } +func TestJobEndpoint_Scale(t *testing.T) { + t.Parallel() + require := require.New(t) + + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + state := s1.fsm.State() + + job := mock.Job() + count := job.TaskGroups[0].Count + err := state.UpsertJob(1000, job) + require.Nil(err) + + scale := &structs.JobScaleRequest{ + JobID: job.ID, + Target: map[string]string{ + structs.ScalingTargetGroup: job.TaskGroups[0].Name, + }, + Count: helper.Int64ToPtr(int64(count + 1)), + Reason: helper.StringToPtr("this should fail"), + Meta: map[string]interface{}{ + "metrics": map[string]string{ + "1": "a", + "2": "b", + }, + "other": "value", + }, + PolicyOverride: false, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + var resp structs.JobRegisterResponse + err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) + require.NoError(err) + require.NotEmpty(resp.EvalID) +} + +func TestJobEndpoint_Scale_ACL(t *testing.T) { + t.Parallel() + require := require.New(t) + + s1, root, cleanupS1 := TestACLServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + state := s1.fsm.State() + + job := mock.Job() + err := state.UpsertJob(1000, job) + require.Nil(err) + + scale := &structs.JobScaleRequest{ + JobID: job.ID, + Target: map[string]string{ + structs.ScalingTargetGroup: job.TaskGroups[0].Name, + }, + Reason: helper.StringToPtr("this should fail"), + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + + // Scale without a token should fail + var resp structs.JobRegisterResponse + err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) + require.NotNil(err) + require.Contains(err.Error(), "Permission denied") + + // Expect failure for request with an invalid token + invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) + scale.AuthToken = invalidToken.SecretID + var invalidResp structs.JobRegisterResponse + require.NotNil(err) + err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &invalidResp) + require.Contains(err.Error(), "Permission denied") + + type testCase struct { + authToken string + name string + } + cases := []testCase{ + { + name: "mgmt token should succeed", + authToken: root.SecretID, + }, + { + name: "write disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-write", + mock.NamespacePolicy(structs.DefaultNamespace, "write", nil)). + SecretID, + }, + { + name: "autoscaler disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-autoscaler", + mock.NamespacePolicy(structs.DefaultNamespace, "autoscaler", nil)). + SecretID, + }, + { + name: "submit-job capability should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-submit-job", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilitySubmitJob})).SecretID, + }, + { + name: "scale-job capability should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-scale-job", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityScaleJob})). + SecretID, + }, + } + + for _, tc := range cases { + scale.AuthToken = tc.authToken + var resp structs.JobRegisterResponse + err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) + require.NoError(err, tc.name) + require.NotNil(resp.EvalID) + } + +} + +func TestJobEndpoint_Scale_Invalid(t *testing.T) { + t.Parallel() + require := require.New(t) + + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + state := s1.fsm.State() + + job := mock.Job() + count := job.TaskGroups[0].Count + + // check before job registration + scale := &structs.JobScaleRequest{ + JobID: job.ID, + Target: map[string]string{ + structs.ScalingTargetGroup: job.TaskGroups[0].Name, + }, + Count: helper.Int64ToPtr(int64(count) + 1), + Reason: helper.StringToPtr("this should fail"), + Meta: map[string]interface{}{ + "metrics": map[string]string{ + "1": "a", + "2": "b", + }, + "other": "value", + }, + PolicyOverride: false, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: job.Namespace, + }, + } + var resp structs.JobRegisterResponse + err := msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) + require.Error(err) + require.Contains(err.Error(), "not found") + + // register the job + err = state.UpsertJob(1000, job) + require.Nil(err) + + scale.Count = nil + scale.Error = helper.StringToPtr("error and reason") + scale.Reason = helper.StringToPtr("is not allowed") + err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) + require.Error(err) + require.Contains(err.Error(), "should not contain error and scaling reason") + + scale.Count = helper.Int64ToPtr(10) + scale.Reason = nil + scale.Error = helper.StringToPtr("error and count is not allowed") + err = msgpackrpc.CallWithCodec(codec, "Job.Scale", scale, &resp) + require.Error(err) + require.Contains(err.Error(), "should not contain error and count") +} + func TestJobEndpoint_GetScaleStatus(t *testing.T) { t.Parallel() require := require.New(t) - s1, cleanupS1 := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 // Prevent automatic dequeue - }) + s1, cleanupS1 := TestServer(t, nil) defer cleanupS1() codec := rpcClient(t, s1) testutil.WaitForLeader(t, s1.RPC) + state := s1.fsm.State() job := mock.Job() @@ -5295,19 +5479,8 @@ func TestJobEndpoint_GetScaleStatus(t *testing.T) { require.Nil(resp2.JobScaleStatus) // Create the register request - req := &structs.JobRegisterRequest{ - Job: job, - WriteRequest: structs.WriteRequest{ - Region: "global", - Namespace: job.Namespace, - }, - } - - // Fetch the response - var resp structs.JobRegisterResponse - require.NoError(msgpackrpc.CallWithCodec(codec, "Job.Register", req, &resp)) - job.CreateIndex = resp.JobModifyIndex - job.ModifyIndex = resp.JobModifyIndex + err := state.UpsertJob(1000, job) + require.Nil(err) // check after job registration require.NoError(msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp2)) @@ -5366,7 +5539,6 @@ func TestJobEndpoint_GetScaleStatus_ACL(t *testing.T) { // Expect failure for request with an invalid token invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) - get.AuthToken = invalidToken.SecretID var invalidResp structs.JobScaleStatusResponse require.NotNil(err) diff --git a/nomad/scaling_endpoint_test.go b/nomad/scaling_endpoint_test.go index c6d06e22b6e..2a3bf36f8d3 100644 --- a/nomad/scaling_endpoint_test.go +++ b/nomad/scaling_endpoint_test.go @@ -51,6 +51,35 @@ func TestScalingEndpoint_GetPolicy(t *testing.T) { require.Nil(resp.Policy) } +func TestScalingEndpoint_ListPolicies(t *testing.T) { + assert := assert.New(t) + t.Parallel() + + s1, cleanupS1 := TestServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the register request + p1 := mock.ScalingPolicy() + p2 := mock.ScalingPolicy() + + s1.fsm.State().UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) + + // Lookup the policies + get := &structs.ScalingPolicyListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + var resp structs.ACLPolicyListResponse + if err := msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp); err != nil { + t.Fatalf("err: %v", err) + } + assert.EqualValues(1000, resp.Index) + assert.Len(resp.Policies, 2) +} + func TestScalingEndpoint_ListPolicies_Blocking(t *testing.T) { t.Parallel() @@ -95,32 +124,3 @@ func TestScalingEndpoint_ListPolicies_Blocking(t *testing.T) { require.Len(resp.Policies, 2) require.ElementsMatch([]string{p1.ID, p2.ID}, []string{resp.Policies[0].ID, resp.Policies[1].ID}) } - -func TestScalingEndpoint_ListPolicies(t *testing.T) { - assert := assert.New(t) - t.Parallel() - - s1, cleanupS1 := TestServer(t, nil) - defer cleanupS1() - codec := rpcClient(t, s1) - testutil.WaitForLeader(t, s1.RPC) - - // Create the register request - p1 := mock.ScalingPolicy() - p2 := mock.ScalingPolicy() - - s1.fsm.State().UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) - - // Lookup the policies - get := &structs.ScalingPolicyListRequest{ - QueryOptions: structs.QueryOptions{ - Region: "global", - }, - } - var resp structs.ACLPolicyListResponse - if err := msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp); err != nil { - t.Fatalf("err: %v", err) - } - assert.EqualValues(1000, resp.Index) - assert.Len(resp.Policies, 2) -} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 36fdd5eacd3..1a355b31b41 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -624,7 +624,7 @@ type JobScaleRequest struct { Namespace string JobID string Target map[string]string - Count int64 + Count *int64 Reason *string Error *string Meta map[string]interface{} From ff652bff037cf5ed5377ffde25b3bb7e2837652f Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Mon, 23 Mar 2020 01:01:36 +0000 Subject: [PATCH 28/30] add acl validation to Scaling.ListPolicies and Scaling.GetPolicy --- nomad/job_endpoint_test.go | 3 +- nomad/scaling_endpoint.go | 73 ++++--------- nomad/scaling_endpoint_test.go | 186 +++++++++++++++++++++++++++++---- 3 files changed, 192 insertions(+), 70 deletions(-) diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 409d2020464..62d0e4b75da 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -5540,9 +5540,8 @@ func TestJobEndpoint_GetScaleStatus_ACL(t *testing.T) { invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) get.AuthToken = invalidToken.SecretID - var invalidResp structs.JobScaleStatusResponse require.NotNil(err) - err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &invalidResp) + err = msgpackrpc.CallWithCodec(codec, "Job.ScaleStatus", get, &resp) require.Contains(err.Error(), "Permission denied") type testCase struct { diff --git a/nomad/scaling_endpoint.go b/nomad/scaling_endpoint.go index 44be4989588..89e1aa0a679 100644 --- a/nomad/scaling_endpoint.go +++ b/nomad/scaling_endpoint.go @@ -7,6 +7,7 @@ import ( log "github.com/hashicorp/go-hclog" memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" ) @@ -26,31 +27,17 @@ func (a *Scaling) ListPolicies(args *structs.ScalingPolicyListRequest, } defer metrics.MeasureSince([]string{"nomad", "scaling", "list_policies"}, time.Now()) - // Check management level permissions - // acl, err := a.srv.ResolveToken(args.AuthToken) - // if err != nil { - // return err - // } else if acl == nil { - // return structs.ErrPermissionDenied - // } - - // If it is not a management token determine the policies that may be listed - // mgt := acl.IsManagement() - // var policies map[string]struct{} - // if !mgt { - // token, err := a.requestACLToken(args.AuthToken) - // if err != nil { - // return err - // } - // if token == nil { - // return structs.ErrTokenNotFound - // } - // - // policies = make(map[string]struct{}, len(token.Policies)) - // for _, p := range token.Policies { - // policies[p] = struct{}{} - // } - // } + // Check for list-job permissions + if aclObj, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if aclObj != nil { + hasListScalingPolicies := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityListScalingPolicies) + hasListAndReadJobs := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityListJobs) && + aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) + if !(hasListScalingPolicies || hasListAndReadJobs) { + return structs.ErrPermissionDenied + } + } // Setup the blocking query opts := blockingOptions{ @@ -103,31 +90,17 @@ func (a *Scaling) GetPolicy(args *structs.ScalingPolicySpecificRequest, } defer metrics.MeasureSince([]string{"nomad", "scaling", "get_policy"}, time.Now()) - // Check management level permissions - // acl, err := a.srv.ResolveToken(args.AuthToken) - // if err != nil { - // return err - // } else if acl == nil { - // return structs.ErrPermissionDenied - // } - - // If it is not a management token determine the policies that may be listed - // mgt := acl.IsManagement() - // var policies map[string]struct{} - // if !mgt { - // token, err := a.requestACLToken(args.AuthToken) - // if err != nil { - // return err - // } - // if token == nil { - // return structs.ErrTokenNotFound - // } - // - // policies = make(map[string]struct{}, len(token.Policies)) - // for _, p := range token.Policies { - // policies[p] = struct{}{} - // } - // } + // Check for list-job permissions + if aclObj, err := a.srv.ResolveToken(args.AuthToken); err != nil { + return err + } else if aclObj != nil { + hasReadScalingPolicy := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadScalingPolicy) + hasListAndReadJobs := aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityListJobs) && + aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) + if !(hasReadScalingPolicy || hasListAndReadJobs) { + return structs.ErrPermissionDenied + } + } // Setup the blocking query opts := blockingOptions{ diff --git a/nomad/scaling_endpoint_test.go b/nomad/scaling_endpoint_test.go index 2a3bf36f8d3..9fd00de4a6b 100644 --- a/nomad/scaling_endpoint_test.go +++ b/nomad/scaling_endpoint_test.go @@ -1,13 +1,12 @@ package nomad import ( - "testing" - "time" - msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "testing" + "time" + "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -16,7 +15,6 @@ import ( func TestScalingEndpoint_GetPolicy(t *testing.T) { t.Parallel() - require := require.New(t) s1, cleanupS1 := TestServer(t, nil) @@ -24,7 +22,6 @@ func TestScalingEndpoint_GetPolicy(t *testing.T) { codec := rpcClient(t, s1) testutil.WaitForLeader(t, s1.RPC) - // Create the register request p1 := mock.ScalingPolicy() p2 := mock.ScalingPolicy() s1.fsm.State().UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) @@ -39,7 +36,7 @@ func TestScalingEndpoint_GetPolicy(t *testing.T) { var resp structs.SingleScalingPolicyResponse err := msgpackrpc.CallWithCodec(codec, "Scaling.GetPolicy", get, &resp) require.NoError(err) - require.Equal(uint64(1000), resp.Index) + require.EqualValues(1000, resp.Index) require.Equal(*p1, *resp.Policy) // Lookup non-existing policy @@ -47,42 +44,195 @@ func TestScalingEndpoint_GetPolicy(t *testing.T) { resp = structs.SingleScalingPolicyResponse{} err = msgpackrpc.CallWithCodec(codec, "Scaling.GetPolicy", get, &resp) require.NoError(err) - require.Equal(uint64(1000), resp.Index) + require.EqualValues(1000, resp.Index) require.Nil(resp.Policy) } +func TestScalingEndpoint_GetPolicy_ACL(t *testing.T) { + t.Parallel() + require := require.New(t) + + s1, root, cleanupS1 := TestACLServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + state := s1.fsm.State() + + p1 := mock.ScalingPolicy() + p2 := mock.ScalingPolicy() + state.UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) + + get := &structs.ScalingPolicySpecificRequest{ + ID: p1.ID, + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + + // lookup without token should fail + var resp structs.SingleScalingPolicyResponse + err := msgpackrpc.CallWithCodec(codec, "Scaling.GetPolicy", get, &resp) + require.Error(err) + require.Contains(err.Error(), "Permission denied") + + // Expect failure for request with an invalid token + invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListScalingPolicies})) + get.AuthToken = invalidToken.SecretID + err = msgpackrpc.CallWithCodec(codec, "Scaling.GetPolicy", get, &resp) + require.Error(err) + require.Contains(err.Error(), "Permission denied") + type testCase struct { + authToken string + name string + } + cases := []testCase{ + { + name: "mgmt token should succeed", + authToken: root.SecretID, + }, + { + name: "read disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read", + mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).SecretID, + }, + { + name: "write disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-write", + mock.NamespacePolicy(structs.DefaultNamespace, "write", nil)).SecretID, + }, + { + name: "autoscaler disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-autoscaler", + mock.NamespacePolicy(structs.DefaultNamespace, "autoscaler", nil)).SecretID, + }, + { + name: "list-jobs+read-job capability should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read-job-scaling", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs, acl.NamespaceCapabilityReadJob})).SecretID, + }, + } + + for _, tc := range cases { + get.AuthToken = tc.authToken + err = msgpackrpc.CallWithCodec(codec, "Scaling.GetPolicy", get, &resp) + require.NoError(err, tc.name) + require.EqualValues(1000, resp.Index) + require.NotNil(resp.Policy) + } + +} + func TestScalingEndpoint_ListPolicies(t *testing.T) { - assert := assert.New(t) t.Parallel() + require := require.New(t) s1, cleanupS1 := TestServer(t, nil) defer cleanupS1() codec := rpcClient(t, s1) testutil.WaitForLeader(t, s1.RPC) - // Create the register request + // Lookup the policies + get := &structs.ScalingPolicyListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + var resp structs.ACLPolicyListResponse + err := msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp) + require.NoError(err) + require.Empty(resp.Policies) + p1 := mock.ScalingPolicy() p2 := mock.ScalingPolicy() - s1.fsm.State().UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) - // Lookup the policies + err = msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp) + require.NoError(err) + require.EqualValues(1000, resp.Index) + require.Len(resp.Policies, 2) +} + +func TestScalingEndpoint_ListPolicies_ACL(t *testing.T) { + t.Parallel() + require := require.New(t) + + s1, root, cleanupS1 := TestACLServer(t, nil) + defer cleanupS1() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + state := s1.fsm.State() + + p1 := mock.ScalingPolicy() + p2 := mock.ScalingPolicy() + state.UpsertScalingPolicies(1000, []*structs.ScalingPolicy{p1, p2}) + get := &structs.ScalingPolicyListRequest{ QueryOptions: structs.QueryOptions{ Region: "global", }, } + + // lookup without token should fail var resp structs.ACLPolicyListResponse - if err := msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp); err != nil { - t.Fatalf("err: %v", err) + err := msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp) + require.Error(err) + require.Contains(err.Error(), "Permission denied") + + // Expect failure for request with an invalid token + invalidToken := mock.CreatePolicyAndToken(t, state, 1003, "test-invalid", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListScalingPolicies})) + get.AuthToken = invalidToken.SecretID + require.Error(err) + require.Contains(err.Error(), "Permission denied") + + type testCase struct { + authToken string + name string + } + cases := []testCase{ + { + name: "mgmt token should succeed", + authToken: root.SecretID, + }, + { + name: "read disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read", + mock.NamespacePolicy(structs.DefaultNamespace, "read", nil)).SecretID, + }, + { + name: "write disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-write", + mock.NamespacePolicy(structs.DefaultNamespace, "write", nil)).SecretID, + }, + { + name: "autoscaler disposition should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-autoscaler", + mock.NamespacePolicy(structs.DefaultNamespace, "autoscaler", nil)).SecretID, + }, + { + name: "list-scaling-policies capability should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-list-scaling-policies", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListScalingPolicies})).SecretID, + }, + { + name: "list-jobs+read-job capability should succeed", + authToken: mock.CreatePolicyAndToken(t, state, 1005, "test-valid-read-job-scaling", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs, acl.NamespaceCapabilityReadJob})).SecretID, + }, + } + + for _, tc := range cases { + get.AuthToken = tc.authToken + err = msgpackrpc.CallWithCodec(codec, "Scaling.ListPolicies", get, &resp) + require.NoError(err, tc.name) + require.EqualValues(1000, resp.Index) + require.Len(resp.Policies, 2) } - assert.EqualValues(1000, resp.Index) - assert.Len(resp.Policies, 2) } func TestScalingEndpoint_ListPolicies_Blocking(t *testing.T) { t.Parallel() - require := require.New(t) s1, cleanupS1 := TestServer(t, nil) @@ -120,7 +270,7 @@ func TestScalingEndpoint_ListPolicies_Blocking(t *testing.T) { require.NoError(err) require.True(time.Since(start) > 200*time.Millisecond, "should block: %#v", resp) - require.Equal(uint64(200), resp.Index, "bad index") + require.EqualValues(200, resp.Index, "bad index") require.Len(resp.Policies, 2) require.ElementsMatch([]string{p1.ID, p2.ID}, []string{resp.Policies[0].ID, resp.Policies[1].ID}) } From e831ec3bd5523f53c5491ad82b2d5f0c3583cfd0 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Mon, 23 Mar 2020 12:15:50 +0000 Subject: [PATCH 29/30] added new int64ToPtr method to api/util to avoid pulling in other packages --- api/jobs.go | 4 +--- api/util_test.go | 9 ++------- api/utils.go | 5 +++++ 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index ab90cf5e8d7..f706d887a66 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -8,8 +8,6 @@ import ( "time" "github.com/gorhill/cronexpr" - - "github.com/hashicorp/nomad/helper" ) const ( @@ -161,7 +159,7 @@ func (j *Jobs) Scale(jobID, group string, count *int, reason, error *string, meta map[string]interface{}, q *WriteOptions) (*JobRegisterResponse, *WriteMeta, error) { var count64 *int64 if count != nil { - count64 = helper.Int64ToPtr(int64(*count)) + count64 = int64ToPtr(int64(*count)) } req := &ScalingRequest{ Count: count64, diff --git a/api/util_test.go b/api/util_test.go index c796766e32c..b63cdf75138 100644 --- a/api/util_test.go +++ b/api/util_test.go @@ -52,8 +52,8 @@ func testJobWithScalingPolicy() *Job { job := testJob() job.TaskGroups[0].Scaling = &ScalingPolicy{ Policy: map[string]interface{}{}, - Min: int64ToPtr(1), - Max: 1, + Min: int64ToPtr(1), + Max: 1, Enabled: boolToPtr(true), } return job @@ -94,11 +94,6 @@ func testQuotaSpec() *QuotaSpec { // conversions utils only used for testing // added here to avoid linter warning -// int64ToPtr returns the pointer to an int -func int64ToPtr(i int64) *int64 { - return &i -} - // float64ToPtr returns the pointer to an float64 func float64ToPtr(f float64) *float64 { return &f diff --git a/api/utils.go b/api/utils.go index c5faa73ecf8..d56d889a59c 100644 --- a/api/utils.go +++ b/api/utils.go @@ -26,6 +26,11 @@ func uint64ToPtr(u uint64) *uint64 { return &u } +// int64ToPtr returns the pointer to a int64 +func int64ToPtr(u int64) *int64 { + return &u +} + // stringToPtr returns the pointer to a string func stringToPtr(str string) *string { return &str From 4330fe8497201e12dbc3f18ddc23c95b04e7b6e9 Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Mon, 23 Mar 2020 17:57:36 +0000 Subject: [PATCH 30/30] bad conversion between api.ScalingPolicy and structs.ScalingPolicy meant that we were throwing away .Min if provided --- api/scaling_test.go | 6 ++++-- command/agent/scaling_endpoint.go | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/api/scaling_test.go b/api/scaling_test.go index 53238af9d21..38f28a429bc 100644 --- a/api/scaling_test.go +++ b/api/scaling_test.go @@ -74,8 +74,8 @@ func TestScalingPolicies_GetPolicy(t *testing.T) { job := testJob() policy := &ScalingPolicy{ Enabled: boolToPtr(true), - Min: int64ToPtr(1), - Max: 1, + Min: int64ToPtr(1), + Max: 1, Policy: map[string]interface{}{ "key": "value", }, @@ -115,4 +115,6 @@ func TestScalingPolicies_GetPolicy(t *testing.T) { require.Equal(expectedTarget, resp.Target) require.Equal(policy.Policy, resp.Policy) require.Equal(policy.Enabled, resp.Enabled) + require.Equal(*policy.Min, *resp.Min) + require.Equal(policy.Max, resp.Max) } diff --git a/command/agent/scaling_endpoint.go b/command/agent/scaling_endpoint.go index fe94d4dbcda..df0df44bf5a 100644 --- a/command/agent/scaling_endpoint.go +++ b/command/agent/scaling_endpoint.go @@ -82,7 +82,9 @@ func ApiScalingPolicyToStructs(count int, ap *api.ScalingPolicy) *structs.Scalin Policy: ap.Policy, Target: map[string]string{}, } - if ap.Min == nil { + if ap.Min != nil { + p.Min = *ap.Min + } else { p.Min = int64(count) } return &p