Fixed tests. Added vault integration into parameters

cmd/gaia/main.go

@@ -146,6 +146,18 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Initiating Vault
+	// Check Vault path
+	if gaia.Cfg.VaultPath == "" {
+		// Set default to data folder
+		gaia.Cfg.VaultPath = gaia.Cfg.DataPath
+	}
+	_, err = services.VaultService(nil)
+	if err != nil {
+		gaia.Cfg.Logger.Error("error initiating vault")
+		os.Exit(1)
+	}
+
 	// Initialize scheduler
 	_, err = services.SchedulerService()
 	if err != nil {
@@ -159,18 +171,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	// Initiating Vault
-	// Check Vault path
-	if gaia.Cfg.VaultPath == "" {
-		// Set default to data folder
-		gaia.Cfg.VaultPath = gaia.Cfg.DataPath
-	}
-	_, err = services.VaultService(nil)
-	if err != nil {
-		gaia.Cfg.Logger.Error("error initiating vault")
-		os.Exit(1)
-	}
-
 	// Start ticker. Periodic job to check for new plugins.
 	pipeline.InitTicker()
 

frontend/client/views/overview/index.vue

@@ -105,7 +105,7 @@ export default {
     checkPipelineArgs (pipeline) {
       // check if this pipeline has args
       for (let x = 0, y = pipeline.jobs.length; x < y; x++) {
-        if (pipeline.jobs[x].args) {
+        if (pipeline.jobs[x].args && pipeline.jobs[x].args.type !== 'vault') {
           // we found args. Redirect user to params view.
           this.$router.push({path: '/pipeline/params', query: { pipelineid: pipeline.id }})
         }

frontend/client/views/pipeline/detail.vue

@@ -2,7 +2,7 @@
   <div class="tile is-ancestor">
     <div class="tile is-vertical">
       <div class="tile is-parent">
-        <a class="button is-primary" @click="startPipeline(pipelineID)" style="margin-right: 10px;">
+        <a class="button is-primary" @click="checkPipelineArgs" style="margin-right: 10px;">
           <span class="icon">
             <i class="fa fa-play-circle"></i>
           </span>
@@ -120,7 +120,8 @@ export default {
           },
           arrows: {to: true}
         }
-      }
+      },
+      pipeline: null
     }
   },
 
@@ -183,6 +184,7 @@ export default {
               this.drawPipelineDetail(pipeline.data, pipelineRun.data)
             }
             this.runsRows = pipelineRuns.data
+            this.pipeline = pipeline.data
           }.bind(this)))
           .catch((error) => {
             this.$store.commit('clearIntervals')
@@ -201,6 +203,7 @@ export default {
             if (pipelineRuns.data) {
               this.runsRows = pipelineRuns.data
             }
+            this.pipeline = pipeline.data
           }.bind(this)))
           .catch((error) => {
             this.$store.commit('clearIntervals')
@@ -355,13 +358,26 @@ export default {
       this.$router.push({path: '/pipeline/log', query: { pipelineid: this.pipelineID, runid: this.runID }})
     },
 
-    startPipeline (pipelineid) {
+    checkPipelineArgs () {
+      // check if this pipeline has args
+      for (let x = 0, y = this.pipeline.jobs.length; x < y; x++) {
+        if (this.pipeline.jobs[x].args && this.pipeline.jobs[x].args.type !== 'vault') {
+          // we found args. Redirect user to params view.
+          this.$router.push({path: '/pipeline/params', query: { pipelineid: this.pipeline.id }})
+        }
+      }
+
+      // No args. Just start pipeline.
+      this.startPipeline()
+    },
+
+    startPipeline () {
       // Send start request
       this.$http
-        .post('/api/v1/pipeline/' + pipelineid + '/start')
+        .post('/api/v1/pipeline/' + this.pipeline.id + '/start')
         .then(response => {
           if (response.data) {
-            this.$router.push({path: '/pipeline/detail', query: { pipelineid: pipelineid, runid: response.data.id }})
+            this.$router.push({path: '/pipeline/detail', query: { pipelineid: this.pipeline.id, runid: response.data.id }})
           }
         })
         .catch((error) => {

frontend/client/views/pipeline/params.vue

@@ -65,9 +65,16 @@ export default {
 
             // Get all arguments
             for (let x = 0, y = pipeline.jobs.length; x < y; x++) {
-              if (pipeline.jobs[x].args) {
-                // Merge arguments from all jobs
-                this.args = this.args.concat(pipeline.jobs[x].args)
+              let args = pipeline.jobs[x].args
+              // we skip vault cause this is automatically filled by the vault.
+              if (args) {
+                // iterate all arguments
+                for (let argID = 0, argTotal = args.length; argID < argTotal; argID++) {
+                  // we skip vault arguments cause they are autofilled in the backend.
+                  if (args[argID].type !== 'vault') {
+                    this.args.push(args[argID])
+                  }
+                }
               }
             }
           }

handlers/pipeline.go

@@ -241,11 +241,10 @@ func PipelineStart(c echo.Context) error {
 	schedulerService, _ := services.SchedulerService()
 	pipelineIDStr := c.Param("pipelineid")
 
-	// Look for arguments
+	// Look for arguments.
+	// We do not check for errors here cause arguments are optional.
 	args := []gaia.Argument{}
-	if err := c.Bind(&args); err != nil {
-		return c.String(http.StatusBadRequest, "bad arguments given")
-	}
+	c.Bind(&args)
 
 	// Convert string to int because id is int
 	pipelineID, err := strconv.Atoi(pipelineIDStr)

handlers/pipeline_test.go

@@ -27,7 +27,7 @@ type mockScheduleService struct {
 	err         error
 }
 
-func (ms *mockScheduleService) SchedulePipeline(p *gaia.Pipeline) (*gaia.PipelineRun, error) {
+func (ms *mockScheduleService) SchedulePipeline(p *gaia.Pipeline, args []gaia.Argument) (*gaia.PipelineRun, error) {
 	return ms.pipelineRun, ms.err
 }
 

scheduler/scheduler.go

@@ -25,6 +25,9 @@ const (
 
 	// errCircularDep is thrown when a circular dependency has been detected.
 	errCircularDep = "circular dependency detected between %s and %s"
+
+	// argTypeVault is the argument type vault.
+	argTypeVault = "vault"
 )
 
 var (
@@ -79,16 +82,20 @@ type Scheduler struct {
 
 	// ca is the instance of the CA used to handle certs.
 	ca security.CAAPI
+
+	// vault is the instance of the vault.
+	vault security.VaultAPI
 }
 
 // NewScheduler creates a new instance of Scheduler.
-func NewScheduler(store store.GaiaStore, pS Plugin, ca security.CAAPI) *Scheduler {
+func NewScheduler(store store.GaiaStore, pS Plugin, ca security.CAAPI, vault security.VaultAPI) *Scheduler {
 	// Create new scheduler
 	s := &Scheduler{
 		scheduledRuns: make(chan gaia.PipelineRun, schedulerBufferLimit),
 		storeService:  store,
 		pluginSystem:  pS,
 		ca:            ca,
+		vault:         vault,
 	}
 
 	return s
@@ -253,15 +260,33 @@ func (s *Scheduler) SchedulePipeline(p *gaia.Pipeline, args []gaia.Argument) (*g
 		return nil, err
 	}
 
+	// Load secret from vault and set it
+	err = s.vault.LoadSecrets()
+	if err != nil {
+		gaia.Cfg.Logger.Error("cannot load secrets from vault during schedule pipeline", "error", err.Error())
+		return nil, err
+	}
+
 	// We have to go through all jobs to find the related arguments.
 	// We will only pass related arguments to the specific job.
 	for jobID, job := range jobs {
 		if job.Args != nil {
 			for argID, arg := range job.Args {
-				// Find related argument in given arguments
-				for _, givenArg := range args {
-					if arg.Key == givenArg.Key {
-						jobs[jobID].Args[argID] = givenArg
+				// check if it's of type vault
+				if arg.Type == argTypeVault {
+					// Get & Set argument
+					s, err := s.vault.Get(arg.Key)
+					if err != nil {
+						gaia.Cfg.Logger.Error("cannot find secret with given key in vault", "key", arg.Key, "pipeline", p)
+						return nil, err
+					}
+					jobs[jobID].Args[argID].Value = string(s)
+				} else {
+					// Find related argument in given arguments
+					for _, givenArg := range args {
+						if arg.Key == givenArg.Key {
+							jobs[jobID].Args[argID] = givenArg
+						}
 					}
 				}
 			}

scheduler/scheduler_test.go

@@ -31,6 +31,15 @@ func (c *CAFake) GenerateTLSConfig(certPath, keyPath string) (*tls.Config, error
 func (c *CAFake) CleanupCerts(crt, key string) error                              { return nil }
 func (c *CAFake) GetCACertPath() (string, string)                                 { return "", "" }
 
+type VaultFake struct{}
+
+func (v *VaultFake) LoadSecrets() error             { return nil }
+func (v *VaultFake) GetAll() []string               { return []string{} }
+func (v *VaultFake) SaveSecrets() error             { return nil }
+func (v *VaultFake) Add(key string, value []byte)   {}
+func (v *VaultFake) Remove(key string)              {}
+func (v *VaultFake) Get(key string) ([]byte, error) { return []byte{}, nil }
+
 func TestInit(t *testing.T) {
 	gaia.Cfg = &gaia.Config{}
 	storeInstance := store.NewBoltStore()
@@ -47,9 +56,7 @@ func TestInit(t *testing.T) {
 	if err := storeInstance.Init(); err != nil {
 		t.Fatal(err)
 	}
-	var ca security.CAAPI
-	ca = &CAFake{}
-	s := NewScheduler(storeInstance, &PluginFake{}, ca)
+	s := NewScheduler(storeInstance, &PluginFake{}, &CAFake{}, &VaultFake{})
 	err := s.Init()
 	if err != nil {
 		t.Fatal(err)
@@ -82,7 +89,7 @@ func TestPrepareAndExecFail(t *testing.T) {
 	}
 	p, r := prepareTestData()
 	storeInstance.PipelinePut(&p)
-	s := NewScheduler(storeInstance, &PluginFakeFailed{}, &CAFake{})
+	s := NewScheduler(storeInstance, &PluginFakeFailed{}, &CAFake{}, &VaultFake{})
 	s.prepareAndExec(r)
 
 	// get pipeline run from store
@@ -121,9 +128,7 @@ func TestPrepareAndExecInvalidType(t *testing.T) {
 	p, r := prepareTestData()
 	p.Type = gaia.PTypeUnknown
 	storeInstance.PipelinePut(&p)
-	var ca security.CAAPI
-	ca = &CAFake{}
-	s := NewScheduler(storeInstance, &PluginFake{}, ca)
+	s := NewScheduler(storeInstance, &PluginFake{}, &CAFake{}, &VaultFake{})
 	s.prepareAndExec(r)
 
 	// get pipeline run from store
@@ -158,7 +163,7 @@ func TestPrepareAndExecJavaType(t *testing.T) {
 	javaExecuteableName = "go"
 	p.Type = gaia.PTypeJava
 	storeInstance.PipelinePut(&p)
-	s := NewScheduler(storeInstance, &PluginFake{}, &CAFake{})
+	s := NewScheduler(storeInstance, &PluginFake{}, &CAFake{}, &VaultFake{})
 	s.prepareAndExec(r)
 
 	// get pipeline run from store
@@ -200,9 +205,7 @@ func TestSchedulePipeline(t *testing.T) {
 	}
 	p, _ := prepareTestData()
 	storeInstance.PipelinePut(&p)
-	var ca security.CAAPI
-	ca = &CAFake{}
-	s := NewScheduler(storeInstance, &PluginFake{}, ca)
+	s := NewScheduler(storeInstance, &PluginFake{}, &CAFake{}, &VaultFake{})
 	err := s.Init()
 	if err != nil {
 		t.Fatal(err)
@@ -231,9 +234,7 @@ func TestSchedule(t *testing.T) {
 	}
 	p, _ := prepareTestData()
 	storeInstance.PipelinePut(&p)
-	var ca security.CAAPI
-	ca = &CAFake{}
-	s := NewScheduler(storeInstance, &PluginFake{}, ca)
+	s := NewScheduler(storeInstance, &PluginFake{}, &CAFake{}, &VaultFake{})
 	_, err := s.SchedulePipeline(&p, prepareArgs())
 	if err != nil {
 		t.Fatal(err)
@@ -265,9 +266,7 @@ func TestSetPipelineJobs(t *testing.T) {
 	}
 	p, _ := prepareTestData()
 	p.Jobs = nil
-	var ca security.CAAPI
-	ca = &CAFake{}
-	s := NewScheduler(storeInstance, &PluginFake{}, ca)
+	s := NewScheduler(storeInstance, &PluginFake{}, &CAFake{}, &VaultFake{})
 	err := s.SetPipelineJobs(&p)
 	if err != nil {
 		t.Fatal(err)
@@ -338,6 +337,7 @@ func prepareTestData() (pipeline gaia.Pipeline, pipelineRun gaia.PipelineRun) {
 		PipelineID: 1,
 		Status:     gaia.RunNotScheduled,
 		UniqueID:   uuid.Must(uuid.NewV4(), nil).String(),
+		Jobs:       pipeline.Jobs,
 	}
 	return
 }

services/service_provider.go

@@ -58,7 +58,7 @@ func SchedulerService() (scheduler.GaiaScheduler, error) {
 		return schedulerService, nil
 	}
 	pS := &plugin.Plugin{}
-	schedulerService = scheduler.NewScheduler(storeService, pS, certificateService)
+	schedulerService = scheduler.NewScheduler(storeService, pS, certificateService, vaultService)
 	err := schedulerService.Init()
 	if err != nil {
 		gaia.Cfg.Logger.Error("cannot initialize scheduler:", "error", err.Error())