From d1e381b04f9706c6edbe840ac19ece81faba3c18 Mon Sep 17 00:00:00 2001 From: Antoine Eiche Date: Wed, 24 Apr 2024 20:58:03 +0200 Subject: [PATCH] Store deployments in a JSON file --- cmd/run.go | 9 ++- internal/manager/manager.go | 18 +++++- internal/manager/manager_test.go | 11 ++-- internal/storage/storage.go | 97 ++++++++++++++++++++++++++++++++ internal/storage/storage_test.go | 66 ++++++++++++++++++++++ 5 files changed, 193 insertions(+), 8 deletions(-) create mode 100644 internal/storage/storage.go create mode 100644 internal/storage/storage_test.go diff --git a/cmd/run.go b/cmd/run.go index 1058ec7..e28c3cc 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -2,6 +2,7 @@ package cmd import ( "os" + "path" "github.com/nlewo/comin/internal/config" "github.com/nlewo/comin/internal/http" @@ -9,6 +10,7 @@ import ( "github.com/nlewo/comin/internal/poller" "github.com/nlewo/comin/internal/prometheus" "github.com/nlewo/comin/internal/repository" + "github.com/nlewo/comin/internal/storage" "github.com/nlewo/comin/internal/utils" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -41,8 +43,13 @@ var runCmd = &cobra.Command{ } metrics := prometheus.New() + storageFilename := path.Join(cfg.StateDir, "storage.json") + storage := storage.New(storageFilename, 10, 10) + if err := storage.Load(); err != nil { + logrus.Errorf("Ignoring the state file %s because of the loading error: %s", storageFilename, err) + } metrics.SetBuildInfo(cmd.Version) - manager := manager.New(repository, metrics, gitConfig.Path, cfg.Hostname, machineId) + manager := manager.New(repository, storage, metrics, gitConfig.Path, cfg.Hostname, machineId) go poller.Poller(manager, cfg.Remotes) http.Serve(manager, metrics, diff --git a/internal/manager/manager.go b/internal/manager/manager.go index c119cf3..4806f52 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -9,6 +9,7 @@ import ( "github.com/nlewo/comin/internal/nix" "github.com/nlewo/comin/internal/prometheus" "github.com/nlewo/comin/internal/repository" + "github.com/nlewo/comin/internal/storage" "github.com/nlewo/comin/internal/utils" "github.com/sirupsen/logrus" ) @@ -55,10 +56,11 @@ type Manager struct { triggerDeploymentCh chan generation.Generation prometheus prometheus.Prometheus + storage storage.Storage } -func New(r repository.Repository, p prometheus.Prometheus, path, hostname, machineId string) Manager { - return Manager{ +func New(r repository.Repository, s storage.Storage, p prometheus.Prometheus, path, hostname, machineId string) Manager { + m := Manager{ repository: r, repositoryPath: path, hostname: hostname, @@ -74,7 +76,15 @@ func New(r repository.Repository, p prometheus.Prometheus, path, hostname, machi repositoryStatusCh: make(chan repository.RepositoryStatus), triggerDeploymentCh: make(chan generation.Generation, 1), prometheus: p, + storage: s, } + if len(s.DeploymentList()) > 0 { + d := s.DeploymentList()[0] + logrus.Infof("Restoring the manager state from the last deployment %s", d.UUID) + m.deployment = d + m.generation = d.Generation + } + return m } func (m Manager) GetState() State { @@ -136,6 +146,10 @@ func (m Manager) onDeployment(ctx context.Context, deploymentResult deployment.D } m.isRunning = false m.prometheus.SetDeploymentInfo(m.deployment.Generation.SelectedCommitId, deployment.StatusToString(m.deployment.Status)) + m.storage.DeploymentInsert(m.deployment) + if err := m.storage.Commit(); err != nil { + logrus.Errorf("Error while commiting the state.json file: %s", err) + } return m } diff --git a/internal/manager/manager_test.go b/internal/manager/manager_test.go index 7e53dfe..fd40fce 100644 --- a/internal/manager/manager_test.go +++ b/internal/manager/manager_test.go @@ -8,6 +8,7 @@ import ( "github.com/nlewo/comin/internal/deployment" "github.com/nlewo/comin/internal/prometheus" "github.com/nlewo/comin/internal/repository" + "github.com/nlewo/comin/internal/storage" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) @@ -33,7 +34,7 @@ func (r *repositoryMock) FetchAndUpdate(ctx context.Context, remoteName string) func TestRun(t *testing.T) { logrus.SetLevel(logrus.DebugLevel) r := newRepositoryMock() - m := New(r, prometheus.New(), "", "", "") + m := New(r, storage.New("", 1, 1), prometheus.New(), "", "", "") evalDone := make(chan struct{}) buildDone := make(chan struct{}) @@ -94,7 +95,7 @@ func TestRun(t *testing.T) { func TestFetchBusy(t *testing.T) { logrus.SetLevel(logrus.DebugLevel) r := newRepositoryMock() - m := New(r, prometheus.New(), "", "", "machine-id") + m := New(r, storage.New("", 1, 1), prometheus.New(), "", "", "machine-id") go m.Run() assert.Equal(t, State{}, m.GetState()) @@ -109,7 +110,7 @@ func TestFetchBusy(t *testing.T) { func TestRestartComin(t *testing.T) { logrus.SetLevel(logrus.DebugLevel) r := newRepositoryMock() - m := New(r, prometheus.New(), "", "", "machine-id") + m := New(r, storage.New("", 1, 1), prometheus.New(), "", "", "machine-id") dCh := make(chan deployment.DeploymentResult) m.deploymentResultCh = dCh isCominRestarted := false @@ -131,7 +132,7 @@ func TestRestartComin(t *testing.T) { func TestOptionnalMachineId(t *testing.T) { logrus.SetLevel(logrus.DebugLevel) r := newRepositoryMock() - m := New(r, prometheus.New(), "", "", "the-test-machine-id") + m := New(r, storage.New("", 1, 1), prometheus.New(), "", "", "the-test-machine-id") evalDone := make(chan struct{}) buildDone := make(chan struct{}) @@ -163,7 +164,7 @@ func TestOptionnalMachineId(t *testing.T) { func TestIncorrectMachineId(t *testing.T) { logrus.SetLevel(logrus.DebugLevel) r := newRepositoryMock() - m := New(r, prometheus.New(), "", "", "the-test-machine-id") + m := New(r, storage.New("", 1, 1), prometheus.New(), "", "", "the-test-machine-id") evalDone := make(chan struct{}) buildDone := make(chan struct{}) diff --git a/internal/storage/storage.go b/internal/storage/storage.go new file mode 100644 index 0000000..83993ff --- /dev/null +++ b/internal/storage/storage.go @@ -0,0 +1,97 @@ +package storage + +import ( + "encoding/json" + "errors" + "os" + + "github.com/nlewo/comin/internal/deployment" + "github.com/sirupsen/logrus" +) + +type Data struct { + Version string `json:"version"` + // Deployments are order from the most recent to older + Deployments []deployment.Deployment `json:"deployments"` +} + +type Storage struct { + Data + filename string + capacityMain int + capacityTesting int +} + +func New(filename string, capacityMain, capacityTesting int) Storage { + s := Storage{ + filename: filename, + capacityMain: capacityMain, + capacityTesting: capacityTesting, + } + s.Deployments = make([]deployment.Deployment, 0) + s.Version = "1" + return s + +} + +// DeploymentInsert inserts a deployment and return an evicted +// deployment because the capacity has been reached. +func (s *Storage) DeploymentInsert(dpl deployment.Deployment) (getsEvicted bool, evicted deployment.Deployment) { + var qty, older int + capacity := s.capacityMain + if dpl.IsTesting() { + capacity = s.capacityTesting + } + for i, d := range s.Deployments { + if dpl.IsTesting() == d.IsTesting() { + older = i + qty += 1 + } + } + // If the capacity is reached, we remove the older elements + if qty >= capacity { + evicted = s.Deployments[older] + getsEvicted = true + s.Deployments = append(s.Deployments[:older], s.Deployments[older+1:]...) + } + s.Deployments = append([]deployment.Deployment{dpl}, s.Deployments...) + return +} + +func (s *Storage) DeploymentList() []deployment.Deployment { + return s.Deployments +} + +func (s *Storage) LastDeployment() (ok bool, d deployment.Deployment) { + if len(s.DeploymentList()) > 1 { + return true, s.DeploymentList()[0] + } + return +} + +func (s *Storage) Load() (err error) { + var data Data + content, err := os.ReadFile(s.filename) + if errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return + } + err = json.Unmarshal(content, &data) + if err != nil { + return + } + // FIXME: we should check the version + s.Deployments = data.Deployments + logrus.Infof("Loaded %d deployments from %s", len(s.Deployments), s.filename) + return +} + +func (s *Storage) Commit() (err error) { + content, err := json.Marshal(s) + if err != nil { + return + } + err = os.WriteFile(s.filename, content, 0644) + return +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go new file mode 100644 index 0000000..8f9f17e --- /dev/null +++ b/internal/storage/storage_test.go @@ -0,0 +1,66 @@ +package storage + +import ( + "testing" + + "github.com/nlewo/comin/internal/deployment" + "github.com/stretchr/testify/assert" +) + +func TestDeploymentCommitAndLoad(t *testing.T) { + tmp := t.TempDir() + filename := tmp + "/state.json" + s := New(filename, 2, 2) + err := s.Commit() + assert.Nil(t, err) + + s1 := New(filename, 2, 2) + err = s1.Load() + assert.Nil(t, err) + assert.Equal(t, 0, len(s.Deployments)) + + s.DeploymentInsert(deployment.Deployment{UUID: "1", Operation: "switch"}) + s.Commit() + assert.Nil(t, err) + + s1 = New(filename, 2, 2) + err = s1.Load() + assert.Nil(t, err) + assert.Equal(t, 1, len(s.Deployments)) +} + +func TestLastDeployment(t *testing.T) { + s := New("", 2, 2) + ok, _ := s.LastDeployment() + assert.False(t, ok) + s.DeploymentInsert(deployment.Deployment{UUID: "1", Operation: "switch"}) + s.DeploymentInsert(deployment.Deployment{UUID: "2", Operation: "switch"}) + ok, last := s.LastDeployment() + assert.True(t, ok) + assert.Equal(t, "2", last.UUID) +} + +func TestDeploymentInsert(t *testing.T) { + s := New("", 2, 2) + var hasEvicted bool + var evicted deployment.Deployment + hasEvicted, _ = s.DeploymentInsert(deployment.Deployment{UUID: "1", Operation: "switch"}) + assert.False(t, hasEvicted) + hasEvicted, _ = s.DeploymentInsert(deployment.Deployment{UUID: "2", Operation: "switch"}) + assert.False(t, hasEvicted) + hasEvicted, evicted = s.DeploymentInsert(deployment.Deployment{UUID: "3", Operation: "switch"}) + assert.True(t, hasEvicted) + assert.Equal(t, "1", evicted.UUID) + + hasEvicted, _ = s.DeploymentInsert(deployment.Deployment{UUID: "4", Operation: "testing"}) + assert.False(t, hasEvicted) + hasEvicted, _ = s.DeploymentInsert(deployment.Deployment{UUID: "5", Operation: "testing"}) + assert.False(t, hasEvicted) + hasEvicted, evicted = s.DeploymentInsert(deployment.Deployment{UUID: "6", Operation: "testing"}) + assert.True(t, hasEvicted) + assert.Equal(t, "4", evicted.UUID) + + hasEvicted, evicted = s.DeploymentInsert(deployment.Deployment{UUID: "7", Operation: "switch"}) + assert.True(t, hasEvicted) + assert.Equal(t, "2", evicted.UUID) +}