switch deletion column in changefeed to a category column

Signed-off-by: David Lawrence <[email protected]> (github: endophage)

migrations/server/mysql/0005_changefeed.up.sql

@@ -1,11 +1,19 @@
+CREATE TABLE `change_category` (
+    `category` VARCHAR(20) NOT NULL,
+    PRIMARY KEY (`category`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+INSERT INTO `change_category` VALUES ("update"), ("deletion");
+
 CREATE TABLE `changefeed` (
     `id` int(11) NOT NULL AUTO_INCREMENT,
     `created_at` timestamp DEFAULT CURRENT_TIMESTAMP,
     `gun` varchar(255) NOT NULL,
     `version` int(11) NOT NULL,
     `sha256` CHAR(64) DEFAULT NULL,
-    `deletion` BOOLEAN DEFAULT FALSE,
+    `category` VARCHAR(20) NOT NULL DEFAULT "update",
     PRIMARY KEY (`id`),
+    FOREIGN KEY (`category`) REFERENCES `change_category` (`category`),
     INDEX `idx_changefeed_gun` (`gun`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 

migrations/server/postgresql/0002_changefeed.up.sql

@@ -1,10 +1,16 @@
+CREATE TABLE "change_category" (
+    "category" VARCHAR(20) PRIMARY KEY
+);
+
+INSERT INTO "change_category" VALUES ('update'), ('deletion');
+
 CREATE TABLE "changefeed" (
     "id" serial PRIMARY KEY,
     "created_at" timestamp DEFAULT CURRENT_TIMESTAMP,
     "gun" varchar(255) NOT NULL,
     "version" integer NOT NULL,
     "sha256" CHAR(64) DEFAULT NULL,
-    "deletion" BOOLEAN DEFAULT FALSE
+    "category" VARCHAR(20) NOT NULL DEFAULT 'update' REFERENCES "change_category"
 );
 
 CREATE INDEX "idx_changefeed_gun" ON "changefeed" ("gun");

server/handlers/changefeed.go

@@ -2,6 +2,7 @@ package handlers
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"strconv"
 
@@ -69,7 +70,9 @@ func checkChangefeedInputs(logger ctxu.Logger, s interface{}, r string) (
 	pageSize, err = strconv.ParseInt(r, 10, 32)
 	if err != nil {
 		logger.Errorf("400 GET invalid pageSize: %s", r)
-		err = errors.ErrInvalidParams.WithDetail("invalid records parameter, must be an integer >= 0")
+		err = errors.ErrInvalidParams.WithDetail(
+			fmt.Sprintf("invalid records parameter: %s", err.Error()),
+		)
 		return
 	}
 	if pageSize == 0 {

server/storage/errors.go

@@ -40,13 +40,3 @@ type ErrNoKey struct {
 func (err ErrNoKey) Error() string {
 	return fmt.Sprintf("Error, no timestamp key found for %s", err.gun)
 }
-
-// ErrBadChangeID indicates the change ID provided by the user is not
-// valid for some reason, i.e. it is out of bounds
-type ErrBadChangeID struct {
-	id string
-}
-
-func (err ErrBadChangeID) Error() string {
-	return fmt.Sprintf("Error, the change ID \"%s\" is not valid", err.id)
-}

server/storage/interface.go

@@ -45,9 +45,9 @@ type MetaStore interface {
 	// on the assumption that if a user provides an ID, they've seen that change.
 	// If changeID is 0, it starts from the
 	// beginning, and if changeID is -1, it starts from the most recent
-	// change. The number of results returned is limited by pageSize.
-	// Reversed indicates we are fetching pages going backwards in time, the
-	// default being to fetch pageSize from changeID going forwards in time.
+	// change. The number of results returned is limited by records.
+	// If records is negative, we will return that number of changes preceding
+	// the given changeID.
 	// The returned []Change should always be ordered oldest to newest.
 	GetChanges(changeID string, records int, filterName string) ([]Change, error)
 }

server/storage/memory.go

@@ -90,6 +90,7 @@ func (st *MemStorage) writeChange(gun string, version int, checksum string) {
 		Version:   version,
 		SHA256:    checksum,
 		CreatedAt: time.Now(),
+		Category:  changeCategoryUpdate,
 	}
 	st.changes = append(st.changes, c)
 }
@@ -172,16 +173,21 @@ func (st *MemStorage) GetChecksum(gun, role, checksum string) (*time.Time, []byt
 func (st *MemStorage) Delete(gun string) error {
 	st.lock.Lock()
 	defer st.lock.Unlock()
+	l := len(st.tufMeta)
 	for k := range st.tufMeta {
 		if strings.HasPrefix(k, gun) {
 			delete(st.tufMeta, k)
 		}
 	}
+	if l == len(st.tufMeta) {
+		// we didn't delete anything, don't write change.
+		return nil
+	}
 	delete(st.checksums, gun)
 	c := Change{
 		ID:        uint(len(st.changes) + 1),
 		GUN:       gun,
-		Deletion:  true,
+		Category:  changeCategoryDeletion,
 		CreatedAt: time.Now(),
 	}
 	st.changes = append(st.changes, c)

server/storage/memory_test.go

@@ -5,7 +5,6 @@ package storage
 import (
 	"testing"
 
-	"github.com/docker/notary/tuf/data"
 	"github.com/stretchr/testify/require"
 )
 
@@ -95,131 +94,5 @@ func TestGetChecksumNotFound(t *testing.T) {
 func TestMemoryGetChanges(t *testing.T) {
 	s := NewMemStorage()
 
-	// non-int changeID
-	c, err := s.GetChanges("foo", 10, "")
-	require.Error(t, err)
-	require.Len(t, c, 0)
-
-	// add some records
-	require.NoError(t, s.UpdateMany("alpine", []MetaUpdate{
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 1,
-			Data:    []byte{},
-		},
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 2,
-			Data:    []byte{},
-		},
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 3,
-			Data:    []byte{},
-		},
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 4,
-			Data:    []byte{},
-		},
-	}))
-	require.NoError(t, s.UpdateMany("busybox", []MetaUpdate{
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 1,
-			Data:    []byte{},
-		},
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 2,
-			Data:    []byte{},
-		},
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 3,
-			Data:    []byte{},
-		},
-		{
-			Role:    data.CanonicalTimestampRole,
-			Version: 4,
-			Data:    []byte{},
-		},
-	}))
-
-	c, err = s.GetChanges("0", 8, "")
-	require.NoError(t, err)
-	require.Len(t, c, 8)
-	for i := 0; i < 4; i++ {
-		require.Equal(t, uint(i+1), c[i].ID)
-		require.Equal(t, "alpine", c[i].GUN)
-		require.Equal(t, i+1, c[i].Version)
-	}
-	for i := 4; i < 8; i++ {
-		require.Equal(t, uint(i+1), c[i].ID)
-		require.Equal(t, "busybox", c[i].GUN)
-		require.Equal(t, i-3, c[i].Version)
-	}
-
-	c, err = s.GetChanges("0", 4, "")
-	require.NoError(t, err)
-	require.Len(t, c, 4)
-	for i := 0; i < 4; i++ {
-		require.Equal(t, uint(i+1), c[i].ID)
-		require.Equal(t, "alpine", c[i].GUN)
-		require.Equal(t, i+1, c[i].Version)
-	}
-
-	c, err = s.GetChanges("-1", 4, "")
-	require.NoError(t, err)
-	require.Len(t, c, 4)
-	for i := 0; i < 4; i++ {
-		require.Equal(t, uint(i+5), c[i].ID)
-		require.Equal(t, "busybox", c[i].GUN)
-		require.Equal(t, i+1, c[i].Version)
-	}
-
-	c, err = s.GetChanges("10", 4, "")
-	require.NoError(t, err)
-	require.Len(t, c, 0)
-
-	c, err = s.GetChanges("10", -4, "")
-	require.NoError(t, err)
-	require.Len(t, c, 4)
-	for i := 0; i < 4; i++ {
-		require.Equal(t, uint(i+5), c[i].ID)
-		require.Equal(t, "busybox", c[i].GUN)
-		require.Equal(t, i+1, c[i].Version)
-	}
-
-	c, err = s.GetChanges("7", -4, "")
-	require.NoError(t, err)
-	require.Len(t, c, 4)
-	for i := 0; i < 2; i++ {
-		require.Equal(t, uint(i+3), c[i].ID)
-		require.Equal(t, "alpine", c[i].GUN)
-		require.Equal(t, i+3, c[i].Version)
-	}
-	for i := 2; i < 4; i++ {
-		require.Equal(t, uint(i+3), c[i].ID)
-		require.Equal(t, "busybox", c[i].GUN)
-		require.Equal(t, i-1, c[i].Version)
-	}
-
-	c, err = s.GetChanges("7", -2, "alpine")
-	require.NoError(t, err)
-	require.Len(t, c, 2)
-	for i := 0; i < 2; i++ {
-		require.Equal(t, uint(i+3), c[i].ID)
-		require.Equal(t, "alpine", c[i].GUN)
-		require.Equal(t, i+3, c[i].Version)
-	}
-
-	c, err = s.GetChanges("0", 8, "busybox")
-	require.NoError(t, err)
-	require.Len(t, c, 4)
-	for i := 0; i < 4; i++ {
-		require.Equal(t, uint(i+5), c[i].ID)
-		require.Equal(t, "busybox", c[i].GUN)
-		require.Equal(t, i+1, c[i].Version)
-	}
+	testGetChanges(t, s)
 }

server/storage/sql_models.go

@@ -6,6 +6,11 @@ import (
 	"github.com/jinzhu/gorm"
 )
 
+const (
+	changeCategoryUpdate   = "update"
+	changeCategoryDeletion = "deletion"
+)
+
 // TUFFile represents a TUF file in the database
 type TUFFile struct {
 	gorm.Model
@@ -23,12 +28,12 @@ func (g TUFFile) TableName() string {
 
 // Change defines the the fields required for an object in the changefeed
 type Change struct {
-	ID        uint `gorm:"primary_key" sql:"not null"`
+	ID        uint `gorm:"primary_key" sql:"not null" json:",string"`
 	CreatedAt time.Time
 	GUN       string `gorm:"column:gun" sql:"type:varchar(255);not null"`
 	Version   int    `sql:"not null"`
 	SHA256    string `gorm:"column:sha256" sql:"type:varchar(64);"`
-	Deletion  bool
+	Category  string `sql:"type:varchar(20);not null;"`
 }
 
 // TableName sets a specific table name for Changefeed

server/storage/sqldb.go

@@ -167,9 +167,10 @@ func (db *SQLStorage) UpdateMany(gun string, updates []MetaUpdate) error {
 
 func (db *SQLStorage) writeChangefeed(tx *gorm.DB, gun string, version int, checksum string) error {
 	c := &Change{
-		GUN:     gun,
-		Version: version,
-		SHA256:  checksum,
+		GUN:      gun,
+		Version:  version,
+		SHA256:   checksum,
+		Category: changeCategoryUpdate,
 	}
 	return tx.Create(c).Error
 }
@@ -229,7 +230,7 @@ func (db *SQLStorage) Delete(gun string) error {
 		}
 		c := &Change{
 			GUN:      gun,
-			Deletion: true,
+			Category: changeCategoryDeletion,
 		}
 		return tx.Create(c).Error
 	}(); err != nil {