switchover: refactor and move SWO toolset to UI

Makefile

@@ -51,6 +51,9 @@ Makefile.binaries.mk: devtools/genmake/*
 $(BIN_DIR)/tools/protoc: protoc.version
 	go run ./devtools/gettool -t protoc -v $(shell cat protoc.version) -o $@
 
+$(BIN_DIR)/tools/sqlc: sqlc.version
+	go run ./devtools/gettool -t sqlc -v $(shell cat sqlc.version) -o $@
+
 $(BIN_DIR)/tools/prometheus: prometheus.version
 	go run ./devtools/gettool -t prometheus -v $(shell cat prometheus.version) -o $@
 
@@ -98,6 +101,9 @@ cy-wide-prod-run: web/src/build/static/app.js cypress
 cy-mobile-prod-run: web/src/build/static/app.js cypress
 	$(MAKE) $(MFLAGS) cy-mobile-prod CY_ACTION=run CONTAINER_TOOL=$(CONTAINER_TOOL) BUNDLE=1
 
+swo/swodb/queries.sql.go: $(BIN_DIR)/tools/sqlc sqlc.yaml swo/*/*.sql migrate/migrations/*.sql
+	$(BIN_DIR)/tools/sqlc generate
+
 web/src/schema.d.ts: graphql2/schema.graphql node_modules web/src/genschema.go
 	go generate ./web/src
 
@@ -114,6 +120,14 @@ start-prod: web/src/build/static/app.js $(BIN_DIR)/tools/prometheus ## Start the
 	$(MAKE) $(MFLAGS) bin/goalert BUNDLE=1
 	go run ./devtools/runproc -f Procfile.prod -l Procfile.local
 
+
+start-swo: bin/psql-lite bin/goalert bin/waitfor bin/runproc node_modules web/src/schema.d.ts $(BIN_DIR)/tools/prometheus ## Start the developer version of the application in switchover mode (SWO)
+	./bin/waitfor -timeout 1s  "$(DB_URL)" || make postgres
+	./bin/goalert migrate --db-url=postgres://goalert@localhost/goalert
+	./bin/psql-lite -d postgres://goalert@localhost -c "update switchover_state set current_state = 'idle'; truncate table switchover_log; drop database if exists goalert2; create database goalert2;"
+	./bin/goalert migrate --db-url=postgres://goalert@localhost/goalert2
+	GOALERT_VERSION=$(GIT_VERSION) ./bin/runproc -f Procfile.swo -l Procfile.local
+
 start-integration: web/src/build/static/app.js bin/goalert bin/psql-lite bin/waitfor bin/runproc $(BIN_DIR)/tools/prometheus
 	./bin/waitfor -timeout 1s  "$(DB_URL)" || make postgres
 	./bin/psql-lite -d "$(DB_URL)" -c 'DROP DATABASE IF EXISTS $(INT_DB); CREATE DATABASE $(INT_DB);'
@@ -164,7 +178,8 @@ pkg/sysapi/sysapi_grpc.pb.go: pkg/sysapi/sysapi.proto $(BIN_DIR)/tools/protoc-ge
 pkg/sysapi/sysapi.pb.go: pkg/sysapi/sysapi.proto $(BIN_DIR)/tools/protoc-gen-go $(BIN_DIR)/tools/protoc
 	PATH="$(BIN_DIR)/tools" protoc --go_out=. --go_opt=paths=source_relative pkg/sysapi/sysapi.proto
 
-generate: node_modules pkg/sysapi/sysapi.pb.go pkg/sysapi/sysapi_grpc.pb.go
+generate: node_modules pkg/sysapi/sysapi.pb.go pkg/sysapi/sysapi_grpc.pb.go $(BIN_DIR)/tools/sqlc
+	$(BIN_DIR)/tools/sqlc generate
 	go generate ./...
 
 
@@ -228,6 +243,7 @@ postgres: bin/waitfor
 		-e POSTGRES_USER=goalert \
 		-e POSTGRES_HOST_AUTH_METHOD=trust \
 		--name goalert-postgres \
+		--shm-size 1g \
 		-p 5432:5432 \
 		docker.io/library/postgres:13-alpine && ./bin/waitfor "$(DB_URL)" && make regendb) || ($(CONTAINER_TOOL) start goalert-postgres && ./bin/waitfor "$(DB_URL)")
 

Makefile.binaries.mk

@@ -5,7 +5,7 @@
 
 BIN_DIR=bin
 GO_DEPS := Makefile.binaries.mk $(shell find . -path ./web/src -prune -o -path ./vendor -prune -o -path ./.git -prune -o -type f -name "*.go" -print) go.sum
-GO_DEPS += migrate/migrations/ migrate/migrations/*.sql web/index.html graphql2/graphqlapp/slack.manifest.yaml
+GO_DEPS += migrate/migrations/ migrate/migrations/*.sql web/index.html graphql2/graphqlapp/slack.manifest.yaml swo/*/*.sql
 GO_DEPS += graphql2/mapconfig.go graphql2/maplimit.go graphql2/generated.go graphql2/models_gen.go
 GO_DEPS += web/explore.html web/live.js
 

Procfile.swo

@@ -0,0 +1,39 @@
+build: while true; do make -qs bin/goalert || make bin/goalert || (echo '\033[0;31mBuild Failure'; sleep 3); sleep 0.1; done
+
+@watch-file=./bin/goalert
+goalert: ./bin/goalert -l=localhost:3040 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435 --listen-sysapi=localhost:1234 --listen-prometheus=localhost:2112 --db-url-next=postgres://goalert@localhost:5435/goalert2
+
+smtp: go run github.com/mailhog/MailHog -ui-bind-addr=localhost:8025 -api-bind-addr=localhost:8025 -smtp-bind-addr=localhost:1025 | grep -v KEEPALIVE
+
+prom: bin/tools/prometheus --log.level=warn --config.file=devtools/prometheus/prometheus-swo.yml --storage.tsdb.path=bin/prom-data/ --web.listen-address=localhost:9090
+
+@watch-file=./web/src/esbuild.config.js
+ui: yarn workspace goalert-web run esbuild --watch
+
+@watch-file=./bin/goalert
+ga2: ./bin/goalert -l=localhost:3050 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2113 --db-url-next=postgres://goalert@localhost:5435/goalert2 --api-only
+
+@watch-file=./bin/goalert
+ga3: ./bin/goalert -l=localhost:3051 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2114 --db-url-next=postgres://goalert@localhost:5435/goalert2 --api-only
+
+@watch-file=./bin/goalert
+ga4: ./bin/goalert -l=localhost:3052 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2115 --db-url-next=postgres://goalert@localhost:5435/goalert2 --api-only
+
+@watch-file=./bin/goalert
+ga5: ./bin/goalert -l=localhost:3053 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2116 --db-url-next=postgres://goalert@localhost:5435/goalert2 --api-only
+
+@watch-file=./bin/goalert
+ga6: ./bin/goalert -l=localhost:3054 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2117 --db-url-next=postgres://goalert@localhost:5435/goalert2 --api-only
+
+@watch-file=./bin/goalert
+ga7: ./bin/goalert -l=localhost:3055 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2118 --db-url-next=postgres://goalert@localhost:5435/goalert2
+
+@watch-file=./bin/goalert
+ga8: ./bin/goalert -l=localhost:3056 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2119 --db-url-next=postgres://goalert@localhost:5435/goalert2
+
+@watch-file=./bin/goalert
+ga9: ./bin/goalert -l=localhost:3057 --ui-dir=web/src/build --db-url=postgres://goalert@localhost:5435/goalert?sslmode=disable --listen-prometheus=localhost:2120 --db-url-next=postgres://goalert@localhost:5435/goalert2
+
+proxy: go run ./devtools/simpleproxy -addr localhost:3030 /=http://localhost:3040,http://localhost:3050,http://localhost:3051,http://localhost:3052,http://localhost:3053,http://localhost:3054,http://localhost:3055,http://localhost:3056,http://localhost:3057
+
+slow: go run github.com/kffl/speedbump --host localhost --port=5435 --latency=10ms --saw-amplitude=25ms --saw-period=1s localhost:5432

app/app.go

@@ -77,9 +77,8 @@ type App struct {
 	sysAPISrv *grpc.Server
 	hSrv      *health.Server
 
-	srv         *http.Server
-	requestLock *contextLocker
-	startupErr  error
+	srv        *http.Server
+	startupErr error
 
 	notificationManager *notification.Manager
 	Engine              *engine.Engine
@@ -183,8 +182,6 @@ func NewApp(c Config, db *sql.DB) (*App, error) {
 		db:     db,
 		cfg:    c,
 		doneCh: make(chan struct{}),
-
-		requestLock: newContextLocker(),
 	}
 
 	gCfg := &gorm.Config{
@@ -263,9 +260,3 @@ func (a *App) URL() string {
 func (a *App) Status() lifecycle.Status {
 	return a.mgr.Status()
 }
-
-// ActiveRequests returns the current number of active
-// requests, not including pending ones during pause.
-func (a *App) ActiveRequests() int {
-	return a.requestLock.RLockCount()
-}

app/cmd.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/jackc/pgx/v4/stdlib"
 	toml "github.com/pelletier/go-toml"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -24,8 +23,7 @@ import (
 	"github.com/target/goalert/migrate"
 	"github.com/target/goalert/permission"
 	"github.com/target/goalert/remotemonitor"
-	"github.com/target/goalert/switchover"
-	"github.com/target/goalert/switchover/dbsync"
+	"github.com/target/goalert/swo"
 	"github.com/target/goalert/user"
 	"github.com/target/goalert/util"
 	"github.com/target/goalert/util/log"
@@ -85,82 +83,58 @@ var RootCmd = &cobra.Command{
 			return err
 		}
 
-		wrappedDriver := sqldrv.NewRetryDriver(&stdlib.Driver{}, 10)
-
-		u, err := url.Parse(cfg.DBURL)
-		if err != nil {
-			return errors.Wrap(err, "parse old URL")
-		}
-		q := u.Query()
-		if cfg.DBURLNext != "" {
-			q.Set("application_name", fmt.Sprintf("GoAlert %s (S/O Mode)", version.GitVersion()))
-		} else {
-			q.Set("application_name", fmt.Sprintf("GoAlert %s", version.GitVersion()))
-		}
-		q.Set("enable_seqscan", "off")
-		u.RawQuery = q.Encode()
-		cfg.DBURL = u.String()
-
-		if cfg.APIOnly {
-			err = migrate.VerifyAll(log.WithDebug(ctx), cfg.DBURL)
-			if err != nil {
-				return errors.Wrap(err, "verify migrations")
+		doMigrations := func(url string) error {
+			if cfg.APIOnly {
+				err = migrate.VerifyAll(log.WithDebug(ctx), url)
+				if err != nil {
+					return errors.Wrap(err, "verify migrations")
+				}
+				return nil
 			}
-		} else {
+
 			s := time.Now()
-			n, err := migrate.ApplyAll(log.WithDebug(ctx), cfg.DBURL)
+			n, err := migrate.ApplyAll(log.WithDebug(ctx), url)
 			if err != nil {
 				return errors.Wrap(err, "apply migrations")
 			}
 			if n > 0 {
 				log.Logf(ctx, "Applied %d migrations in %s.", n, time.Since(s))
 			}
+
+			return nil
 		}
 
-		dbc, err := wrappedDriver.OpenConnector(cfg.DBURL)
+		err = doMigrations(cfg.DBURL)
 		if err != nil {
-			return errors.Wrap(err, "connect to postgres")
+			return err
 		}
+
 		var db *sql.DB
-		var h *switchover.Handler
 		if cfg.DBURLNext != "" {
-			u, err := url.Parse(cfg.DBURLNext)
+			err = doMigrations(cfg.DBURLNext)
 			if err != nil {
-				return errors.Wrap(err, "parse next URL")
+				return errors.Wrap(err, "nextdb")
 			}
-			q := u.Query()
-			q.Set("application_name", fmt.Sprintf("GoAlert %s (S/O Mode)", version.GitVersion()))
-			q.Set("enable_seqscan", "off")
-			u.RawQuery = q.Encode()
-			cfg.DBURLNext = u.String()
 
-			dbcNext, err := wrappedDriver.OpenConnector(cfg.DBURLNext)
+			mgr, err := swo.NewManager(swo.Config{OldDBURL: cfg.DBURL, NewDBURL: cfg.DBURLNext, CanExec: !cfg.APIOnly, Logger: cfg.Logger})
 			if err != nil {
-				return errors.Wrap(err, "connect to postres (next)")
+				return errors.Wrap(err, "init switchover handler")
 			}
-			h, err = switchover.NewHandler(ctx, l, dbc, dbcNext, cfg.DBURL, cfg.DBURLNext)
+			db = mgr.DB()
+			cfg.SWO = mgr
+		} else {
+			db, err = sqldrv.NewDB(cfg.DBURL, fmt.Sprintf("GoAlert %s", version.GitVersion()))
 			if err != nil {
-				return errors.Wrap(err, "init changeover handler")
+				return errors.Wrap(err, "connect to postgres")
 			}
-			db = h.DB()
-		} else {
-			db = sql.OpenDB(dbc)
 		}
 
 		app, err := NewApp(cfg, db)
 		if err != nil {
 			return errors.Wrap(err, "init app")
 		}
-		if h != nil {
-			h.SetApp(app)
-		}
 
-		go handleShutdown(ctx, func(ctx context.Context) error {
-			if h != nil {
-				h.Abort()
-			}
-			return app.Shutdown(ctx)
-		})
+		go handleShutdown(ctx, app.Shutdown)
 
 		// trigger engine cycles by process signal
 		trigCh := make(chan os.Signal, 1)
@@ -351,23 +325,6 @@ Migration: %s (#%d)
 		},
 	}
 
-	switchCmd = &cobra.Command{
-		Use:   "switchover-shell",
-		Short: "Start a the switchover shell, used to initiate, control, and monitor a DB switchover operation.",
-		RunE: func(cmd *cobra.Command, args []string) error {
-			cfg, err := getConfig(cmd.Context())
-			if err != nil {
-				return err
-			}
-
-			if cfg.DBURLNext == "" {
-				return validation.NewFieldError("DBURLNext", "must not be empty for switchover")
-			}
-
-			return dbsync.RunShell(log.FromContext(cmd.Context()), cfg.DBURL, cfg.DBURLNext)
-		},
-	}
-
 	monitorCmd = &cobra.Command{
 		Use:   "monitor",
 		Short: "Start a remote-monitoring process that functionally tests alerts.",
@@ -731,7 +688,7 @@ func init() {
 	RootCmd.Flags().String("region-name", def.RegionName, "Name of region for message processing (case sensitive). Only one instance per-region-name will process outgoing messages.")
 
 	RootCmd.PersistentFlags().String("db-url", def.DBURL, "Connection string for Postgres.")
-	RootCmd.PersistentFlags().String("db-url-next", def.DBURLNext, "Connection string for the *next* Postgres server (enables DB switch-over mode).")
+	RootCmd.PersistentFlags().String("db-url-next", def.DBURLNext, "Connection string for the *next* Postgres server (enables DB switchover mode).")
 
 	RootCmd.Flags().String("jaeger-endpoint", "", "Jaeger HTTP Thrift endpoint")
 	RootCmd.Flags().String("jaeger-agent-endpoint", "", "Instructs Jaeger exporter to send spans to jaeger-agent at this address.")
@@ -788,7 +745,7 @@ func init() {
 
 	monitorCmd.Flags().StringP("config-file", "f", "", "Configuration file for monitoring (required).")
 	initCertCommands()
-	RootCmd.AddCommand(versionCmd, testCmd, migrateCmd, exportCmd, monitorCmd, switchCmd, addUserCmd, getConfigCmd, setConfigCmd, genCerts)
+	RootCmd.AddCommand(versionCmd, testCmd, migrateCmd, exportCmd, monitorCmd, addUserCmd, getConfigCmd, setConfigCmd, genCerts)
 
 	err := viper.BindPFlags(RootCmd.Flags())
 	if err != nil {

app/config.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/target/goalert/config"
 	"github.com/target/goalert/keyring"
+	"github.com/target/goalert/swo"
 	"github.com/target/goalert/util/log"
 )
 
@@ -61,4 +62,7 @@ type Config struct {
 	// InitialConfig will be pushed into the config store
 	// if specified before the engine is started.
 	InitialConfig *config.Config
+
+	// SWO should be set to operate in switchover mode.
+	SWO *swo.Manager
 }