Skip to content

Commit

Permalink
Merge pull request #398 from sgotti/new_db_schema
Browse files Browse the repository at this point in the history
*: new db schema
  • Loading branch information
sgotti authored Apr 13, 2023
2 parents 8859b18 + 8423c70 commit c486af0
Show file tree
Hide file tree
Showing 144 changed files with 20,956 additions and 6,447 deletions.
22 changes: 21 additions & 1 deletion .agola/config.jsonnet
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ local go_runtime(version, arch) = {
{
image: 'golang:' + version + '-buster',
},
{
image: 'postgres',
environment: {
POSTGRES_PASSWORD: 'password',
},
},
],
};

Expand Down Expand Up @@ -36,7 +42,21 @@ local task_build_go(version, arch) = {
{ type: 'run', command: 'golangci-lint run --deadline 5m' },
{ type: 'run', name: 'build docker/k8s drivers tests binary', command: 'CGO_ENABLED=0 go test -c ./internal/services/executor/driver -o ./bin/docker-tests' },
{ type: 'run', name: 'build integration tests binary', command: 'go test -tags "sqlite_unlock_notify" -c ./tests -o ./bin/integration-tests' },
{ type: 'run', name: 'run tests', command: 'SKIP_DOCKER_TESTS=1 SKIP_K8S_TESTS=1 go test -tags "sqlite_unlock_notify" -v -count 1 -parallel 5 $(go list ./... | grep -v /tests)' },
{ type: 'run', name: 'run tests (sqlite3)',
environment: {
DB_TYPE: "sqlite3",
SKIP_DOCKER_TESTS: "1",
SKIP_K8S_TESTS: "1",
},
command: 'go test -tags "sqlite_unlock_notify" -v -count 1 -parallel 5 $(go list ./... | grep -v /tests)' },
{ type: 'run', name: 'run tests (postgres)',
environment: {
DB_TYPE: "postgres",
PG_CONNSTRING: "postgres://postgres:postgres@localhost/%s?sslmode=disable",
SKIP_DOCKER_TESTS: "1",
SKIP_K8S_TESTS: "1",
},
command: 'go test -tags "sqlite_unlock_notify" -v -count 1 -parallel 5 $(go list ./... | grep -v /tests)' },
{ type: 'run', name: 'fetch gitea binary for integration tests', command: 'curl -L https://github.com/go-gitea/gitea/releases/download/v1.15.11/gitea-1.15.11-linux-amd64 -o ./bin/gitea && chmod +x ./bin/gitea' },
{ type: 'save_to_workspace', contents: [{ source_dir: './bin', dest_dir: '/bin/', paths: ['*'] }] },
],
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,11 @@ webbundle/bindata.go: go-bindata $(WEBDISTPATH)

.PHONY: generate
generate: generators
go generate ./...
go generate $(PROJDIR)/...

.PHONY: generators
generators:
GOBIN=$(PROJDIR)/tools/bin go install ./internal/generators
go build -o $(PROJDIR)/tools/bin/dbgenerator ./internal/generators/db

.PHONY: docker-agola
docker-agola:
Expand Down
110 changes: 80 additions & 30 deletions cmd/agola/cmd/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,22 @@ package cmd

import (
"context"
"os"

"github.com/rs/zerolog/log"
"github.com/sorintlab/errors"
"github.com/spf13/cobra"

"agola.io/agola/internal/migration"
"agola.io/agola/internal/services/config"
csdb "agola.io/agola/internal/services/configstore/db"
rsdb "agola.io/agola/internal/services/runservice/db"
"agola.io/agola/internal/sqlg/lock"
"agola.io/agola/internal/sqlg/manager"
"agola.io/agola/internal/sqlg/sql"
)

var cmdMigrate = &cobra.Command{
Use: "migrate",
Short: "migrate from an old data format export to the new data format",
Short: "migrate component database to latest version",
Run: func(cmd *cobra.Command, args []string) {
if err := migrate(cmd, args); err != nil {
log.Fatal().Err(err).Send()
Expand All @@ -36,61 +40,107 @@ var cmdMigrate = &cobra.Command{
}

type migrateOptions struct {
config string
serviceName string
inFilePath string
outFilePath string
}

var migrateOpts migrateOptions

func init() {
flags := cmdMigrate.Flags()

flags.StringVar(&migrateOpts.config, "config", "./config.yml", "config file path")
flags.StringVar(&migrateOpts.serviceName, "service", "", "service name (runservice or configstore)")
flags.StringVar(&migrateOpts.inFilePath, "in", "-", "input file path")
flags.StringVar(&migrateOpts.outFilePath, "out", "-", "output file path")

cmdAgola.AddCommand(cmdMigrate)
}

func migrate(cmd *cobra.Command, args []string) error {
ctx := context.Background()

if migrateOpts.serviceName != "runservice" && migrateOpts.serviceName != "configstore" {
return errors.Errorf("service option must be runservice or configstore")
}

var r *os.File
if migrateOpts.inFilePath == "-" {
r = os.Stdin
} else {
var err error
r, err = os.Open(migrateOpts.inFilePath)
if err != nil {
return errors.WithStack(err)
}
components := []string{migrateOpts.serviceName}

c, err := config.Parse(migrateOpts.config, components)
if err != nil {
return errors.Wrapf(err, "config error")
}

var w *os.File
if migrateOpts.outFilePath == "-" {
w = os.Stdout
} else {
var sdb *sql.DB
var d manager.DB
switch migrateOpts.serviceName {
case "runservice":
var err error
w, err = os.Create(migrateOpts.outFilePath)

dbConf := c.Runservice.DB

sdb, err = sql.NewDB(dbConf.Type, dbConf.ConnString)
if err != nil {
return errors.WithStack(err)
return errors.Wrapf(err, "new db error")
}
}

log.Info().Msgf("migrating %s", migrateOpts.serviceName)
switch migrateOpts.serviceName {
case "runservice":
if err := migration.MigrateRunService(context.Background(), r, w); err != nil {
return errors.WithStack(err)
d, err = rsdb.NewDB(log.Logger, sdb)
if err != nil {
return errors.Wrapf(err, "new db error")
}

case "configstore":
if err := migration.MigrateConfigStore(context.Background(), r, w); err != nil {
return errors.WithStack(err)
var err error

dbConf := c.Configstore.DB

sdb, err = sql.NewDB(dbConf.Type, dbConf.ConnString)
if err != nil {
return errors.Wrapf(err, "new db error")
}

d, err = csdb.NewDB(log.Logger, sdb)
if err != nil {
return errors.Wrapf(err, "new db error")
}
}

var lf lock.LockFactory
switch d.DBType() {
case sql.Sqlite3:
ll := lock.NewLocalLocks()
lf = lock.NewLocalLockFactory(ll)
case sql.Postgres:
lf = lock.NewPGLockFactory(sdb)
default:
return errors.Errorf("unknown db type %q", d.DBType())
}

dbm := manager.NewDBManager(log.Logger, d, lf)

log.Info().Msgf("migrating service %s", migrateOpts.serviceName)

curDBVersion, err := dbm.GetVersion(ctx)
if err != nil {
return errors.WithStack(err)
}

if err := dbm.CheckVersion(curDBVersion, d.Version()); err != nil {
return errors.WithStack(err)
}

migrationRequired, err := dbm.CheckMigrationRequired(curDBVersion, d.Version())
if err != nil {
return errors.WithStack(err)
}
if !migrationRequired {
log.Info().Msgf("db already at latest version: %d", curDBVersion)
return nil
}

if err := dbm.Migrate(ctx); err != nil {
return errors.Wrap(err, "migrate db error")
}

log.Info().Msgf("db migrated to version: %d", d.Version())

return nil
}
116 changes: 116 additions & 0 deletions cmd/agola/cmd/migrateexport.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
// Copyright 2022 Sorint.lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
"context"
"os"

"github.com/rs/zerolog/log"
"github.com/sorintlab/errors"
"github.com/spf13/cobra"

migration248a9e0ad "agola.io/agola/internal/migration/248a9e0ad"
migrationv07x "agola.io/agola/internal/migration/v0.7.x"
)

var cmdMigrateExport = &cobra.Command{
Use: "migrateexport",
Short: "migrate from an old data format export to the new data format",
Run: func(cmd *cobra.Command, args []string) {
if err := migrateExport(cmd, args); err != nil {
log.Fatal().Err(err).Send()
}
},
}

type migrateExportOptions struct {
serviceName string
sourceVersion string
inFilePath string
outFilePath string
}

var migrateExportOpts migrateExportOptions

func init() {
flags := cmdMigrateExport.Flags()

flags.StringVar(&migrateExportOpts.serviceName, "service", "", "service name (runservice or configstore)")
flags.StringVar(&migrateExportOpts.sourceVersion, "source-version", "v0.7.x", "export source version (v0.7.x or 248a9e0ad)")
flags.StringVar(&migrateExportOpts.inFilePath, "in", "-", "input file path")
flags.StringVar(&migrateExportOpts.outFilePath, "out", "-", "output file path")

cmdAgola.AddCommand(cmdMigrateExport)
}

func migrateExport(cmd *cobra.Command, args []string) error {
if migrateExportOpts.serviceName != "runservice" && migrateExportOpts.serviceName != "configstore" {
return errors.Errorf("service option must be runservice or configstore")
}
if migrateExportOpts.sourceVersion != "v0.7.x" && migrateExportOpts.sourceVersion != "248a9e0ad" {
return errors.Errorf("source version option must be v0.7.x or 248a9e0ad")
}

var r *os.File
if migrateExportOpts.inFilePath == "-" {
r = os.Stdin
} else {
var err error
r, err = os.Open(migrateExportOpts.inFilePath)
if err != nil {
return errors.WithStack(err)
}
}

var w *os.File
if migrateExportOpts.outFilePath == "-" {
w = os.Stdout
} else {
var err error
w, err = os.Create(migrateExportOpts.outFilePath)
if err != nil {
return errors.WithStack(err)
}
}

log.Info().Msgf("migrating export of service %s", migrateExportOpts.serviceName)
switch migrateExportOpts.serviceName {
case "runservice":
switch migrateExportOpts.sourceVersion {
case "v0.7.x":
if err := migrationv07x.MigrateRunService(context.Background(), r, w); err != nil {
return errors.WithStack(err)
}
case "248a9e0ad":
if err := migration248a9e0ad.MigrateRunService(context.Background(), r, w); err != nil {
return errors.WithStack(err)
}
}
case "configstore":
switch migrateExportOpts.sourceVersion {
case "v0.7.x":
if err := migrationv07x.MigrateConfigStore(context.Background(), r, w); err != nil {
return errors.WithStack(err)
}
case "248a9e0ad":
if err := migration248a9e0ad.MigrateConfigStore(context.Background(), r, w); err != nil {
return errors.WithStack(err)
}
}
}

return nil
}
10 changes: 7 additions & 3 deletions doc/migrating_from_v0.7.x.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ We suggest to test this migration on a test environment before doing this on you

`curl -v http://$CONFIGSTOREHOST:PORT/api/v1alpha/export > /tmp/configstore-export`

1. Generate the migrated data using the new agola binary migrate command:
1. Generate the migrated data using the new agola binary `migrateexport` command:

`cat /tmp/runservice-export | ./tmp/agola migrate --service runservice > /tmp/runservice-migrated`
`cat /tmp/runservice-export | ./tmp/agola migrateexport --service runservice > /tmp/runservice-migrated`

`cat /tmp/runservice-export | ./tmp/agola migrate --service configstore > /tmp/configstore-migrated`
`cat /tmp/runservice-export | ./tmp/agola migratexporte --service configstore > /tmp/configstore-migrated`

1. Update the agola binaries on your environment or use a test enviroment and start only the runservice and configstore.
1. Update the agola config file and remove the runservice, configstore, notification service etcd entries and add the db entries. Every component should have its own dedicated database. DO NOT use the same database for all the services. For PostgresSQL it can be the same postgres instance but with different databases.
Expand All @@ -42,3 +42,7 @@ We suggest to test this migration on a test environment before doing this on you
`curl -v -XDELETE http://$NEWCONFIGSTOREHOST:PORT/api/v1alpha/maintenance`

1. Start the gateway and test if the migration was successfull

## Migrating from db after commit 248a9e0ad and before v0.8.x

Use the same steps but provide the `migrateexport` option `--source-version 248a9e0ad`
Loading

0 comments on commit c486af0

Please sign in to comment.