Skip to content

Commit

Permalink
feat: badger auto migration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
zl03jsj committed Oct 14, 2022
1 parent 86c0805 commit f921dbc
Show file tree
Hide file tree
Showing 13 changed files with 325 additions and 61 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/filecoin-project/go-cbor-util v0.0.1
github.com/filecoin-project/go-commp-utils v0.1.3
github.com/filecoin-project/go-data-transfer v1.15.2
github.com/filecoin-project/go-ds-versioning v0.1.1
github.com/filecoin-project/go-fil-commcid v0.1.0
github.com/filecoin-project/go-fil-commp-hashhash v0.1.0
github.com/filecoin-project/go-fil-markets v1.23.1
Expand Down Expand Up @@ -93,6 +94,7 @@ require (
go.uber.org/multierr v1.8.0
go.uber.org/zap v1.21.0
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f
gorm.io/driver/mysql v1.1.1
gorm.io/driver/sqlite v1.1.4
gorm.io/gorm v1.21.12
Expand Down Expand Up @@ -143,7 +145,6 @@ require (
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect
github.com/filecoin-project/go-bitfield v0.2.4 // indirect
github.com/filecoin-project/go-crypto v0.0.1 // indirect
github.com/filecoin-project/go-ds-versioning v0.1.1 // indirect
github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect
github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect
github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect
Expand Down Expand Up @@ -320,7 +321,6 @@ require (
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.1.10 // indirect
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
google.golang.org/api v0.56.0 // indirect
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
google.golang.org/grpc v1.45.0 // indirect
Expand Down
18 changes: 11 additions & 7 deletions models/badger/migrate/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,10 @@ var migrateDs = map[string][]migrateFunc{
},
DsNameCidInfoDs: {
{version: "1", mf: func(old *v220.CIDInfo) (*piecestore.CIDInfo, error) {
return old, nil
return &piecestore.CIDInfo{
CID: old.CID,
PieceBlockLocations: old.PieceBlockLocations,
}, nil
}},
},
DsNameRetrievalDealsDs: {
Expand Down Expand Up @@ -211,11 +214,11 @@ func migrateOne(ctx context.Context, name string, mfs migrateFuncSchedule, ds da
if len(oldVersion) == 0 {
dsWithOldVersion = ds
} else {
dsWithOldVersion = namespace.Wrap(ds, datastore.NewKey(oldVersion))

if oldVersion == string(targetVersion) {
log.Infof("doesn't need migration for %s, current version is:%s", name, oldVersion)
return namespace.Wrap(ds, datastore.NewKey(oldVersion)), nil
} else {
dsWithOldVersion = namespace.Wrap(ds, datastore.NewKey(oldVersion))
return dsWithOldVersion, nil
}
}

Expand Down Expand Up @@ -252,7 +255,9 @@ func Migrate(ctx context.Context, dss map[string]datastore.Batching) (map[string
mfs, exist := migrateDs[name]

if !exist {
return nil, fmt.Errorf("migration function for %s not found", name)
dss[name] = ds
log.Warnf("no migration sechedules for : %s", name)
continue
}

var versionedDs datastore.Batching
Expand All @@ -263,8 +268,7 @@ func Migrate(ctx context.Context, dss map[string]datastore.Batching) (map[string
// 后续的版本升级中如果出错, 应该直接返回.
if err != nil {
dss[name] = ds

log.Warnf("migrate:%s failed:%w", name, err)
log.Warnf("migrate:%s failed:%s", name, err.Error())
continue
}
dss[name] = versionedDs
Expand Down
7 changes: 7 additions & 0 deletions models/badger/migrate/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package migrate

import "github.com/ipfs/go-datastore"

type DsKeyAble interface {
KeyWithNamespace() datastore.Key
}
94 changes: 94 additions & 0 deletions models/badger/migrate/v2.2.0/testing/testing.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package testing

import (
"context"
"testing"

"github.com/filecoin-project/venus-market/v2/models/badger/migrate"

"github.com/ipfs/go-cid"

v220 "github.com/filecoin-project/venus-market/v2/models/badger/migrate/v2.2.0"
"github.com/filecoin-project/venus/venus-shared/testutil"
"github.com/ipfs/go-datastore"
"github.com/stretchr/testify/assert"
cbg "github.com/whyrusleeping/cbor-gen"

cbor "github.com/filecoin-project/go-cbor-util"
)

func dsPutObj(ctx context.Context, t *testing.T, v migrate.DsKeyAble, ds datastore.Batching) {
data, err := cbor.Dump(v)
assert.NoError(t, err)
assert.NoError(t, ds.Put(ctx, v.KeyWithNamespace(), data))
}

func WriteTestcasesToDS(ctx context.Context, t *testing.T, ds datastore.Batching, count int) (payChMsgCIDs []cid.Cid) {
{
for i := 0; i < count; i++ {
var stat v220.FundedAddressState
testutil.Provide(t, &stat)
dsPutObj(ctx, t, &stat, ds)
}
}

{
for i := 0; i < count; i++ {
var deal v220.MinerDeal
testutil.Provide(t, &deal)
dsPutObj(ctx, t, &deal, ds)
}
}

{
for i := 0; i < count; i++ {
var info v220.MsgInfo
testutil.Provide(t, &info)
dsPutObj(ctx, t, &info, ds)
payChMsgCIDs = append(payChMsgCIDs, info.MsgCid)
}
}

{
for i := 0; i < count; i++ {
var info v220.ChannelInfo
testutil.Provide(t, &info)
dsPutObj(ctx, t, &info, ds)
}
}

{
for i := 0; i < count; i++ {
var ask v220.SignedStorageAsk
testutil.Provide(t, &ask.Ask)
testutil.Provide(t, &ask.Signature)
dsPutObj(ctx, t, &ask, ds)
}
}

{
for i := 0; i < count; i++ {
var ask v220.RetrievalAsk
testutil.Provide(t, &ask)
dsPutObj(ctx, t, &ask, ds)
}
}

{
for i := 0; i < count; i++ {
var cidInfo v220.CIDInfo
testutil.Provide(t, &cidInfo.CID)
testutil.Provide(t, &cidInfo.PieceBlockLocations, testutil.WithSliceLen(2))
dsPutObj(ctx, t, &cidInfo, ds)
}
}

{
for i := 0; i < count; i++ {
var stat v220.ProviderDealState
testutil.Provide(t, &stat, func(t *testing.T) *cbg.Deferred { return nil })
dsPutObj(ctx, t, &stat, ds)
}
}
return
}
Loading

0 comments on commit f921dbc

Please sign in to comment.