From 6337e11ee1eb467b2b030fe39db30387b3dc570c Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 12:58:17 +0200 Subject: [PATCH 01/11] add repair script --- repair.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 repair.go diff --git a/repair.go b/repair.go new file mode 100644 index 000000000..ea8188ff8 --- /dev/null +++ b/repair.go @@ -0,0 +1,49 @@ +package iavl + +import ( + "github.com/pkg/errors" + dbm "github.com/tendermint/tm-db" +) + +// Repair013Orphans repairs incorrect orphan entries written by IAVL 0.13 pruning. To use it, close +// a database using IAVL 0.13, make a backup copy, and then run this function before opening the +// database with IAVL 0.14 or later. It returns the number of faulty orphan entries removed. +// +// Note that this cannot be used directly on Cosmos SDK databases, since they store multiple IAVL +// trees in the same underlying database via a prefix scheme. +// +// The pruning functionality enabled with Options.KeepEvery > 1 would write orphans entries to disk +// for versions that should only have been saved in memory, and these orphan entries were clamped +// to the last version persisted to disk instead of the version that generated them. If the +// database was reopened at the last persisted version and this version was later deleted, the +// orphaned nodes could be deleted prematurely or incorrectly, causing data loss and database +// corruption. +// +// This function removes these incorrect orphan entries by deleting all orphan entries that have a +// to-version equal to or greater than the latest persisted version. Correct orphans will never +// have this, since they must have been deleted in the next (non-existent) version for that to be +// the case. +func Repair013Orphans(db dbm.DB) (uint64, error) { + ndb := newNodeDB(db, 0, &Options{Sync: true}) + version := ndb.getLatestVersion() + if version == 0 { + return 0, errors.New("no versions found") + } + + batch := db.NewBatch() + repaired := uint64(0) + ndb.traverseOrphans(func(key, hash []byte) { + var toVersion int64 + orphanKeyFormat.Scan(key, &toVersion) + if toVersion >= version { + repaired++ + batch.Delete(key) + } + }) + err := batch.WriteSync() + if err != nil { + return 0, err + } + + return repaired, nil +} From 1c4552a887e53e5a43f512946ad73bd1eb5f56c6 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 13:08:18 +0200 Subject: [PATCH 02/11] comment tweak --- repair.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repair.go b/repair.go index ea8188ff8..0a4979d6a 100644 --- a/repair.go +++ b/repair.go @@ -15,8 +15,8 @@ import ( // The pruning functionality enabled with Options.KeepEvery > 1 would write orphans entries to disk // for versions that should only have been saved in memory, and these orphan entries were clamped // to the last version persisted to disk instead of the version that generated them. If the -// database was reopened at the last persisted version and this version was later deleted, the -// orphaned nodes could be deleted prematurely or incorrectly, causing data loss and database +// database is reopened at the last persisted version and this version is later deleted, the +// orphaned nodes can be deleted prematurely or incorrectly, causing data loss and database // corruption. // // This function removes these incorrect orphan entries by deleting all orphan entries that have a From 5abe3dd7f8d1af16a23abe76384a1426b746da9b Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 13:35:15 +0200 Subject: [PATCH 03/11] add test database from 0.13.3 with incorrect orphans --- repair_test.go | 43 +++++++++++++++++++++++ testdata/0.13-orphans.db/000001.log | Bin 0 -> 4300 bytes testdata/0.13-orphans.db/CURRENT | 1 + testdata/0.13-orphans.db/LOCK | 0 testdata/0.13-orphans.db/LOG | 6 ++++ testdata/0.13-orphans.db/MANIFEST-000000 | Bin 0 -> 54 bytes 6 files changed, 50 insertions(+) create mode 100644 repair_test.go create mode 100644 testdata/0.13-orphans.db/000001.log create mode 100644 testdata/0.13-orphans.db/CURRENT create mode 100644 testdata/0.13-orphans.db/LOCK create mode 100644 testdata/0.13-orphans.db/LOG create mode 100644 testdata/0.13-orphans.db/MANIFEST-000000 diff --git a/repair_test.go b/repair_test.go new file mode 100644 index 000000000..88719b9c1 --- /dev/null +++ b/repair_test.go @@ -0,0 +1,43 @@ +package iavl + +// Generate013Orphans generates a GoLevelDB orphan database in testdata/0.13-orphans.db +// for testing Repair013Orphans(). It must be run with IAVL 0.13.x. +/*func TestGenerate013Orphans(t *testing.T) { + err := os.RemoveAll("testdata/0.13-orphans.db") + require.NoError(t, err) + db, err := dbm.NewGoLevelDB("0.13-orphans", "testdata") + require.NoError(t, err) + tree, err := NewMutableTreeWithOpts(db, dbm.NewMemDB(), 0, &Options{ + KeepEvery: 3, + KeepRecent: 1, + Sync: true, + }) + require.NoError(t, err) + version, err := tree.Load() + require.NoError(t, err) + require.EqualValues(t, 0, version) + + // We generate 8 versions. In each version, we create a "addX" key, delete a "rmX" key, + // and update the "current" key, where "X" is the current version. Values are the version in + // which the key was last set. + tree.Set([]byte("rm1"), []byte{1}) + tree.Set([]byte("rm2"), []byte{1}) + tree.Set([]byte("rm3"), []byte{1}) + tree.Set([]byte("rm4"), []byte{1}) + tree.Set([]byte("rm5"), []byte{1}) + tree.Set([]byte("rm6"), []byte{1}) + tree.Set([]byte("rm7"), []byte{1}) + tree.Set([]byte("rm8"), []byte{1}) + + for v := byte(1); v <= 8; v++ { + tree.Set([]byte("current"), []byte{v}) + tree.Set([]byte(fmt.Sprintf("add%v", v)), []byte{v}) + tree.Remove([]byte(fmt.Sprintf("rm%v", v))) + _, version, err = tree.SaveVersion() + require.NoError(t, err) + require.EqualValues(t, v, version) + } + + // At this point, the database will contain incorrect orphans in version 6 that, when + // version 6 is deleted, will cause "current", "rm7", and "rm8" to go missing. +}*/ diff --git a/testdata/0.13-orphans.db/000001.log b/testdata/0.13-orphans.db/000001.log new file mode 100644 index 0000000000000000000000000000000000000000..95ef16dcaf9eb82f3f7e3d1fa4e6ab313229b121 GIT binary patch literal 4300 zcmds5Z8Vf=7@n6grdee}zM5m6Sl?EW8C2UkX|p4guid0DF;Uh>!-^)XNKx&U(z1z` z&9-YZm6V8-iFOFpQnrpgO~}WJwu~~WeP)bK=Ztgi-JiSj=lyd%&vo6`^W67)z2j(7 z&0V2J5TvP}7Jd*5qWc(I89kdRWo-$jV$HN=8P%cbNm+ssuV#}LbL#x^<1~_r3YilY zW=9Yh9XxPXSSr%{YGl}=Cfsv3@kV-W=*Xa@dl<#KzPgy-jnQNo+MZCsXlYhSaKt0@ zg@u)HFx>FeL6J1FX?ER(32IZHc=Wx54tEtYMUF<03Iq{jwyjIAv<7lFYjz%(D61X6 z{=DR;fQRKA^%K?{dfH|_zXWTOvn?i6F*<1PY1X11(y?n5ec61WoB4~fN?rf0I$nq7 zWoCSw*Kal(qt!#dtz>n})SFH0Gqy`DX__ zaWT^i5Tpe`^jM+Ui#J`xEyHoto&2mx6JgR_uh;D!%bmWLJjwFCaO(55v+&8N@}eB% z2a_Y{T|XLGb`YcK@t!| zi^~Z9!!N`8rA=xbY17V3=Rz;59Z#)x5AFHZrkl5~sWvlJq0CXP?6HlVCHK~xvxg0C z$0kR#ndW6I$YciATP<6WZ_eD00u*2;Iau@OO74)0>Fc@aRYomm#3?2ICM%0h_x~Q_ z5jSUa<;mTOGMo@30YS9549%nW8F#GgXbIPrCneP``tkkf#fIt|?yR;IpOpBGZ7*b# zsWQ(T5o9icWMi`EF7wXr+Die6@m2cv3ynHf?=&z&O}c*19xN)atBed}Q?+Ebnu(wr zi6Gg;ES_R6sv?Nhr>vewRw1+%3n;QdY8NxTT=68JK7t!fX-`#187N&@rz&V21ejq@ zsH@?&SFRl#To7OeozN5p$F%r*{aPwP7`$`%$}|KxJ|MshdxGr+0cO|}Y7(%E?2QN0 z3e*U?!VX!7;4X$>_S4Zpjh2grX zviRdbkg-d~ujZ~_dO{E_T~)Y46v^|xnpmf!swT)WBA4!(pcRO4Sfa_%aJYboFm zN7-#^RzV+`#xtMCp67hCrbs$xu+DM=c0yholTgFxo4Jn}vGYqjvMa{&6AO&?u8Q^K zJ@g4|7`Tz^^VGY+0i)&GSOf*5Q?`_oUs#K(PIhf`7I`-p#zaJzq#vUX96267S)5li zANw|iMvw{w5o0!Ph?}m#8pamA_LBAYn`sxGEj0RYY}A-efpU4X z`7gC}`m(o-*IDaXxP}Uw2SlO%l8LbHhz~YO-AsBs$&|MX5L63-=&?fRCI27E83tlb zkFQ&!An-%V70+OSB$f7?WxJtpZT)kO_W4WUQZ|LAA(8~7Mw7Ml8 zblw-*>ma$DP8eQCnWY6a+>ygk?o=)L9bzJ=ZxEz=hOrBUM>t)+4KEY~l_E%WlB`o9 z7Ai)xx@<(hec8&QjO<51mk5lKGwccFJ@}xMbW)&dK|7RnQlJz37e_qpCk4Eu5MTzK m&@6z5g|eUE0D=G&dOaQ?B!V?{k4UmV8`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD literal 0 HcmV?d00001 From ab51dc2d6df545e88831999d95bcb9d0f9e7ac89 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 14:28:02 +0200 Subject: [PATCH 04/11] add repair test case --- repair_test.go | 146 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/repair_test.go b/repair_test.go index 88719b9c1..808dea665 100644 --- a/repair_test.go +++ b/repair_test.go @@ -1,5 +1,120 @@ package iavl +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" +) + +func TestRepair013Orphans(t *testing.T) { + dir, err := ioutil.TempDir("", "test-iavl-repair") + require.NoError(t, err) + defer os.RemoveAll(dir) + err = copyDB("testdata/0.13-orphans.db", filepath.Join(dir, "0.13-orphans.db")) + require.NoError(t, err) + + db, err := dbm.NewGoLevelDB("0.13-orphans", dir) + require.NoError(t, err) + + // Repair the database. + repaired, err := Repair013Orphans(db) + require.NoError(t, err) + assert.EqualValues(t, 8, repaired) + + // Load the database. + tree, err := NewMutableTreeWithOpts(db, 0, &Options{Sync: true}) + require.NoError(t, err) + version, err := tree.Load() + require.NoError(t, err) + require.EqualValues(t, 6, version) + + // We now generate two empty versions, and check all persisted versions. + _, version, err = tree.SaveVersion() + require.NoError(t, err) + require.EqualValues(t, 7, version) + _, version, err = tree.SaveVersion() + require.NoError(t, err) + require.EqualValues(t, 8, version) + + // Check all persisted versions. + require.Equal(t, []int{3, 6, 7, 8}, tree.AvailableVersions()) + assertVersion(t, tree, 0) + assertVersion(t, tree, 3) + assertVersion(t, tree, 6) + assertVersion(t, tree, 7) + assertVersion(t, tree, 8) + + // We then delete version 6 (the last persisted one with 0.13). + err = tree.DeleteVersion(6) + require.NoError(t, err) + + // Reading "rm7" (which should not have been deleted now) would panic with a broken database. + _, value := tree.Get([]byte("rm7")) + require.Equal(t, []byte{1}, value) + + // Check all persisted versions. + require.Equal(t, []int{3, 7, 8}, tree.AvailableVersions()) + assertVersion(t, tree, 0) + assertVersion(t, tree, 3) + assertVersion(t, tree, 7) + assertVersion(t, tree, 8) + + // Delete all historical versions, and check the latest. + err = tree.DeleteVersion(3) + require.NoError(t, err) + err = tree.DeleteVersion(7) + require.NoError(t, err) + + require.Equal(t, []int{8}, tree.AvailableVersions()) + assertVersion(t, tree, 0) + assertVersion(t, tree, 8) +} + +// assertVersion checks the given version (or current if 0) against the expected values. +func assertVersion(t *testing.T, tree *MutableTree, version int64) { + t.Logf("Checking version %v", version) + var err error + itree := tree.ImmutableTree + if version > 0 { + itree, err = tree.GetImmutable(version) + require.NoError(t, err) + } + version = itree.version + + // The "current" value should have the current version for <= 6, then 6 afterwards + _, value := itree.Get([]byte("current")) + if version >= 6 { + require.EqualValues(t, []byte{6}, value) + } else { + require.EqualValues(t, []byte{byte(version)}, value) + } + + // The "addX" entries should exist for 1-6 in the respective versions, and the + // "rmX" entries should have been removed for 1-6 in the respective versions. + for i := byte(1); i < 8; i++ { + _, value = itree.Get([]byte(fmt.Sprintf("add%v", i))) + if i <= 6 && int64(i) <= version { + require.Equal(t, []byte{i}, value) + } else { + require.Nil(t, value) + } + + _, value = itree.Get([]byte(fmt.Sprintf("rm%v", i))) + if i <= 6 && version >= int64(i) { + require.Nil(t, value) + } else { + require.Equal(t, []byte{1}, value) + } + } +} + // Generate013Orphans generates a GoLevelDB orphan database in testdata/0.13-orphans.db // for testing Repair013Orphans(). It must be run with IAVL 0.13.x. /*func TestGenerate013Orphans(t *testing.T) { @@ -41,3 +156,34 @@ package iavl // At this point, the database will contain incorrect orphans in version 6 that, when // version 6 is deleted, will cause "current", "rm7", and "rm8" to go missing. }*/ + +// copyDB makes a shallow copy of the source database directory. +func copyDB(src, dest string) error { + entries, err := ioutil.ReadDir(src) + if err != nil { + return err + } + err = os.MkdirAll(dest, 0777) + if err != nil { + return err + } + for _, entry := range entries { + out, err := os.Create(filepath.Join(dest, entry.Name())) + if err != nil { + return err + } + defer out.Close() + + in, err := os.Open(filepath.Join(src, entry.Name())) + defer in.Close() // nolint + if err != nil { + return err + } + + _, err = io.Copy(out, in) + if err != nil { + return err + } + } + return nil +} From 4518b0a52a273c99b022a1a431d32fa86c630626 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 14:33:44 +0200 Subject: [PATCH 05/11] add v6 testdata as well --- repair_test.go | 5 ++++- testdata/0.13-orphans-v6.db/000001.log | Bin 0 -> 3590 bytes testdata/0.13-orphans-v6.db/CURRENT | 1 + testdata/0.13-orphans-v6.db/LOCK | 0 testdata/0.13-orphans-v6.db/LOG | 6 ++++++ testdata/0.13-orphans-v6.db/MANIFEST-000000 | Bin 0 -> 54 bytes 6 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 testdata/0.13-orphans-v6.db/000001.log create mode 100644 testdata/0.13-orphans-v6.db/CURRENT create mode 100644 testdata/0.13-orphans-v6.db/LOCK create mode 100644 testdata/0.13-orphans-v6.db/LOG create mode 100644 testdata/0.13-orphans-v6.db/MANIFEST-000000 diff --git a/repair_test.go b/repair_test.go index 808dea665..df7d06f27 100644 --- a/repair_test.go +++ b/repair_test.go @@ -17,7 +17,10 @@ func TestRepair013Orphans(t *testing.T) { dir, err := ioutil.TempDir("", "test-iavl-repair") require.NoError(t, err) defer os.RemoveAll(dir) - err = copyDB("testdata/0.13-orphans.db", filepath.Join(dir, "0.13-orphans.db")) + + // There is also 0.13-orphans-v6.db containing a database closed immediately after writing + // version 6, which should not contain any broken orphans. + err = copyDB("testdata/0.13-orphans-v6.db", filepath.Join(dir, "0.13-orphans.db")) require.NoError(t, err) db, err := dbm.NewGoLevelDB("0.13-orphans", dir) diff --git a/testdata/0.13-orphans-v6.db/000001.log b/testdata/0.13-orphans-v6.db/000001.log new file mode 100644 index 0000000000000000000000000000000000000000..13bc49ab48d66ec10f0a1aa9c3fadff3086764e5 GIT binary patch literal 3590 zcmds3TU1j=6y?Gt#1y4GjUloBvd1S41kQBYd} zu}~2#snnKAe4z+}C=qNxKrJZMl`bP7k0MwlC_(My8vKE^#^3h-+@F1B?{nsyGuw+^ zBUq-7VLER<6ZpgC3;SqWXg!-q741oON~828nKh9aDcRByb_>1LfjqDBC`CtC7w7T$ z?idEq+<`l?a=FDfBg2lhQGvU$*XpZ7hX$Pj_#~HxnliBhqHzt{1Ji|Qb+&{X{SbWR z=qwwIvOaN8u8wJ*U4MRp+}x)eeJ`mqKo=(wXn;}yP{fdJ%i_yzVS-IYT?ZyA>c+1< zll&6;pps{B%!Nly-y{}Gpf-YSA*Kh>;d@Uq7xd^Hxq7xQM=bMqcwSLu9{gSysNUCg_B4cFhHdNP_$54TkGMXc4l}i zHRJbxHf(%($r2QITiDH9?evC zF83(Zf|&%E?|-4E!=va?9MiVVZ2Z2?;%Jk?YpM(#16uW&)H1fTD$I zaQ_O<Q;Pe zbh}+)=KL&1ScCJDWkn8*{UB5e_9nnazg7u`eC=KtXH?s?o>HbsLg>p&Pxk*Aw<=-I z=`qC#x-=!FZo$v*3l~}&tiQd|O?g}uJhrWbg_AWk z^8%>i0+eh>ChT_TQq)}xO-`(~^!Ui8bL9>zd(dni{Pg~U%KEC9Fc#TF)71=s>PUc+ zjm;8i_o5y^u^LPGLR=R!)dr9>i_|@CDtzjX>TGoDPd^hmyePm7I#HVly{e|^#85i~ zy=11HsfW51BagORa*|tgup3AJ?{Tki=nc-wJbvB*OYL5t0~uOSWyEKvwF+FYQBAzN9R zSkRsNw5istc9A-&=J97NJlBj(izw%wON`OGToHhi4K-{F(N8!^IAjZn@J{VYtzZ~G z%xTwm4*%FLk?}149Phi;rRq6@^-k+yC5YS@m_9_`D0oDRpC?(Bb9SsKx!7j!iugd$ zgSBCe1J?`IKH)TaLNuX`iIE^WZF435+(lk}ynCyUoYPVg7adK{I6@sbbTn$Rtgv(* ztlJwjK&b#IV#p?l@He+wP1|hIAz62?rOv6hQE@lcmWplN*clk&XxImBqv4bbB;?H& zzc4ZD%h^0$?_%NT7b$BQkVl57Cisfz4>qX%={1_Z&cY@ x_C@x3sxD<<*4IE*d2y{^BtJHQY(n%91EA&xpmfi$c2VXL?JnP%7ZO0F@-G-${zm`+ literal 0 HcmV?d00001 diff --git a/testdata/0.13-orphans-v6.db/CURRENT b/testdata/0.13-orphans-v6.db/CURRENT new file mode 100644 index 000000000..feda7d6b2 --- /dev/null +++ b/testdata/0.13-orphans-v6.db/CURRENT @@ -0,0 +1 @@ +MANIFEST-000000 diff --git a/testdata/0.13-orphans-v6.db/LOCK b/testdata/0.13-orphans-v6.db/LOCK new file mode 100644 index 000000000..e69de29bb diff --git a/testdata/0.13-orphans-v6.db/LOG b/testdata/0.13-orphans-v6.db/LOG new file mode 100644 index 000000000..f890e80b8 --- /dev/null +++ b/testdata/0.13-orphans-v6.db/LOG @@ -0,0 +1,6 @@ +=============== Jun 25, 2020 (CEST) =============== +14:30:10.673317 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed +14:30:10.688689 db@open opening +14:30:10.689548 version@stat F·[] S·0B[] Sc·[] +14:30:10.702481 db@janitor F·2 G·0 +14:30:10.702564 db@open done T·13.82376ms diff --git a/testdata/0.13-orphans-v6.db/MANIFEST-000000 b/testdata/0.13-orphans-v6.db/MANIFEST-000000 new file mode 100644 index 0000000000000000000000000000000000000000..9d54f6733b1364dc8d53dd15ca59a6ec36a1c29d GIT binary patch literal 54 zcmdmC5aOo9z{n_-lUkOVlai$8R9TW*o>`pgoS$2eSd>_jU&O?~%*ev9Y~pbaHU>r} JMrI}!1^~s!4paaD literal 0 HcmV?d00001 From ca11fc22085099d291cbf3908fcb8156bda48566 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 14:35:41 +0200 Subject: [PATCH 06/11] add comment on when repair is needed --- repair.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repair.go b/repair.go index 0a4979d6a..623a7efb1 100644 --- a/repair.go +++ b/repair.go @@ -23,6 +23,9 @@ import ( // to-version equal to or greater than the latest persisted version. Correct orphans will never // have this, since they must have been deleted in the next (non-existent) version for that to be // the case. +// +// If 0.13 is running with KeepEvery:1 (the default), or if the latest version _ever_ saved to the +// IAVL tree is persisted to disk (i.e. a multiple of KeepEvery), then this repair is not necessary. func Repair013Orphans(db dbm.DB) (uint64, error) { ndb := newNodeDB(db, 0, &Options{Sync: true}) version := ndb.getLatestVersion() From 62adc8d0e1b3d1300c00d5d0bf9c9e3a1f07e34c Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 14:36:24 +0200 Subject: [PATCH 07/11] fix test --- repair_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repair_test.go b/repair_test.go index df7d06f27..ec6b598b3 100644 --- a/repair_test.go +++ b/repair_test.go @@ -20,7 +20,7 @@ func TestRepair013Orphans(t *testing.T) { // There is also 0.13-orphans-v6.db containing a database closed immediately after writing // version 6, which should not contain any broken orphans. - err = copyDB("testdata/0.13-orphans-v6.db", filepath.Join(dir, "0.13-orphans.db")) + err = copyDB("testdata/0.13-orphans.db", filepath.Join(dir, "0.13-orphans.db")) require.NoError(t, err) db, err := dbm.NewGoLevelDB("0.13-orphans", dir) @@ -82,7 +82,6 @@ func TestRepair013Orphans(t *testing.T) { // assertVersion checks the given version (or current if 0) against the expected values. func assertVersion(t *testing.T, tree *MutableTree, version int64) { - t.Logf("Checking version %v", version) var err error itree := tree.ImmutableTree if version > 0 { From 06428420f334063667577aa09b4f1573f2440721 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 14:39:09 +0200 Subject: [PATCH 08/11] add CHANGELOG entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e547c21..c9a1ed078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Improvements + +- Add `Repair013Orphans()` to repair faulty orphans in a database last written to by IAVL 0.13.x + ### Bug Fixes - Remove unnecessary Protobuf dependencies From 24d9c179bed3ae6fdb23ea1b678c2f25dd571046 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 14:55:56 +0200 Subject: [PATCH 09/11] more efficient orphan scan --- repair.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/repair.go b/repair.go index 623a7efb1..68aff58d7 100644 --- a/repair.go +++ b/repair.go @@ -1,6 +1,8 @@ package iavl import ( + "math" + "github.com/pkg/errors" dbm "github.com/tendermint/tm-db" ) @@ -33,17 +35,27 @@ func Repair013Orphans(db dbm.DB) (uint64, error) { return 0, errors.New("no versions found") } + var ( + repaired uint64 + err error + ) batch := db.NewBatch() - repaired := uint64(0) - ndb.traverseOrphans(func(key, hash []byte) { + defer batch.Close() + ndb.traverseRange(orphanKeyFormat.Key(version), orphanKeyFormat.Key(math.MaxInt64), func(k, v []byte) { + // Sanity check so we don't remove stuff we shouldn't var toVersion int64 - orphanKeyFormat.Scan(key, &toVersion) - if toVersion >= version { - repaired++ - batch.Delete(key) + orphanKeyFormat.Scan(k, &toVersion) + if toVersion < version { + err = errors.Errorf("Found unexpected orphan with toVersion=%v, lesser than latest version %v", + toVersion, version) } + repaired++ + batch.Delete(k) }) - err := batch.WriteSync() + if err != nil { + return 0, err + } + err = batch.WriteSync() if err != nil { return 0, err } From a38b1cfd63056039589bbf56e1807a5e07134416 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Thu, 25 Jun 2020 14:56:29 +0200 Subject: [PATCH 10/11] add early return --- repair.go | 1 + 1 file changed, 1 insertion(+) diff --git a/repair.go b/repair.go index 68aff58d7..b5f45f2fa 100644 --- a/repair.go +++ b/repair.go @@ -48,6 +48,7 @@ func Repair013Orphans(db dbm.DB) (uint64, error) { if toVersion < version { err = errors.Errorf("Found unexpected orphan with toVersion=%v, lesser than latest version %v", toVersion, version) + return } repaired++ batch.Delete(k) From 80c65017700301a3d9169421f14eb2dcb2fabe9f Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Fri, 26 Jun 2020 10:39:15 +0200 Subject: [PATCH 11/11] comment tweak --- repair.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/repair.go b/repair.go index b5f45f2fa..157eb1fc3 100644 --- a/repair.go +++ b/repair.go @@ -9,25 +9,25 @@ import ( // Repair013Orphans repairs incorrect orphan entries written by IAVL 0.13 pruning. To use it, close // a database using IAVL 0.13, make a backup copy, and then run this function before opening the -// database with IAVL 0.14 or later. It returns the number of faulty orphan entries removed. +// database with IAVL 0.14 or later. It returns the number of faulty orphan entries removed. If the +// 0.13 database was written with KeepEvery:1 (the default) or the last version _ever_ saved to the +// tree was a multiple of `KeepEvery` and thus saved to disk, this repair is not necessary. // // Note that this cannot be used directly on Cosmos SDK databases, since they store multiple IAVL // trees in the same underlying database via a prefix scheme. // // The pruning functionality enabled with Options.KeepEvery > 1 would write orphans entries to disk // for versions that should only have been saved in memory, and these orphan entries were clamped -// to the last version persisted to disk instead of the version that generated them. If the +// to the last version persisted to disk instead of the version that generated them (so a delete at +// version 749 might generate an orphan entry ending at version 700 for KeepEvery:100). If the // database is reopened at the last persisted version and this version is later deleted, the // orphaned nodes can be deleted prematurely or incorrectly, causing data loss and database // corruption. // // This function removes these incorrect orphan entries by deleting all orphan entries that have a // to-version equal to or greater than the latest persisted version. Correct orphans will never -// have this, since they must have been deleted in the next (non-existent) version for that to be +// have this, since they must have been deleted in a future (non-existent) version for that to be // the case. -// -// If 0.13 is running with KeepEvery:1 (the default), or if the latest version _ever_ saved to the -// IAVL tree is persisted to disk (i.e. a multiple of KeepEvery), then this repair is not necessary. func Repair013Orphans(db dbm.DB) (uint64, error) { ndb := newNodeDB(db, 0, &Options{Sync: true}) version := ndb.getLatestVersion()