Skip to content

Commit

Permalink
Add new config param to disable snapshot file creation. Defaults to f…
Browse files Browse the repository at this point in the history 
…alse.
  • Loading branch information
@alexsporn
alexsporn committed Aug 29, 2022
1 parent 7945f5e commit f20efb0
Show file tree
Hide file tree
Showing 9 changed files with 48 additions and 49 deletions.
1 change: 1 addition & 0 deletions config_defaults.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@
"whiteFlagParentsSolidTimeout": "2s"
},
"snapshots": {
"enabled": false,
"depth": 50,
"interval": 200,
"fullPath": "testnet/snapshots/full_snapshot.bin",
Expand Down
8 changes: 1 addition & 7 deletions core/snapshot/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,6 @@ const (

// SolidEntryPointCheckAdditionalThresholdFuture is the additional future cone (to BMD) that is needed to calculate solid entry points correctly.
SolidEntryPointCheckAdditionalThresholdFuture = 5

// AdditionalPruningThreshold is the additional threshold (to BMD), which is needed, because the blocks in the getMilestoneParents call in solidEntryPoints
// can reference older blocks as well.
AdditionalPruningThreshold = 5
)

const (
Expand Down Expand Up @@ -163,7 +159,6 @@ func provide(c *dig.Container) error {

solidEntryPointCheckThresholdPast := syncmanager.MilestoneIndexDelta(deps.ProtocolManager.Current().BelowMaxDepth + SolidEntryPointCheckAdditionalThresholdPast)
solidEntryPointCheckThresholdFuture := syncmanager.MilestoneIndexDelta(deps.ProtocolManager.Current().BelowMaxDepth + SolidEntryPointCheckAdditionalThresholdFuture)
pruningThreshold := syncmanager.MilestoneIndexDelta(deps.ProtocolManager.Current().BelowMaxDepth + AdditionalPruningThreshold)

snapshotDepth := syncmanager.MilestoneIndexDelta(ParamsSnapshots.Depth)
if snapshotDepth < solidEntryPointCheckThresholdFuture {
Expand All @@ -176,14 +171,13 @@ func provide(c *dig.Container) error {
deps.Storage,
deps.SyncManager,
deps.UTXOManager,
deps.ProtocolManager,
ParamsSnapshots.Enabled,
deps.SnapshotsFullPath,
deps.SnapshotsDeltaPath,
ParamsSnapshots.DeltaSizeThresholdPercentage,
deltaSnapshotSizeThresholdMinSizeBytes,
solidEntryPointCheckThresholdPast,
solidEntryPointCheckThresholdFuture,
pruningThreshold,
snapshotDepth,
syncmanager.MilestoneIndexDelta(ParamsSnapshots.Interval),
)
Expand Down
2 changes: 2 additions & 0 deletions core/snapshot/params.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (

// ParametersSnapshots contains the definition of the parameters used by snapshots.
type ParametersSnapshots struct {
// Enabled defines whether to generate snapshot files.
Enabled bool `default:"false" usage:"whether to generate snapshot files"`
// Depth defines the depth, respectively the starting point, at which a snapshot of the ledger is generated
Depth int `default:"50" usage:"the depth, respectively the starting point, at which a snapshot of the ledger is generated"`
// Interval defines the interval, in milestones, at which snapshot files are created (snapshots are only created if the node is synced)
Expand Down
1 change: 1 addition & 0 deletions docker/config_defaults.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@
"whiteFlagParentsSolidTimeout": "2s"
},
"snapshots": {
"enabled": false,
"depth": 50,
"interval": 200,
"fullPath": "testnet/snapshots/full_snapshot.bin",
Expand Down
20 changes: 11 additions & 9 deletions documentation/docs/references/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,15 +324,16 @@ Example:

## <a id="snapshots"></a> 9. Snapshots

| Name | Description | Type | Default value |
| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | -------------------------------------- |
| depth | The depth, respectively the starting point, at which a snapshot of the ledger is generated | int | 50 |
| interval | Interval, in milestones, at which snapshot files are created (snapshots are only created if the node is synced) | int | 200 |
| fullPath | Path to the full snapshot file | string | "testnet/snapshots/full_snapshot.bin" |
| deltaPath | Path to the delta snapshot file | string | "testnet/snapshots/delta_snapshot.bin" |
| deltaSizeThresholdPercentage | Create a full snapshot if the size of a delta snapshot reaches a certain percentage of the full snapshot (0.0 = always create delta snapshot to keep ms diff history) | float | 50.0 |
| deltaSizeThresholdMinSize | The minimum size of the delta snapshot file before the threshold percentage condition is checked (below that size the delta snapshot is always created) | string | "50M" |
| [downloadURLs](#snapshots_downloadurls) | Configuration for downloadURLs | array | see example below |
| Name | Description | Type | Default value |
| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | -------------------------------------- |
| enabled | Whether to generate snapshot files | boolean | false |
| depth | The depth, respectively the starting point, at which a snapshot of the ledger is generated | int | 50 |
| interval | Interval, in milestones, at which snapshot files are created (snapshots are only created if the node is synced) | int | 200 |
| fullPath | Path to the full snapshot file | string | "testnet/snapshots/full_snapshot.bin" |
| deltaPath | Path to the delta snapshot file | string | "testnet/snapshots/delta_snapshot.bin" |
| deltaSizeThresholdPercentage | Create a full snapshot if the size of a delta snapshot reaches a certain percentage of the full snapshot (0.0 = always create delta snapshot to keep ms diff history) | float | 50.0 |
| deltaSizeThresholdMinSize | The minimum size of the delta snapshot file before the threshold percentage condition is checked (below that size the delta snapshot is always created) | string | "50M" |
| [downloadURLs](#snapshots_downloadurls) | Configuration for downloadURLs | array | see example below |

### <a id="snapshots_downloadurls"></a> DownloadURLs

Expand All @@ -346,6 +347,7 @@ Example:
```json
{
"snapshots": {
"enabled": false,
"depth": 50,
"interval": 200,
"fullPath": "testnet/snapshots/full_snapshot.bin",
Expand Down
16 changes: 3 additions & 13 deletions pkg/pruning/pruning.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,25 +255,15 @@ func (p *Manager) pruneDatabase(ctx context.Context, targetIndex iotago.Mileston
}

targetIndexMax := p.getMinimumTangleHistory()
if targetIndex > targetIndexMax {
targetIndex = targetIndexMax
}

snapshotInfo := p.storage.SnapshotInfo()
if snapshotInfo == nil {
return 0, errors.Wrap(common.ErrCritical, common.ErrSnapshotInfoNotFound.Error())
}

//if snapshotInfo.SnapshotIndex() < p.solidEntryPointCheckThresholdPast+p.additionalPruningThreshold+1 {
if snapshotInfo.SnapshotIndex() < p.additionalPruningThreshold+1 {
// Not enough history
//return 0, errors.Wrapf(ErrNotEnoughHistory, "minimum index: %d, target index: %d", p.solidEntryPointCheckThresholdPast+p.additionalPruningThreshold+1, targetIndex)
return 0, errors.Wrapf(ErrNotEnoughHistory, "minimum index: %d, target index: %d", p.additionalPruningThreshold+1, targetIndex)
}

// TODO
//targetIndexMax := snapshotInfo.SnapshotIndex() - p.solidEntryPointCheckThresholdPast - p.additionalPruningThreshold - 1
if targetIndex > targetIndexMax {
targetIndex = targetIndexMax
}

if snapshotInfo.PruningIndex() >= targetIndex {
// no pruning needed
return 0, errors.Wrapf(ErrNoPruningNeeded, "pruning index: %d, target index: %d", snapshotInfo.PruningIndex(), targetIndex)
Expand Down
31 changes: 20 additions & 11 deletions pkg/snapshot/snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import (
storagepkg "github.com/iotaledger/hornet/v2/pkg/model/storage"
"github.com/iotaledger/hornet/v2/pkg/model/syncmanager"
"github.com/iotaledger/hornet/v2/pkg/model/utxo"
"github.com/iotaledger/hornet/v2/pkg/protocol"
iotago "github.com/iotaledger/iota.go/v3"
)

Expand Down Expand Up @@ -59,6 +58,7 @@ type Manager struct {
storage *storagepkg.Storage
syncManager *syncmanager.SyncManager
utxoManager *utxo.Manager
snapshotCreationEnabled bool
snapshotFullPath string
snapshotDeltaPath string
deltaSnapshotSizeThresholdPercentage float64
Expand All @@ -81,14 +81,13 @@ func NewSnapshotManager(
storage *storagepkg.Storage,
syncManager *syncmanager.SyncManager,
utxoManager *utxo.Manager,
protocolManager *protocol.Manager,
snapshotCreationEnabled bool,
snapshotFullPath string,
snapshotDeltaPath string,
deltaSnapshotSizeThresholdPercentage float64,
deltaSnapshotSizeThresholdMinSizeBytes int64,
solidEntryPointCheckThresholdPast syncmanager.MilestoneIndexDelta,
solidEntryPointCheckThresholdFuture syncmanager.MilestoneIndexDelta,
additionalPruningThreshold iotago.MilestoneIndex,
snapshotDepth syncmanager.MilestoneIndexDelta,
snapshotInterval iotago.MilestoneIndex,
) *Manager {
Expand All @@ -98,6 +97,7 @@ func NewSnapshotManager(
storage: storage,
syncManager: syncManager,
utxoManager: utxoManager,
snapshotCreationEnabled: snapshotCreationEnabled,
snapshotFullPath: snapshotFullPath,
snapshotDeltaPath: snapshotDeltaPath,
deltaSnapshotSizeThresholdPercentage: deltaSnapshotSizeThresholdPercentage,
Expand All @@ -115,18 +115,24 @@ func NewSnapshotManager(
}

func (s *Manager) MinimumMilestoneIndex() iotago.MilestoneIndex {
minimumIndex := s.syncManager.ConfirmedMilestoneIndex()

snapshotInfo := s.storage.SnapshotInfo()
if snapshotInfo == nil {
s.LogPanic(common.ErrSnapshotInfoNotFound)
if s.snapshotCreationEnabled {

return 0
}
snapshotInfo := s.storage.SnapshotInfo()
if snapshotInfo == nil {
s.LogPanic(common.ErrSnapshotInfoNotFound)

return 0
}

minimumIndex := snapshotInfo.SnapshotIndex()
minimumIndex -= s.snapshotDepth
minimumIndex -= s.solidEntryPointCheckThresholdPast
minimumIndex = snapshotInfo.SnapshotIndex()
minimumIndex -= s.snapshotDepth
}

if minimumIndex >= s.solidEntryPointCheckThresholdPast {
minimumIndex -= s.solidEntryPointCheckThresholdPast
}
return minimumIndex
}

Expand All @@ -138,6 +144,9 @@ func (s *Manager) IsSnapshotting() bool {
}

func (s *Manager) shouldTakeSnapshot(confirmedMilestoneIndex iotago.MilestoneIndex) bool {
if !s.snapshotCreationEnabled {
return false
}

snapshotInfo := s.storage.SnapshotInfo()
if snapshotInfo == nil {
Expand Down
6 changes: 3 additions & 3 deletions tools/gendoc/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ require (
github.com/iotaledger/go-ds-kvstore v1.0.0-beta.2 // indirect
github.com/iotaledger/grocksdb v1.7.5-0.20220808142449-1dc0b8ac4d7d // indirect
github.com/iotaledger/hive.go/serializer/v2 v2.0.0-beta.1 // indirect
github.com/iotaledger/inx-app v1.0.0-beta.7 // indirect
github.com/iotaledger/inx-app v1.0.0-beta.9 // indirect
github.com/iotaledger/inx/go v1.0.0-beta.5 // indirect
github.com/iotaledger/iota.go v1.0.0 // indirect
github.com/iotaledger/iota.go/v3 v3.0.0-beta.6 // indirect
Expand All @@ -79,7 +79,7 @@ require (
github.com/kr/pretty v0.3.0 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/labstack/echo-contrib v0.13.0 // indirect
github.com/labstack/echo/v4 v4.7.2 // indirect
github.com/labstack/echo/v4 v4.8.0 // indirect
github.com/labstack/gommon v0.3.1 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
Expand Down Expand Up @@ -152,7 +152,7 @@ require (
github.com/valyala/fasttemplate v1.2.1 // indirect
github.com/wollac/iota-crypto-demo v0.0.0-20220407192531-0c9bc107c733 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/dig v1.15.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.22.0 // indirect
Expand Down
12 changes: 6 additions & 6 deletions tools/gendoc/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -410,8 +410,8 @@ github.com/iotaledger/hive.go/core v1.0.0-beta.2 h1:+d+lhK8TvbTWkJsHTpnRxsXSiyDz
github.com/iotaledger/hive.go/core v1.0.0-beta.2/go.mod h1:bYl+7/qkM+rWiw8F6JF7dF5/9MW3y8F7j7DxDnTqWeE=
github.com/iotaledger/hive.go/serializer/v2 v2.0.0-beta.1 h1:0NwAA/6N8/pUm7IQdd6rjmExfdwNZigYFhE31fOuQrU=
github.com/iotaledger/hive.go/serializer/v2 v2.0.0-beta.1/go.mod h1:beZKjVT4HPayWfwsmItNNI5E81rS783vGx5ZwRbZQgY=
github.com/iotaledger/inx-app v1.0.0-beta.7 h1:LH5+DoMC5awrPL7SQR+BZvYAdUHH37a/0a4p02m/o6w=
github.com/iotaledger/inx-app v1.0.0-beta.7/go.mod h1:8/OGpwyPNR+MKPXI/knUu0gXBYAwPLAAhCrB/DJ89qw=
github.com/iotaledger/inx-app v1.0.0-beta.9 h1:CZdLhPLXOGz3rosWFBwAVheCMxpiErU3Omqf/80qSh0=
github.com/iotaledger/inx-app v1.0.0-beta.9/go.mod h1:7LnY3qUCiESvM0oRc8qRyV5n0YnCgloVU0v7VSRM2LQ=
github.com/iotaledger/inx/go v1.0.0-beta.5 h1:DVnY6JJghXaLJmoThABhDku27XChApUw96V9a6hm5hc=
github.com/iotaledger/inx/go v1.0.0-beta.5/go.mod h1:/3amwyiPVRnipGfNjAnf1eNC9inmkCJSKrncftn17bc=
github.com/iotaledger/iota.go v1.0.0 h1:tqm1FxJ/zOdzbrAaQ5BQpVF8dUy2eeGlSeWlNG8GoXY=
Expand Down Expand Up @@ -519,8 +519,8 @@ github.com/labstack/echo-contrib v0.13.0 h1:bzSG0SpuZZd7BmJLvsWtPfU23W0Enh3K0tok
github.com/labstack/echo-contrib v0.13.0/go.mod h1:IF9+MJu22ADOZEHD+bAV67XMIO3vNXUy7Naz/ABPHEs=
github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
github.com/labstack/echo/v4 v4.7.2 h1:Kv2/p8OaQ+M6Ex4eGimg9b9e6icoxA42JSlOR3msKtI=
github.com/labstack/echo/v4 v4.7.2/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks=
github.com/labstack/echo/v4 v4.8.0 h1:wdc6yKVaHxkNOEdz4cRZs1pQkwSXPiRjq69yWP4QQS8=
github.com/labstack/echo/v4 v4.8.0/go.mod h1:xkCDAdFCIf8jsFQ5NnbK7oqaF/yU1A1X20Ltm0OvSks=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/labstack/gommon v0.3.1 h1:OomWaJXm7xR6L1HmEtGyQf26TEn7V6X88mktX9kee9o=
github.com/labstack/gommon v0.3.1/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM=
Expand Down Expand Up @@ -909,8 +909,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/dig v1.15.0 h1:vq3YWr8zRj1eFGC7Gvf907hE0eRjPTZ1d3xHadD6liE=
go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
Expand Down

0 comments on commit f20efb0

Please sign in to comment.