miner: use atxsdata to generate activeset and persist it (#5552)

related: #5192 #5106

this change accomplishes three things:
- removes lru cache from miner module, after this change it won't be on paths sensitive to timings and can be reduced
- allows to prepare activeset in advance, before epoch starts
- persists prepared activeset, so that node doesn't have to redo the work after restart

in order to prepare activeset we don't have to wait for the first layer of the epoch, all the data required for activeset
must be downloaded as soon as two hours prior to epoch start, but in configuration i set preparation to start 30m before the epoch start.

also this process becomes more and more expensive with the number of atxs growth, and there is a benefit of persisting activeset locally.

codec/codec.go

@@ -96,7 +96,7 @@ func Decode(buf []byte, value Decodable) error {
 	return nil
 }
 
-// EncodeSlice encodes slice to a buffer.
+// EncodeSlice encodes slice with a length prefix.
 func EncodeSlice[V any, H scale.EncodablePtr[V]](value []V) ([]byte, error) {
 	var b bytes.Buffer
 	_, err := scale.EncodeStructSlice[V, H](scale.NewEncoder(&b), value)
@@ -106,15 +106,39 @@ func EncodeSlice[V any, H scale.EncodablePtr[V]](value []V) ([]byte, error) {
 	return b.Bytes(), nil
 }
 
-// DecodeSlice decodes slice from a buffer.
-func DecodeSlice[V any, H scale.DecodablePtr[V]](buf []byte) ([]V, error) {
-	v, _, err := scale.DecodeStructSlice[V, H](scale.NewDecoder(bytes.NewReader(buf)))
+// MustEncodeSlice encodes slice with a length prefix or panics on error.
+func MustEncodeSlice[V any, H scale.EncodablePtr[V]](value []V) []byte {
+	buf, err := EncodeSlice[V, H](value)
+	if err != nil {
+		panic(err)
+	}
+	return buf
+}
+
+// DecodeSliceFromReader accepts a reader and decodes slice with a length prefix.
+func DecodeSliceFromReader[V any, H scale.DecodablePtr[V]](r io.Reader) ([]V, error) {
+	v, _, err := scale.DecodeStructSlice[V, H](scale.NewDecoder(r))
 	if err != nil {
 		return nil, fmt.Errorf("decode struct slice: %w", err)
 	}
 	return v, nil
 }
 
+// MustDecodeSliceFromReader decodes slice with a length prefix or panics on error.
+func MustDecodeSliceFromReader[V any, H scale.DecodablePtr[V]](r io.Reader) []V {
+	v, err := DecodeSliceFromReader[V, H](r)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+// DecodeSlice decodes slice from a buffer.
+func DecodeSlice[V any, H scale.DecodablePtr[V]](buf []byte) ([]V, error) {
+	return DecodeSliceFromReader[V, H](bytes.NewReader(buf))
+}
+
+// EncodeCompact16 encodes uint16 to a buffer.
 // ReadSlice decodes slice from am io.Reader.
 func ReadSlice[V any, H scale.DecodablePtr[V]](r io.Reader) ([]V, int, error) {
 	v, n, err := scale.DecodeStructSlice[V, H](scale.NewDecoder(r))

config/config.go

@@ -22,6 +22,7 @@ import (
 	vm "github.com/spacemeshos/go-spacemesh/genvm"
 	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/miner"
 	"github.com/spacemeshos/go-spacemesh/p2p"
 	"github.com/spacemeshos/go-spacemesh/syncer"
 	timeConfig "github.com/spacemeshos/go-spacemesh/timesync/config"
@@ -60,14 +61,15 @@ type Config struct {
 	VM              vm.Config             `mapstructure:"vm"`
 	POST            activation.PostConfig `mapstructure:"post"`
 	POSTService     activation.PostSupervisorConfig
-	POET            activation.PoetConfig `mapstructure:"poet"`
-	SMESHING        SmeshingConfig        `mapstructure:"smeshing"`
-	LOGGING         LoggerConfig          `mapstructure:"logging"`
-	FETCH           fetch.Config          `mapstructure:"fetch"`
-	Bootstrap       bootstrap.Config      `mapstructure:"bootstrap"`
-	Sync            syncer.Config         `mapstructure:"syncer"`
-	Recovery        checkpoint.Config     `mapstructure:"recovery"`
-	Cache           datastore.Config      `mapstructure:"cache"`
+	POET            activation.PoetConfig      `mapstructure:"poet"`
+	SMESHING        SmeshingConfig             `mapstructure:"smeshing"`
+	LOGGING         LoggerConfig               `mapstructure:"logging"`
+	FETCH           fetch.Config               `mapstructure:"fetch"`
+	Bootstrap       bootstrap.Config           `mapstructure:"bootstrap"`
+	Sync            syncer.Config              `mapstructure:"syncer"`
+	Recovery        checkpoint.Config          `mapstructure:"recovery"`
+	Cache           datastore.Config           `mapstructure:"cache"`
+	ActiveSet       miner.ActiveSetPreparation `mapstructure:"active-set-preparation"`
 }
 
 // DataDir returns the absolute path to use for the node's data. This is the tilde-expanded path given in the config
@@ -197,6 +199,7 @@ func DefaultConfig() Config {
 		Sync:            syncer.DefaultConfig(),
 		Recovery:        checkpoint.DefaultConfig(),
 		Cache:           datastore.DefaultConfig(),
+		ActiveSet:       miner.DefaultActiveSetPrepartion(),
 	}
 }
 

config/mainnet.go

@@ -21,6 +21,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/fetch"
 	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/miner"
 	"github.com/spacemeshos/go-spacemesh/p2p"
 	"github.com/spacemeshos/go-spacemesh/syncer"
 	"github.com/spacemeshos/go-spacemesh/syncer/atxsync"
@@ -190,5 +191,10 @@ func MainnetConfig() Config {
 		},
 		Recovery: checkpoint.DefaultConfig(),
 		Cache:    datastore.DefaultConfig(),
+		ActiveSet: miner.ActiveSetPreparation{
+			Window:        60 * time.Minute,
+			RetryInterval: time.Minute,
+			Tries:         20,
+		},
 	}
 }

config/presets/testnet.go

@@ -22,6 +22,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/fetch"
 	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/miner"
 	"github.com/spacemeshos/go-spacemesh/p2p"
 	"github.com/spacemeshos/go-spacemesh/syncer"
 	"github.com/spacemeshos/go-spacemesh/syncer/atxsync"
@@ -152,5 +153,10 @@ func testnet() config.Config {
 			// this will be upgraded in future with scheduled upgrade.
 			CommitteeSize: 200,
 		},
+		ActiveSet: miner.ActiveSetPreparation{
+			Window:        10 * time.Minute,
+			RetryInterval: time.Minute,
+			Tries:         5,
+		},
 	}
 }