feat: hare preround proposal compaction (#6129)

## Motivation

Current `hare` prerounds use 20-byte `blake3` hashes of the proposals. This PR aims to reduce the message size by adopting the changes described in #5606 with further [discussions](https://community.spacemesh.io/t/compact-encoding-for-hare/427) in the research forum

blocks/generator.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/spacemeshos/go-spacemesh/atxsdata"
 	"github.com/spacemeshos/go-spacemesh/common/types"
-	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/hare4"
 	"github.com/spacemeshos/go-spacemesh/log"
 	"github.com/spacemeshos/go-spacemesh/proposals/store"
 	"github.com/spacemeshos/go-spacemesh/sql"
@@ -39,7 +39,7 @@ type Generator struct {
 	cert      certifier
 	patrol    layerPatrol
 
-	hareCh           <-chan hare3.ConsensusOutput
+	hareCh           <-chan hare4.ConsensusOutput
 	optimisticOutput map[types.LayerID]*proposalMetadata
 }
 
@@ -76,7 +76,7 @@ func WithGeneratorLogger(logger *zap.Logger) GeneratorOpt {
 }
 
 // WithHareOutputChan sets the chan to listen to hare output.
-func WithHareOutputChan(ch <-chan hare3.ConsensusOutput) GeneratorOpt {
+func WithHareOutputChan(ch <-chan hare4.ConsensusOutput) GeneratorOpt {
 	return func(g *Generator) {
 		g.hareCh = ch
 	}
@@ -178,7 +178,7 @@ func (g *Generator) run(ctx context.Context) error {
 	}
 }
 
-func (g *Generator) processHareOutput(ctx context.Context, out hare3.ConsensusOutput) (*types.Block, error) {
+func (g *Generator) processHareOutput(ctx context.Context, out hare4.ConsensusOutput) (*types.Block, error) {
 	var md *proposalMetadata
 	if len(out.Proposals) > 0 {
 		getMetadata := func() error {

blocks/generator_test.go

@@ -20,8 +20,8 @@ import (
 	"github.com/spacemeshos/go-spacemesh/blocks/mocks"
 	"github.com/spacemeshos/go-spacemesh/common/types"
 	"github.com/spacemeshos/go-spacemesh/genvm/sdk/wallet"
-	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/hare4"
 	"github.com/spacemeshos/go-spacemesh/proposals/store"
 	"github.com/spacemeshos/go-spacemesh/signing"
 	"github.com/spacemeshos/go-spacemesh/sql"
@@ -56,13 +56,13 @@ type testGenerator struct {
 	mockFetch  *smocks.MockProposalFetcher
 	mockCert   *mocks.Mockcertifier
 	mockPatrol *mocks.MocklayerPatrol
-	hareCh     chan hare3.ConsensusOutput
+	hareCh     chan hare4.ConsensusOutput
 }
 
 func createTestGenerator(t *testing.T) *testGenerator {
 	types.SetLayersPerEpoch(3)
 	ctrl := gomock.NewController(t)
-	ch := make(chan hare3.ConsensusOutput, 100)
+	ch := make(chan hare4.ConsensusOutput, 100)
 	tg := &testGenerator{
 		mockMesh:   mocks.NewMockmeshProvider(ctrl),
 		mockExec:   mocks.NewMockexecutor(ctrl),
@@ -271,7 +271,7 @@ func genData(
 	store *store.Store,
 	lid types.LayerID,
 	optimistic bool,
-) hare3.ConsensusOutput {
+) hare4.ConsensusOutput {
 	numTXs := 1000
 	numProposals := 10
 	txIDs := createAndSaveTxs(t, numTXs, db)
@@ -283,7 +283,7 @@ func genData(
 	}
 	require.NoError(t, layers.SetMeshHash(db, lid.Sub(1), meshHash))
 	plist := createProposals(t, db, store, lid, meshHash, signers, activeSet, txIDs)
-	return hare3.ConsensusOutput{
+	return hare4.ConsensusOutput{
 		Layer:     lid,
 		Proposals: types.ToProposalIDs(plist),
 	}
@@ -448,7 +448,7 @@ func Test_run(t *testing.T) {
 					})
 			tg.mockPatrol.EXPECT().CompleteHare(layerID)
 			tg.Start(context.Background())
-			tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+			tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 			require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 			tg.Stop()
 		})
@@ -464,7 +464,7 @@ func Test_processHareOutput_EmptyOutput(t *testing.T) {
 	tg.mockCert.EXPECT().CertifyIfEligible(gomock.Any(), layerID, types.EmptyBlockID)
 	tg.mockMesh.EXPECT().ProcessLayerPerHareOutput(gomock.Any(), layerID, types.EmptyBlockID, false)
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -480,7 +480,7 @@ func Test_run_FetchFailed(t *testing.T) {
 			return errors.New("unknown")
 		})
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -502,7 +502,7 @@ func Test_run_DiffHasFromConsensus(t *testing.T) {
 
 	tg.mockFetch.EXPECT().GetProposals(gomock.Any(), pids)
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -535,7 +535,7 @@ func Test_run_ExecuteFailed(t *testing.T) {
 				return nil, errors.New("unknown")
 			})
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -560,7 +560,7 @@ func Test_run_AddBlockFailed(t *testing.T) {
 		Return(block, nil)
 	tg.mockMesh.EXPECT().AddBlockWithTXs(gomock.Any(), gomock.Any()).Return(errors.New("unknown"))
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -588,7 +588,7 @@ func Test_run_RegisterCertFailureIgnored(t *testing.T) {
 	tg.mockCert.EXPECT().CertifyIfEligible(gomock.Any(), layerID, gomock.Any())
 	tg.mockMesh.EXPECT().ProcessLayerPerHareOutput(gomock.Any(), layerID, block.ID(), true)
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -616,7 +616,7 @@ func Test_run_CertifyFailureIgnored(t *testing.T) {
 	tg.mockCert.EXPECT().CertifyIfEligible(gomock.Any(), layerID, gomock.Any()).Return(errors.New("unknown"))
 	tg.mockMesh.EXPECT().ProcessLayerPerHareOutput(gomock.Any(), layerID, block.ID(), true)
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -646,7 +646,7 @@ func Test_run_ProcessLayerFailed(t *testing.T) {
 		ProcessLayerPerHareOutput(gomock.Any(), layerID, block.ID(), true).
 		Return(errors.New("unknown"))
 	tg.mockPatrol.EXPECT().CompleteHare(layerID)
-	tg.hareCh <- hare3.ConsensusOutput{Layer: layerID, Proposals: pids}
+	tg.hareCh <- hare4.ConsensusOutput{Layer: layerID, Proposals: pids}
 	require.Eventually(t, func() bool { return len(tg.hareCh) == 0 }, time.Second, 100*time.Millisecond)
 	tg.Stop()
 }
@@ -673,7 +673,7 @@ func Test_processHareOutput_UnequalHeight(t *testing.T) {
 	activeSet := types.ToATXIDs(atxes)
 	pList := createProposals(t, tg.db, tg.proposals, layerID, types.Hash32{}, signers, activeSet, nil)
 	ctx := context.Background()
-	ho := hare3.ConsensusOutput{
+	ho := hare4.ConsensusOutput{
 		Layer:     layerID,
 		Proposals: types.ToProposalIDs(pList),
 	}
@@ -730,7 +730,7 @@ func Test_processHareOutput_bad_state(t *testing.T) {
 			[]types.TransactionID{types.RandomTransactionID()},
 			1,
 		)
-		ho := hare3.ConsensusOutput{
+		ho := hare4.ConsensusOutput{
 			Layer:     layerID,
 			Proposals: types.ToProposalIDs([]*types.Proposal{p}),
 		}
@@ -759,7 +759,7 @@ func Test_processHareOutput_bad_state(t *testing.T) {
 			1,
 		)
 		ctx := context.Background()
-		ho := hare3.ConsensusOutput{
+		ho := hare4.ConsensusOutput{
 			Layer:     layerID,
 			Proposals: types.ToProposalIDs([]*types.Proposal{p}),
 		}
@@ -783,7 +783,7 @@ func Test_processHareOutput_EmptyProposals(t *testing.T) {
 		plist = append(plist, p)
 	}
 	ctx := context.Background()
-	ho := hare3.ConsensusOutput{
+	ho := hare4.ConsensusOutput{
 		Layer:     lid,
 		Proposals: types.ToProposalIDs(plist),
 	}
@@ -832,7 +832,7 @@ func Test_processHareOutput_StableBlockID(t *testing.T) {
 	activeSet := types.ToATXIDs(atxes)
 	plist := createProposals(t, tg.db, tg.proposals, layerID, types.Hash32{}, signers, activeSet, txIDs)
 	ctx := context.Background()
-	ho1 := hare3.ConsensusOutput{
+	ho1 := hare4.ConsensusOutput{
 		Layer:     layerID,
 		Proposals: types.ToProposalIDs(plist),
 	}
@@ -852,7 +852,7 @@ func Test_processHareOutput_StableBlockID(t *testing.T) {
 	ordered := plist[numProposals/2 : numProposals]
 	ordered = append(ordered, plist[0:numProposals/2]...)
 	require.NotEqual(t, plist, ordered)
-	ho2 := hare3.ConsensusOutput{
+	ho2 := hare4.ConsensusOutput{
 		Layer:     layerID,
 		Proposals: types.ToProposalIDs(ordered),
 	}
@@ -881,7 +881,7 @@ func Test_processHareOutput_SameATX(t *testing.T) {
 		createProposal(t, tg.db, tg.proposals, activeSet, layerID, types.Hash32{}, atxID, signers[0], txIDs[0:500], 1),
 		createProposal(t, tg.db, tg.proposals, activeSet, layerID, types.Hash32{}, atxID, signers[0], txIDs[400:], 1),
 	}
-	ho := hare3.ConsensusOutput{
+	ho := hare4.ConsensusOutput{
 		Layer:     layerID,
 		Proposals: types.ToProposalIDs(plist),
 	}
@@ -905,7 +905,7 @@ func Test_processHareOutput_EmptyATXID(t *testing.T) {
 		txIDs, 1,
 	)
 	plist = append(plist, p)
-	ho := hare3.ConsensusOutput{
+	ho := hare4.ConsensusOutput{
 		Layer:     layerID,
 		Proposals: types.ToProposalIDs(plist),
 	}
@@ -928,7 +928,7 @@ func Test_processHareOutput_MultipleEligibilities(t *testing.T) {
 		createProposal(t, tg.db, tg.proposals, activeSet, layerID, types.Hash32{}, atxes[2].ID(), signers[2], ids, 5),
 	}
 	ctx := context.Background()
-	ho := hare3.ConsensusOutput{
+	ho := hare4.ConsensusOutput{
 		Layer:     layerID,
 		Proposals: types.ToProposalIDs(plist),
 	}

common/types/proposal.go

@@ -28,6 +28,18 @@ type ProposalID Hash20
 // EmptyProposalID is a canonical empty ProposalID.
 var EmptyProposalID = ProposalID{}
 
+type CompactProposalID [4]byte
+
+// EncodeScale implements scale codec interface.
+func (id *CompactProposalID) EncodeScale(e *scale.Encoder) (int, error) {
+	return scale.EncodeByteArray(e, id[:])
+}
+
+// DecodeScale implements scale codec interface.
+func (id *CompactProposalID) DecodeScale(d *scale.Decoder) (int, error) {
+	return scale.DecodeByteArray(d, id[:])
+}
+
 // EncodeScale implements scale codec interface.
 func (id *ProposalID) EncodeScale(e *scale.Encoder) (int, error) {
 	return scale.EncodeByteArray(e, id[:])

common/types/testutil.go

@@ -101,12 +101,16 @@ func RandomTransactionID() TransactionID {
 
 // RandomBallot generates a Ballot with random content for testing.
 func RandomBallot() *Ballot {
+	var vrf VrfSignature
+	_, _ = rand.Read(vrf[:])
+
 	return &Ballot{
 		InnerBallot: InnerBallot{
 			Layer:     LayerID(10),
 			AtxID:     RandomATXID(),
 			RefBallot: RandomBallotID(),
 		},
+		EligibilityProofs: []VotingEligibility{{Sig: vrf}},
 		Votes: Votes{
 			Base:    RandomBallotID(),
 			Support: []Vote{{ID: RandomBlockID()}, {ID: RandomBlockID()}},

config/config.go

@@ -22,6 +22,7 @@ import (
 	vm "github.com/spacemeshos/go-spacemesh/genvm"
 	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/hare4"
 	"github.com/spacemeshos/go-spacemesh/miner"
 	"github.com/spacemeshos/go-spacemesh/p2p"
 	"github.com/spacemeshos/go-spacemesh/syncer"
@@ -54,6 +55,7 @@ type Config struct {
 	P2P             p2p.Config                 `mapstructure:"p2p"`
 	API             grpcserver.Config          `mapstructure:"api"`
 	HARE3           hare3.Config               `mapstructure:"hare3"`
+	HARE4           hare4.Config               `mapstructure:"hare4"`
 	HareEligibility eligibility.Config         `mapstructure:"hare-eligibility"`
 	Certificate     blocks.CertConfig          `mapstructure:"certificate"`
 	Beacon          beacon.Config              `mapstructure:"beacon"`
@@ -191,6 +193,7 @@ func DefaultConfig() Config {
 		P2P:             p2p.DefaultConfig(),
 		API:             grpcserver.DefaultConfig(),
 		HARE3:           hare3.DefaultConfig(),
+		HARE4:           hare4.DefaultConfig(), // DEFAULT HARE4 IS DISABLED
 		HareEligibility: eligibility.DefaultConfig(),
 		Beacon:          beacon.DefaultConfig(),
 		TIME:            timeConfig.DefaultConfig(),

config/mainnet.go

@@ -21,6 +21,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/fetch"
 	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/hare4"
 	"github.com/spacemeshos/go-spacemesh/miner"
 	"github.com/spacemeshos/go-spacemesh/p2p"
 	"github.com/spacemeshos/go-spacemesh/syncer"
@@ -68,6 +69,9 @@ func MainnetConfig() Config {
 		Layer: 105_720, // July 15, 2024, 10:00:00 AM UTC
 		Size:  50,
 	}
+
+	hare4conf := hare4.DefaultConfig()
+	hare4conf.Enable = false
 	return Config{
 		BaseConfig: BaseConfig{
 			DataDirParent:         defaultDataDir,
@@ -136,6 +140,7 @@ func MainnetConfig() Config {
 			},
 		},
 		HARE3: hare3conf,
+		HARE4: hare4conf,
 		HareEligibility: eligibility.Config{
 			ConfidenceParam: 200,
 		},

config/presets/testnet.go

@@ -22,6 +22,7 @@ import (
 	"github.com/spacemeshos/go-spacemesh/fetch"
 	"github.com/spacemeshos/go-spacemesh/hare3"
 	"github.com/spacemeshos/go-spacemesh/hare3/eligibility"
+	"github.com/spacemeshos/go-spacemesh/hare4"
 	"github.com/spacemeshos/go-spacemesh/miner"
 	"github.com/spacemeshos/go-spacemesh/p2p"
 	"github.com/spacemeshos/go-spacemesh/syncer"
@@ -55,6 +56,9 @@ func testnet() config.Config {
 	// NOTE(dshulyak) i forgot to set protocol name for testnet when we configured it manually.
 	// we can't do rolling upgrade if protocol name changes, so lets keep it like that temporarily.
 	hare3conf.ProtocolName = ""
+
+	hare4conf := hare4.DefaultConfig()
+	hare4conf.Enable = false
 	defaultdir := filepath.Join(home, "spacemesh-testnet", "/")
 	return config.Config{
 		Preset: "testnet",
@@ -95,6 +99,7 @@ func testnet() config.Config {
 			MinimalActiveSetWeight:   []types.EpochMinimalActiveWeight{{Weight: 10_000}},
 		},
 		HARE3: hare3conf,
+		HARE4: hare4conf,
 		HareEligibility: eligibility.Config{
 			ConfidenceParam: 20,
 		},

hare3/compat/weakcoin.go

@@ -6,14 +6,14 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/spacemeshos/go-spacemesh/common/types"
-	"github.com/spacemeshos/go-spacemesh/hare3"
+	"github.com/spacemeshos/go-spacemesh/hare4"
 )
 
 type weakCoin interface {
 	Set(types.LayerID, bool) error
 }
 
-func ReportWeakcoin(ctx context.Context, logger *zap.Logger, from <-chan hare3.WeakCoinOutput, to weakCoin) {
+func ReportWeakcoin(ctx context.Context, logger *zap.Logger, from <-chan hare4.WeakCoinOutput, to weakCoin) {
 	for {
 		select {
 		case <-ctx.Done():