systest: partition tests (#4450)

closes: #4402

the result of the partition could be that some block crosses local threshold, but doesn't cross the global threshold. lets say in total weight the fractions would be 55/45. 55 voting in support, 45 voting against the block. once local threshold is crossed  55-side will be always voting in support, 45-side will be seeing that block crosses local threshold, but not global, but won't have data to vote for the block. as such it won't be possible to reach global threshold indefinitely, as it will always remain at ~10 of total weight, as 45-side will never ask for the block.

the change is to force 45-side to ask for the block even if it crossed local threshold.

other changes:
- make traces shorter by removing all unnecessary data (new types to communicate with tortoise) and encoding byte arrays (ids, node) with base64
- enable lfs for tortoise/data/
- collect traces from multiple ci tests for regression testing

.gitattributes

@@ -0,0 +1 @@
+tortoise/data/*.json filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci.yml

@@ -143,6 +143,8 @@ jobs:
           sudo ethtool -K eth0 rx off
       - name: checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: set up go
         uses: actions/setup-go@v4
         with:
@@ -198,6 +200,8 @@ jobs:
           Disable-NetAdapterChecksumOffload -Name * -TcpIPv4 -UdpIPv4 -TcpIPv6 -UdpIPv6
       - name: checkout
         uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: set up go
         uses: actions/setup-go@v4
         with:

.github/workflows/coverage.yml

@@ -17,6 +17,8 @@ jobs:
           sudo ethtool -K eth0 tx off
           sudo ethtool -K eth0 rx off
       - uses: actions/checkout@v3
+        with:
+          lfs: true
       - name: set up go
         uses: actions/setup-go@v4
         with:

Makefile

@@ -97,7 +97,7 @@ clear-test-cache:
 	go clean -testcache
 .PHONY: clear-test-cache
 
-test: get-libs
+test: get-libs lfs
 	@$(ULIMIT) CGO_LDFLAGS="$(CGO_TEST_LDFLAGS)" gotestsum -- -race -timeout 5m -p 1 $(UNIT_TESTS)
 .PHONY: test
 
@@ -198,3 +198,7 @@ endif
 fuzz:
 	@$(ULIMIT) CGO_LDFLAGS="$(CGO_TEST_LDFLAGS)" ./scripts/fuzz.sh $(FUZZTIME)
 .PHONY: fuzz
+
+lfs:
+	git lfs install
+.PHONY: lfs

cmd/node/node.go

@@ -646,7 +646,7 @@ func (app *App) initServices(ctx context.Context, poetClients []activation.PoetP
 		app.Config.TickSize,
 		goldenATXID,
 		validator,
-		[]activation.AtxReceiver{trtl, beaconProtocol},
+		[]activation.AtxReceiver{&atxReceiver{trtl}, beaconProtocol},
 		app.addLogger(ATXHandlerLogger, lg),
 		poetCfg,
 	)
@@ -1363,3 +1363,11 @@ func (w tortoiseWeakCoin) Set(lid types.LayerID, value bool) error {
 	w.tortoise.OnWeakCoin(lid, value)
 	return nil
 }
+
+type atxReceiver struct {
+	tortoise *tortoise.Tortoise
+}
+
+func (a *atxReceiver) OnAtx(header *types.ActivationTxHeader) {
+	a.tortoise.OnAtx(header.ToData())
+}

cmd/root.go

@@ -192,6 +192,8 @@ func AddCommands(cmd *cobra.Command) {
 		cfg.Tortoise.MaxExceptions, "number of exceptions tolerated for a base ballot")
 	cmd.PersistentFlags().Uint32Var(&cfg.Tortoise.BadBeaconVoteDelayLayers, "tortoise-delay-layers",
 		cfg.Tortoise.BadBeaconVoteDelayLayers, "number of layers to ignore a ballot with a different beacon")
+	cmd.PersistentFlags().BoolVar(&cfg.Tortoise.EnableTracer, "tortoise-enable-tracer",
+		cfg.Tortoise.EnableTracer, "recovrd every tortoise input/output into the loggin output")
 
 	// TODO(moshababo): add usage desc
 	cmd.PersistentFlags().Uint64Var(&cfg.POST.LabelsPerUnit, "post-labels-per-unit",

common/types/activation.go

@@ -9,6 +9,7 @@ import (
 	"github.com/spacemeshos/post/shared"
 
 	"github.com/spacemeshos/go-spacemesh/codec"
+	"github.com/spacemeshos/go-spacemesh/common/util"
 	"github.com/spacemeshos/go-spacemesh/hash"
 	"github.com/spacemeshos/go-spacemesh/log"
 )
@@ -56,6 +57,14 @@ func (t *ATXID) DecodeScale(d *scale.Decoder) (int, error) {
 	return scale.DecodeByteArray(d, t[:])
 }
 
+func (t *ATXID) MarshalText() ([]byte, error) {
+	return util.Base64Encode(t[:]), nil
+}
+
+func (t *ATXID) UnmarshalText(buf []byte) error {
+	return util.Base64Decode(t[:], buf)
+}
+
 // EmptyATXID is a canonical empty ATXID.
 var EmptyATXID = ATXID{}
 

common/types/activation_tx_header.go

@@ -46,6 +46,17 @@ func (atxh *ActivationTxHeader) GetWeight() uint64 {
 	return getWeight(uint64(atxh.EffectiveNumUnits), atxh.TickCount)
 }
 
+func (atxh *ActivationTxHeader) ToData() *AtxTortoiseData {
+	return &AtxTortoiseData{
+		ID:          atxh.ID,
+		Smesher:     atxh.NodeID,
+		TargetEpoch: atxh.TargetEpoch(),
+		BaseHeight:  atxh.BaseTickHeight,
+		Height:      atxh.TickHeight(),
+		Weight:      atxh.GetWeight(),
+	}
+}
+
 func getWeight(numUnits, tickCount uint64) uint64 {
 	return safeMul(numUnits, tickCount)
 }

common/types/ballot.go

@@ -9,6 +9,7 @@ import (
 	"github.com/spacemeshos/go-scale"
 
 	"github.com/spacemeshos/go-spacemesh/codec"
+	"github.com/spacemeshos/go-spacemesh/common/util"
 	"github.com/spacemeshos/go-spacemesh/hash"
 	"github.com/spacemeshos/go-spacemesh/log"
 )
@@ -37,6 +38,14 @@ func (id *BallotID) DecodeScale(d *scale.Decoder) (int, error) {
 	return scale.DecodeByteArray(d, id[:])
 }
 
+func (id *BallotID) MarshalText() ([]byte, error) {
+	return util.Base64Encode(id[:]), nil
+}
+
+func (id *BallotID) UnmarshalText(buf []byte) error {
+	return util.Base64Decode(id[:], buf)
+}
+
 // Ballot contains the smeshers signed vote on the mesh history.
 type Ballot struct {
 	InnerBallot
@@ -152,13 +161,13 @@ type InnerBallot struct {
 //	see https://github.com/spacemeshos/go-spacemesh/issues/2369.
 type Votes struct {
 	// Base ballot.
-	Base BallotID
+	Base BallotID `json:"base"`
 	// Support block id at a particular layer and height.
-	Support []Vote `scale:"max=10000"` // sliding vote window size is 10k layers, vote for one block per layer
+	Support []Vote `scale:"max=10000" json:"support,omitempty"` // sliding vote window size is 10k layers, vote for one block per layer
 	// Against previously supported block.
-	Against []Vote `scale:"max=10000"` // sliding vote window size is 10k layers, vote for one block per layer
+	Against []Vote `scale:"max=10000" json:"against,omitempty"` // sliding vote window size is 10k layers, vote for one block per layer
 	// Abstain on layers until they are terminated.
-	Abstain []LayerID `scale:"max=10000"` // sliding vote window size is 10k layers, vote to abstain on any layer
+	Abstain []LayerID `scale:"max=10000" json:"abstain,omitempty"` // sliding vote window size is 10k layers, vote to abstain on any layer
 }
 
 // MarshalLogObject implements logging interface.
@@ -186,9 +195,9 @@ func (v *Votes) MarshalLogObject(encoder log.ObjectEncoder) error {
 }
 
 type BlockHeader struct {
-	ID      BlockID
-	LayerID LayerID
-	Height  uint64
+	ID      BlockID `json:"id"`
+	LayerID LayerID `json:"lid"`
+	Height  uint64  `json:"height"`
 }
 
 // MarshalLogObject implements logging interface.
@@ -205,8 +214,8 @@ type Vote = BlockHeader
 
 // Opinion is a tuple from opinion hash and votes that decode to opinion hash.
 type Opinion struct {
-	Hash Hash32
-	Votes
+	Hash  Hash32 `json:"hash"`
+	Votes `json:",inline"`
 }
 
 // MarshalLogObject implements logging interface.
@@ -306,6 +315,30 @@ func (b *Ballot) MarshalLogObject(encoder log.ObjectEncoder) error {
 	return nil
 }
 
+func (b *Ballot) ToTortoiseData() *BallotTortoiseData {
+	data := &BallotTortoiseData{
+		ID:            b.ID(),
+		Layer:         b.Layer,
+		Eligibilities: uint32(len(b.EligibilityProofs)),
+		AtxID:         b.AtxID,
+		Opinion: Opinion{
+			Votes: b.Votes,
+			Hash:  b.OpinionHash,
+		},
+		Malicious: b.malicious,
+	}
+	if b.EpochData != nil {
+		data.EpochData = &ReferenceData{
+			Beacon:        b.EpochData.Beacon,
+			Eligibilities: uint32(b.EpochData.EligibilityCount),
+			ActiveSet:     b.ActiveSet,
+		}
+	} else {
+		data.Ref = &b.RefBallot
+	}
+	return data
+}
+
 // ToBallotIDs turns a list of Ballot into a list of BallotID.
 func ToBallotIDs(ballots []*Ballot) []BallotID {
 	ids := make([]BallotID, 0, len(ballots))

common/types/beacon.go

@@ -42,6 +42,14 @@ func (b Beacon) Field() log.Field {
 	return log.String("beacon", b.ShortString())
 }
 
+func (b *Beacon) MarshalText() ([]byte, error) {
+	return util.Base64Encode(b[:]), nil
+}
+
+func (b *Beacon) UnmarshalText(buf []byte) error {
+	return util.Base64Decode(b[:], buf)
+}
+
 // BytesToBeacon sets the first BeaconSize bytes of b to the Beacon's data.
 func BytesToBeacon(b []byte) Beacon {
 	var beacon Beacon

common/types/block.go

@@ -10,6 +10,7 @@ import (
 	"github.com/spacemeshos/go-scale"
 
 	"github.com/spacemeshos/go-spacemesh/codec"
+	"github.com/spacemeshos/go-spacemesh/common/util"
 	"github.com/spacemeshos/go-spacemesh/log"
 )
 
@@ -47,6 +48,14 @@ func (id *BlockID) IsEmpty() bool {
 	return *id == EmptyBlockID
 }
 
+func (id *BlockID) MarshalText() ([]byte, error) {
+	return util.Base64Encode(id[:]), nil
+}
+
+func (id *BlockID) UnmarshalText(buf []byte) error {
+	return util.Base64Decode(id[:], buf)
+}
+
 // Block contains the content of a layer on the mesh history.
 type Block struct {
 	InnerBlock

common/types/nodeid.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/spacemeshos/go-scale"
 
+	"github.com/spacemeshos/go-spacemesh/common/util"
 	"github.com/spacemeshos/go-spacemesh/log"
 )
 
@@ -54,6 +55,14 @@ func (id *NodeID) DecodeScale(d *scale.Decoder) (int, error) {
 	return scale.DecodeByteArray(d, id[:])
 }
 
+func (id *NodeID) MarshalText() ([]byte, error) {
+	return util.Base64Encode(id[:]), nil
+}
+
+func (id *NodeID) UnmarshalText(buf []byte) error {
+	return util.Base64Decode(id[:], buf)
+}
+
 // NodeIDsToHashes turns a list of NodeID into their Hash32 representation.
 func NodeIDsToHashes(ids []NodeID) []Hash32 {
 	hashes := make([]Hash32, 0, len(ids))

common/types/result/result.go

@@ -8,10 +8,10 @@ import (
 )
 
 type Layer struct {
-	Layer    types.LayerID
-	Verified bool
-	Opinion  types.Hash32
-	Blocks   []Block
+	Layer    types.LayerID `json:"lid"`
+	Verified bool          `json:"v"`
+	Opinion  types.Hash32  `json:"opinion"`
+	Blocks   []Block       `json:"blocks"`
 }
 
 // FirstValid returns first block that crossed positive tortoise threshold,
@@ -48,10 +48,12 @@ func (l *Layer) MarshalLogObject(encoder log.ObjectEncoder) error {
 }
 
 type Block struct {
-	Header         types.Vote
-	Valid, Invalid bool
-	Hare           bool
-	Data           bool
+	Header  types.Vote `json:"header"`
+	Valid   bool       `json:"v"`
+	Local   bool       `json:"l"` // set to true if block crossed local threshold
+	Invalid bool       `json:"i"`
+	Hare    bool       `json:"h"`
+	Data    bool       `json:"d"`
 }
 
 func (b *Block) MarshalLogObject(encoder log.ObjectEncoder) error {
@@ -60,5 +62,6 @@ func (b *Block) MarshalLogObject(encoder log.ObjectEncoder) error {
 	encoder.AddBool("invalid", b.Invalid)
 	encoder.AddBool("hare", b.Hare)
 	encoder.AddBool("data", b.Data)
+	encoder.AddBool("local", b.Local)
 	return nil
 }

common/types/tortoise_data.go

@@ -0,0 +1,55 @@
+package types
+
+import "github.com/spacemeshos/go-spacemesh/log"
+
+type AtxTortoiseData struct {
+	ID          ATXID   `json:"id"`
+	Smesher     NodeID  `json:"node"`
+	TargetEpoch EpochID `json:"target"`
+	BaseHeight  uint64  `json:"base"`
+	Height      uint64  `json:"height"`
+	Weight      uint64  `json:"weight"`
+}
+
+type BallotTortoiseData struct {
+	ID            BallotID       `json:"id"`
+	Layer         LayerID        `json:"lid"`
+	Eligibilities uint32         `json:"elig"`
+	AtxID         ATXID          `json:"atxid"`
+	Opinion       Opinion        `json:"opinion"`
+	EpochData     *ReferenceData `json:"epochdata"`
+	Ref           *BallotID      `json:"ref"`
+	Malicious     bool           `json:"mal"`
+}
+
+func (b *BallotTortoiseData) SetMalicious() {
+	b.Malicious = true
+}
+
+func (b *BallotTortoiseData) MarshalLogObject(encoder log.ObjectEncoder) error {
+	encoder.AddString("id", b.ID.String())
+	encoder.AddUint32("layer", b.Layer.Uint32())
+	encoder.AddString("atxid", b.AtxID.String())
+	encoder.AddObject("opinion", &b.Opinion)
+	encoder.AddUint32("elig", b.Eligibilities)
+	if b.EpochData != nil {
+		encoder.AddObject("epochdata", b.EpochData)
+	} else if b.Ref != nil {
+		encoder.AddString("ref", b.Ref.String())
+	}
+	encoder.AddBool("malicious", b.Malicious)
+	return nil
+}
+
+type ReferenceData struct {
+	Beacon        Beacon  `json:"beacon"`
+	ActiveSet     []ATXID `json:"set"`
+	Eligibilities uint32  `json:"elig"`
+}
+
+func (r *ReferenceData) MarshalLogObject(encoder log.ObjectEncoder) error {
+	encoder.AddString("beacon", r.Beacon.String())
+	encoder.AddInt("set size", len(r.ActiveSet))
+	encoder.AddUint32("elig", r.Eligibilities)
+	return nil
+}

common/util/json.go

@@ -17,6 +17,7 @@
 package util
 
 import (
+	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -172,3 +173,21 @@ func mapError(err error) error {
 	}
 	return err
 }
+
+func Base64Encode(src []byte) []byte {
+	n := base64.StdEncoding.EncodedLen(len(src))
+	dst := make([]byte, n)
+	base64.StdEncoding.Encode(dst, src)
+	return dst
+}
+
+func Base64Decode(dst, src []byte) error {
+	n, err := base64.StdEncoding.Decode(dst, src)
+	if err != nil {
+		return err
+	}
+	if n != len(dst) {
+		return fmt.Errorf("incomplete decoding: %d != %d", n, len(src))
+	}
+	return err
+}