From 5d620a72f74c1423b70f61b23d44d928e1306eac Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 5 Apr 2024 13:47:11 +1100 Subject: [PATCH] fix(events): harden tests to assert strict event entry order --- chain/events/filter/index.go | 1 + itests/direct_data_onboard_test.go | 86 +++++++++++++-------- itests/direct_data_onboard_verified_test.go | 22 +++--- itests/sector_terminate_test.go | 2 +- 4 files changed, 65 insertions(+), 46 deletions(-) diff --git a/chain/events/filter/index.go b/chain/events/filter/index.go index e04a403bc32..1ff8bb2c2f1 100644 --- a/chain/events/filter/index.go +++ b/chain/events/filter/index.go @@ -557,6 +557,7 @@ func (ei *EventIndex) prefillFilter(ctx context.Context, f *eventFilter, exclude s = s + " WHERE " + strings.Join(clauses, " AND ") } + // retain insertion order of event_entry rows with the implicit _rowid_ column s += " ORDER BY event.height DESC, event_entry._rowid_ ASC" stmt, err := ei.db.Prepare(s) diff --git a/itests/direct_data_onboard_test.go b/itests/direct_data_onboard_test.go index 037fa1dbb53..3d414ba1c22 100644 --- a/itests/direct_data_onboard_test.go +++ b/itests/direct_data_onboard_test.go @@ -11,6 +11,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/multiformats/go-multicodec" "github.com/stretchr/testify/require" @@ -24,6 +25,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/market" minertypes "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/consensus/filcns" @@ -131,15 +133,17 @@ func TestOnboardMixedMarketDDO(t *testing.T) { var pieces []abi.PieceInfo var dealID abi.DealID + // market ddo piece + var marketSector api.SectorOffset + var marketPiece abi.PieceInfo + marketPieceSize := abi.PaddedPieceSize(2048 / 2).Unpadded() { - // market piece - pieceSize := abi.PaddedPieceSize(2048 / 2).Unpadded() - pieceData := make([]byte, pieceSize) + pieceData := make([]byte, marketPieceSize) _, _ = rand.Read(pieceData) - dc, err := miner.ComputeDataCid(ctx, pieceSize, bytes.NewReader(pieceData)) + marketPiece, err = miner.ComputeDataCid(ctx, marketPieceSize, bytes.NewReader(pieceData)) require.NoError(t, err) - pieces = append(pieces, dc) + pieces = append(pieces, marketPiece) head, err := client.ChainHead(ctx) require.NoError(t, err) @@ -148,7 +152,7 @@ func TestOnboardMixedMarketDDO(t *testing.T) { psdParams := market2.PublishStorageDealsParams{ Deals: []market2.ClientDealProposal{ - makeMarketDealProposal(t, client, miner, dc.PieceCID, pieceSize.Padded(), head.Height()+2880*2, head.Height()+2880*400), + makeMarketDealProposal(t, client, miner, marketPiece.PieceCID, marketPieceSize.Padded(), head.Height()+2880*2, head.Height()+2880*400), }, } @@ -177,7 +181,7 @@ func TestOnboardMixedMarketDDO(t *testing.T) { mcid := smsg.Cid() - so, err := miner.SectorAddPieceToAny(ctx, pieceSize, bytes.NewReader(pieceData), piece.PieceDealInfo{ + marketSector, err = miner.SectorAddPieceToAny(ctx, marketPieceSize, bytes.NewReader(pieceData), piece.PieceDealInfo{ PublishCid: &mcid, DealID: dealID, DealProposal: &psdParams.Deals[0].Proposal, @@ -190,25 +194,26 @@ func TestOnboardMixedMarketDDO(t *testing.T) { }) require.NoError(t, err) - require.Equal(t, abi.PaddedPieceSize(0), so.Offset) - require.Equal(t, abi.SectorNumber(2), so.Sector) + require.Equal(t, abi.PaddedPieceSize(0), marketSector.Offset) + require.Equal(t, abi.SectorNumber(2), marketSector.Sector) } + // raw ddo piece + var rawSector api.SectorOffset + var rawPiece abi.PieceInfo + rawPieceSize := abi.PaddedPieceSize(2048 / 2).Unpadded() { - // raw ddo piece - - pieceSize := abi.PaddedPieceSize(2048 / 2).Unpadded() - pieceData := make([]byte, pieceSize) + pieceData := make([]byte, rawPieceSize) _, _ = rand.Read(pieceData) - dc, err := miner.ComputeDataCid(ctx, pieceSize, bytes.NewReader(pieceData)) + rawPiece, err = miner.ComputeDataCid(ctx, rawPieceSize, bytes.NewReader(pieceData)) require.NoError(t, err) - pieces = append(pieces, dc) + pieces = append(pieces, rawPiece) head, err := client.ChainHead(ctx) require.NoError(t, err) - so, err := miner.SectorAddPieceToAny(ctx, pieceSize, bytes.NewReader(pieceData), piece.PieceDealInfo{ + rawSector, err = miner.SectorAddPieceToAny(ctx, rawPieceSize, bytes.NewReader(pieceData), piece.PieceDealInfo{ PublishCid: nil, DealID: 0, DealProposal: nil, @@ -218,18 +223,20 @@ func TestOnboardMixedMarketDDO(t *testing.T) { }, KeepUnsealed: false, PieceActivationManifest: &minertypes.PieceActivationManifest{ - CID: dc.PieceCID, - Size: dc.Size, + CID: rawPiece.PieceCID, + Size: rawPiece.Size, VerifiedAllocationKey: nil, Notify: nil, }, }) require.NoError(t, err) - require.Equal(t, abi.PaddedPieceSize(1024), so.Offset) - require.Equal(t, abi.SectorNumber(2), so.Sector) + require.Equal(t, abi.PaddedPieceSize(1024), rawSector.Offset) + require.Equal(t, abi.SectorNumber(2), rawSector.Sector) } + require.Equal(t, marketSector.Sector, rawSector.Sector) // sanity check same sector + toCheck := map[abi.SectorNumber]struct{}{ 2: {}, } @@ -272,7 +279,7 @@ func TestOnboardMixedMarketDDO(t *testing.T) { fmt.Println("piece", piece.PieceCID, piece.Size) } - // check "deal-published" actor event + // check some actor events var epochZero abi.ChainEpoch allEvents, err := miner.FullNode.GetActorEventsRaw(ctx, &types.ActorEventFilter{ FromHeight: &epochZero, @@ -282,21 +289,32 @@ func TestOnboardMixedMarketDDO(t *testing.T) { var found bool keyBytes := must.One(ipld.Encode(basicnode.NewString(key), dagcbor.Encode)) for _, event := range allEvents { - for _, e := range event.Entries { - if e.Key == "$type" && bytes.Equal(e.Value, keyBytes) { - found = true - switch key { - case "deal-published", "deal-activated": - expectedEntries := []types.EventEntry{ - {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "$type", Value: keyBytes}, - {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "id", Value: must.One(ipld.Encode(basicnode.NewInt(2), dagcbor.Encode))}, - {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "client", Value: must.One(ipld.Encode(basicnode.NewInt(int64(clientId)), dagcbor.Encode))}, - {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, - } - require.ElementsMatch(t, expectedEntries, event.Entries) + require.True(t, len(event.Entries) > 0) + if event.Entries[0].Key == "$type" && bytes.Equal(event.Entries[0].Value, keyBytes) { + found = true + switch key { + case "deal-published", "deal-activated": + expectedEntries := []types.EventEntry{ + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "$type", Value: keyBytes}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "id", Value: must.One(ipld.Encode(basicnode.NewInt(2), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "client", Value: must.One(ipld.Encode(basicnode.NewInt(int64(clientId)), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, + } + require.Equal(t, expectedEntries, event.Entries) + case "sector-activated": + // only one sector, that has both our pieces in it + expectedEntries := []types.EventEntry{ + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "$type", Value: must.One(ipld.Encode(basicnode.NewString("sector-activated"), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "sector", Value: must.One(ipld.Encode(basicnode.NewInt(int64(rawSector.Sector)), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "unsealed-cid", Value: must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: expectCommD}), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "piece-cid", Value: must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: marketPiece.PieceCID}), dagcbor.Encode))}, + {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "piece-size", Value: must.One(ipld.Encode(basicnode.NewInt(int64(marketPieceSize.Padded())), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "piece-cid", Value: must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: rawPiece.PieceCID}), dagcbor.Encode))}, + {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "piece-size", Value: must.One(ipld.Encode(basicnode.NewInt(int64(rawPieceSize.Padded())), dagcbor.Encode))}, } - break + require.Equal(t, expectedEntries, event.Entries) } + break } } require.True(t, found, "expected to find event %s", key) diff --git a/itests/direct_data_onboard_verified_test.go b/itests/direct_data_onboard_verified_test.go index dd42a1881a5..df87a48a98f 100644 --- a/itests/direct_data_onboard_verified_test.go +++ b/itests/direct_data_onboard_verified_test.go @@ -175,11 +175,11 @@ func TestOnboardRawPieceVerified_WithActorEvents(t *testing.T) { // first sector to start mining is CC {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "sector", Value: must.One(ipld.Encode(basicnode.NewInt(int64(so.Sector)-1), dagcbor.Encode))}, } - require.ElementsMatch(t, expectedEntries, precommitedEvents[0].Entries) + require.Equal(t, expectedEntries, precommitedEvents[0].Entries) // second sector has our piece expectedEntries[1].Value = must.One(ipld.Encode(basicnode.NewInt(int64(so.Sector)), dagcbor.Encode)) - require.ElementsMatch(t, expectedEntries, precommitedEvents[1].Entries) + require.Equal(t, expectedEntries, precommitedEvents[1].Entries) } { @@ -192,7 +192,7 @@ func TestOnboardRawPieceVerified_WithActorEvents(t *testing.T) { {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "sector", Value: must.One(ipld.Encode(basicnode.NewInt(int64(so.Sector)-1), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "unsealed-cid", Value: must.One(ipld.Encode(datamodel.Null, dagcbor.Encode))}, } - require.ElementsMatch(t, expectedEntries, activatedEvents[0].Entries) + require.Equal(t, expectedEntries, activatedEvents[0].Entries) // second sector has our piece, and only our piece, so usealed-cid matches piece-cid, // unfortunately we don't have a case with multiple pieces @@ -202,7 +202,7 @@ func TestOnboardRawPieceVerified_WithActorEvents(t *testing.T) { types.EventEntry{Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "piece-cid", Value: must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: dc.PieceCID}), dagcbor.Encode))}, types.EventEntry{Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "piece-size", Value: must.One(ipld.Encode(basicnode.NewInt(int64(pieceSize.Padded())), dagcbor.Encode))}, ) - require.ElementsMatch(t, expectedEntries, activatedEvents[1].Entries) + require.Equal(t, expectedEntries, activatedEvents[1].Entries) } { @@ -232,21 +232,21 @@ func TestOnboardRawPieceVerified_WithActorEvents(t *testing.T) { {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "$type", Value: must.One(ipld.Encode(basicnode.NewString("allocation"), dagcbor.Encode))}, // first, bogus, allocation {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "id", Value: must.One(ipld.Encode(basicnode.NewInt(int64(allocationId)-1), dagcbor.Encode))}, - {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "client", Value: must.One(ipld.Encode(basicnode.NewInt(int64(clientId)), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "piece-cid", Value: must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: bogusPieceCid}), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "piece-size", Value: must.One(ipld.Encode(basicnode.NewInt(int64(pieceSize.Padded())), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "term-min", Value: must.One(ipld.Encode(basicnode.NewInt(verifregtypes13.MinimumVerifiedAllocationTerm), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "term-max", Value: must.One(ipld.Encode(basicnode.NewInt(verifregtypes13.MaximumVerifiedAllocationTerm), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "expiration", Value: must.One(ipld.Encode(basicnode.NewInt(int64(bogusAllocationExpiry)), dagcbor.Encode))}, } - require.ElementsMatch(t, expectedEntries, allocationEvents[0].Entries) + require.Equal(t, expectedEntries, allocationEvents[0].Entries) // the second, real allocation expectedEntries[1].Value = must.One(ipld.Encode(basicnode.NewInt(int64(allocationId)), dagcbor.Encode)) // "id" expectedEntries[4].Value = must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: dc.PieceCID}), dagcbor.Encode)) // "piece-cid" expectedEntries[8].Value = must.One(ipld.Encode(basicnode.NewInt(verifregtypes13.MaximumVerifiedAllocationExpiration), dagcbor.Encode)) // "expiration" - require.ElementsMatch(t, expectedEntries, allocationEvents[1].Entries) + require.Equal(t, expectedEntries, allocationEvents[1].Entries) } { @@ -257,15 +257,15 @@ func TestOnboardRawPieceVerified_WithActorEvents(t *testing.T) { expectedEntries := []types.EventEntry{ {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "$type", Value: must.One(ipld.Encode(basicnode.NewString("allocation-removed"), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "id", Value: must.One(ipld.Encode(basicnode.NewInt(int64(allocationId)-1), dagcbor.Encode))}, - {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "client", Value: must.One(ipld.Encode(basicnode.NewInt(int64(clientId)), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "piece-cid", Value: must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: bogusPieceCid}), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "piece-size", Value: must.One(ipld.Encode(basicnode.NewInt(int64(pieceSize.Padded())), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "term-min", Value: must.One(ipld.Encode(basicnode.NewInt(verifregtypes13.MinimumVerifiedAllocationTerm), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "term-max", Value: must.One(ipld.Encode(basicnode.NewInt(verifregtypes13.MaximumVerifiedAllocationTerm), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "expiration", Value: must.One(ipld.Encode(basicnode.NewInt(int64(bogusAllocationExpiry)), dagcbor.Encode))}, } - require.ElementsMatch(t, expectedEntries, allocationEvents[0].Entries) + require.Equal(t, expectedEntries, allocationEvents[0].Entries) } { @@ -276,8 +276,8 @@ func TestOnboardRawPieceVerified_WithActorEvents(t *testing.T) { {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "$type", Value: must.One(ipld.Encode(basicnode.NewString("claim"), dagcbor.Encode))}, // claimId inherits from its original allocationId {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "id", Value: must.One(ipld.Encode(basicnode.NewInt(int64(allocationId)), dagcbor.Encode))}, - {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "client", Value: must.One(ipld.Encode(basicnode.NewInt(int64(clientId)), dagcbor.Encode))}, + {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "provider", Value: must.One(ipld.Encode(basicnode.NewInt(int64(minerId)), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "piece-cid", Value: must.One(ipld.Encode(basicnode.NewLink(cidlink.Link{Cid: dc.PieceCID}), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "piece-size", Value: must.One(ipld.Encode(basicnode.NewInt(int64(pieceSize.Padded())), dagcbor.Encode))}, {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "term-min", Value: must.One(ipld.Encode(basicnode.NewInt(verifregtypes13.MinimumVerifiedAllocationTerm), dagcbor.Encode))}, @@ -285,7 +285,7 @@ func TestOnboardRawPieceVerified_WithActorEvents(t *testing.T) { {Flags: 0x01, Codec: uint64(multicodec.Cbor), Key: "term-start", Value: must.One(ipld.Encode(basicnode.NewInt(int64(claimEvents[0].Height)), dagcbor.Encode))}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "sector", Value: must.One(ipld.Encode(basicnode.NewInt(int64(si.SectorID)), dagcbor.Encode))}, } - require.ElementsMatch(t, expectedEntries, claimEvents[0].Entries) + require.Equal(t, expectedEntries, claimEvents[0].Entries) } // verify that we can trace a datacap allocation through to a claim with the events, since this diff --git a/itests/sector_terminate_test.go b/itests/sector_terminate_test.go index ac586ff8f45..f3524c1984f 100644 --- a/itests/sector_terminate_test.go +++ b/itests/sector_terminate_test.go @@ -189,7 +189,7 @@ loop: {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "$type", Value: keyBytes}, {Flags: 0x03, Codec: uint64(multicodec.Cbor), Key: "sector", Value: must.One(ipld.Encode(basicnode.NewInt(int64(toTerminate)), dagcbor.Encode))}, } - require.ElementsMatch(t, expectedEntries, event.Entries) + require.Equal(t, expectedEntries, event.Entries) } break }