feat(share): collect absence proof if namespace is not present in mer…

…kle tree (#2242) ## Overview Implements non-inclusion proof collector. Higher level integration (e.g. shrex) is coming as separate PR.
celestiaorg · Jun 15, 2023 · 9f304b4 · 9f304b4
1 parent a4c66f4
commit 9f304b4
Show file tree

Hide file tree

Showing 6 changed files with 234 additions and 97 deletions.
diff --git a/share/get_test.go b/share/get_test.go
@@ -1,8 +1,12 @@
 package share
 
 import (
+	"bytes"
 	"context"
+	"crypto/rand"
+	"errors"
 	mrand "math/rand"
+	"sort"
 	"strconv"
 	"testing"
 	"time"
@@ -141,8 +145,8 @@ func removeRandShares(data [][]byte, d int) [][]byte {
 }
 
 func TestGetSharesByNamespace(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	t.Cleanup(cancel)
 	bServ := mdutils.Bserv()
 
 	var tests = []struct {
@@ -168,6 +172,9 @@ func TestGetSharesByNamespace(t *testing.T) {
 			for _, row := range eds.RowRoots() {
 				rcid := ipld.MustCidFromNamespacedSha256(row)
 				rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots()))
+				if errors.Is(err, ipld.ErrNamespaceOutsideRange) {
+					continue
+				}
 				require.NoError(t, err)
 
 				shares = append(shares, rowShares...)
@@ -182,20 +189,16 @@ func TestGetSharesByNamespace(t *testing.T) {
 }
 
 func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	t.Cleanup(cancel)
 	bServ := mdutils.Bserv()
 
 	shares := RandShares(t, 16)
 
 	// set all shares to the same namespace id
 	nid := shares[0][:NamespaceSize]
 
-	for i, nspace := range shares {
-		if i == len(shares) {
-			break
-		}
-
+	for _, nspace := range shares {
 		copy(nspace[:NamespaceSize], nid)
 	}
 
@@ -230,59 +233,53 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) {
 }
 
 func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	t.Cleanup(cancel)
 	bServ := mdutils.Bserv()
 
 	shares := RandShares(t, 16)
 
-	minNid := make([]byte, NamespaceSize)
-	midNid := make([]byte, NamespaceSize)
-	maxNid := make([]byte, NamespaceSize)
-
-	numberOfShares := len(shares)
-
-	copy(minNid, shares[0][:NamespaceSize])
-	copy(maxNid, shares[numberOfShares-1][:NamespaceSize])
-	copy(midNid, shares[numberOfShares/2][:NamespaceSize])
-
-	// create min nid missing data by replacing first namespace id with second
-	minNidMissingData := make([]Share, len(shares))
-	copy(minNidMissingData, shares)
-	copy(minNidMissingData[0][:NamespaceSize], shares[1][:NamespaceSize])
-
-	// create max nid missing data by replacing last namespace id with second last
-	maxNidMissingData := make([]Share, len(shares))
-	copy(maxNidMissingData, shares)
-	copy(maxNidMissingData[numberOfShares-1][:NamespaceSize], shares[numberOfShares-2][:NamespaceSize])
+	// set all shares to the same namespace id
+	nids, err := randomNids(5)
+	require.NoError(t, err)
+	minNid := nids[0]
+	minIncluded := nids[1]
+	midNid := nids[2]
+	maxIncluded := nids[3]
+	maxNid := nids[4]
 
-	// create mid nid missing data by replacing middle namespace id with the one after
-	midNidMissingData := make([]Share, len(shares))
-	copy(midNidMissingData, shares)
-	copy(midNidMissingData[numberOfShares/2][:NamespaceSize], shares[(numberOfShares/2)+1][:NamespaceSize])
+	secondNamespaceFrom := mrand.Intn(len(shares)-2) + 1
+	for i, nspace := range shares {
+		if i < secondNamespaceFrom {
+			copy(nspace[:NamespaceSize], minIncluded)
+			continue
+		}
+		copy(nspace[:NamespaceSize], maxIncluded)
+	}
 
 	var tests = []struct {
 		name       string
 		data       []Share
 		missingNid []byte
+		isAbsence  bool
 	}{
-		{name: "Namespace id less than the minimum namespace in data", data: minNidMissingData, missingNid: minNid},
-		{name: "Namespace id greater than the maximum namespace in data", data: maxNidMissingData, missingNid: maxNid},
-		{name: "Namespace id in range but still missing", data: midNidMissingData, missingNid: midNid},
+		{name: "Namespace id less than the minimum namespace in data", data: shares, missingNid: minNid},
+		{name: "Namespace id greater than the maximum namespace in data", data: shares, missingNid: maxNid},
+		{name: "Namespace id in range but still missing", data: shares, missingNid: midNid, isAbsence: true},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			eds, err := AddShares(ctx, shares, bServ)
 			require.NoError(t, err)
-			assertNoRowContainsNID(t, bServ, eds, tt.missingNid)
+			assertNoRowContainsNID(ctx, t, bServ, eds, tt.missingNid, tt.isAbsence)
 		})
 	}
 }
 
 func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	t.Cleanup(cancel)
 	bServ := mdutils.Bserv()
 
 	shares := RandShares(t, 16)
@@ -306,6 +303,9 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi
 		rcid := ipld.MustCidFromNamespacedSha256(row)
 		data := ipld.NewNamespaceData(len(shares), nid, ipld.WithLeaves())
 		err := data.CollectLeavesByNamespace(ctx, bServ, rcid)
+		if errors.Is(err, ipld.ErrNamespaceOutsideRange) {
+			continue
+		}
 		assert.Nil(t, err)
 		leaves := data.Leaves()
 		for _, node := range leaves {
@@ -317,7 +317,7 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi
 }
 
 func TestGetSharesWithProofsByNamespace(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	t.Cleanup(cancel)
 	bServ := mdutils.Bserv()
 
@@ -341,7 +341,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) {
 			}
 
 			expected := tt.rawData[from]
-			nID := expected[:NamespaceSize]
+			nID := namespace.ID(expected[:NamespaceSize])
 
 			// change rawData to contain several shares with same nID
 			for i := from; i <= to; i++ {
@@ -356,6 +356,10 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) {
 			for _, row := range eds.RowRoots() {
 				rcid := ipld.MustCidFromNamespacedSha256(row)
 				rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots()))
+				if ipld.NamespaceIsOutsideRange(row, row, nID) {
+					require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange)
+					continue
+				}
 				require.NoError(t, err)
 				if len(rowShares) > 0 {
 					require.NotNil(t, proof)
@@ -432,10 +436,12 @@ func TestBatchSize(t *testing.T) {
 }
 
 func assertNoRowContainsNID(
+	ctx context.Context,
 	t *testing.T,
 	bServ blockservice.BlockService,
 	eds *rsmt2d.ExtendedDataSquare,
 	nID namespace.ID,
+	isAbsent bool,
 ) {
 	rowRootCount := len(eds.RowRoots())
 	// get all row root cids
@@ -445,11 +451,47 @@ func assertNoRowContainsNID(
 	}
 
 	// for each row root cid check if the minNID exists
-	for _, rowCID := range rowRootCIDs {
+	var absentCount, foundAbsenceRows int
+	for _, rowRoot := range eds.RowRoots() {
+		var outsideRange bool
+		if !ipld.NamespaceIsOutsideRange(rowRoot, rowRoot, nID) {
+			// nID does belong to namespace range of the row
+			absentCount++
+		} else {
+			outsideRange = true
+		}
 		data := ipld.NewNamespaceData(rowRootCount, nID, ipld.WithProofs())
-		err := data.CollectLeavesByNamespace(context.Background(), bServ, rowCID)
-		leaves := data.Leaves()
-		assert.Nil(t, leaves)
-		assert.Nil(t, err)
+		rootCID := ipld.MustCidFromNamespacedSha256(rowRoot)
+		err := data.CollectLeavesByNamespace(ctx, bServ, rootCID)
+		if outsideRange {
+			require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange)
+			continue
+		}
+		require.NoError(t, err)
+
+		// if no error returned, check absence proof
+		foundAbsenceRows++
+		verified := data.Proof().VerifyNamespace(sha256.New(), nID, nil, rowRoot)
+		require.True(t, verified)
+	}
+
+	if isAbsent {
+		require.Equal(t, foundAbsenceRows, absentCount)
+		// there should be max 1 row that has namespace range containing nID
+		require.LessOrEqual(t, absentCount, 1)
+	}
+}
+
+func randomNids(total int) ([]namespace.ID, error) {
+	namespaces := make([]namespace.ID, total)
+	for i := range namespaces {
+		nid := make([]byte, NamespaceSize)
+		_, err := rand.Read(nid)
+		if err != nil {
+			return nil, err
+		}
+		namespaces[i] = nid
 	}
+	sort.Slice(namespaces, func(i, j int) bool { return bytes.Compare(namespaces[i], namespaces[j]) < 0 })
+	return namespaces, nil
 }
diff --git a/share/getter.go b/share/getter.go
@@ -10,6 +10,8 @@ import (
 	"github.com/celestiaorg/nmt"
 	"github.com/celestiaorg/nmt/namespace"
 	"github.com/celestiaorg/rsmt2d"
+
+	"github.com/celestiaorg/celestia-node/share/ipld"
 )
 
 var (
@@ -58,7 +60,7 @@ type NamespacedRow struct {
 func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error {
 	originalRoots := make([][]byte, 0)
 	for _, row := range root.RowRoots {
-		if !nID.Less(nmt.MinNamespace(row, nID.Size())) && nID.LessOrEqual(nmt.MaxNamespace(row, nID.Size())) {
+		if !ipld.NamespaceIsOutsideRange(row, row, nID) {
 			originalRoots = append(originalRoots, row)
 		}
 	}

diff --git a/share/getters/utils.go b/share/getters/utils.go
@@ -15,7 +15,6 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/celestiaorg/nmt"
 	"github.com/celestiaorg/nmt/namespace"
 
 	"github.com/celestiaorg/celestia-node/libs/utils"
@@ -35,7 +34,7 @@ var (
 func filterRootsByNamespace(root *share.Root, nID namespace.ID) []cid.Cid {
 	rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots))
 	for _, row := range root.RowRoots {
-		if !nID.Less(nmt.MinNamespace(row, nID.Size())) && nID.LessOrEqual(nmt.MaxNamespace(row, nID.Size())) {
+		if !ipld.NamespaceIsOutsideRange(row, row, nID) {
 			rowRootCIDs = append(rowRootCIDs, ipld.MustCidFromNamespacedSha256(row))
 		}
 	}

diff --git a/share/ipld/get.go b/share/ipld/get.go
@@ -103,7 +103,7 @@ func GetLeaves(ctx context.Context,
 
 	// this buffer ensures writes to 'jobs' are never blocking (bin-tree-feat)
 	jobs := make(chan *job, (maxShares+1)/2) // +1 for the case where 'maxShares' is 1
-	jobs <- &job{id: root, ctx: ctx}
+	jobs <- &job{cid: root, ctx: ctx}
 	// total is an amount of routines spawned and total amount of nodes we process (bin-tree-feat)
 	// so we can specify exact amount of loops we do, and wait for this amount
 	// of routines to finish processing
@@ -123,11 +123,11 @@ func GetLeaves(ctx context.Context,
 				defer wg.Done()
 
 				span.SetAttributes(
-					attribute.String("cid", j.id.String()),
+					attribute.String("cid", j.cid.String()),
 					attribute.Int("pos", j.sharePos),
 				)
 
-				nd, err := GetNode(ctx, bGetter, j.id)
+				nd, err := GetNode(ctx, bGetter, j.cid)
 				if err != nil {
 					// we don't really care about errors here
 					// just fetch as much as possible
@@ -149,7 +149,7 @@ func GetLeaves(ctx context.Context,
 					// send those to be processed
 					select {
 					case jobs <- &job{
-						id: lnk.Cid,
+						cid: lnk.Cid,
 						// calc position for children nodes (bin-tree-feat),
 						// s.t. 'if' above knows where to put a share
 						sharePos: j.sharePos*2 + i,
@@ -213,7 +213,7 @@ func GetProof(
 // chanGroup implements an atomic wait group, closing a jobs chan
 // when fully done.
 type chanGroup struct {
-	jobs    chan *job
+	jobs    chan job
 	counter int64
 }
 
@@ -233,8 +233,29 @@ func (w *chanGroup) done() {
 // job represents an encountered node to investigate during the `GetLeaves`
 // and `CollectLeavesByNamespace` routines.
 type job struct {
-	id       cid.Cid
+	// we pass the context to job so that spans are tracked in a tree
+	// structure
+	ctx context.Context
+	// cid of the node that will be handled
+	cid cid.Cid
+	// sharePos represents potential share position in share slice
 	sharePos int
-	depth    int
-	ctx      context.Context
+	// depth represents the number of edges present in path from the root node of a tree to that node
+	depth int
+	// isAbsent indicates if target namespaceID is not included, only collect absence proofs
+	isAbsent bool
+}
+
+func (j job) next(direction direction, cid cid.Cid, isAbsent bool) job {
+	var i int
+	if direction == right {
+		i++
+	}
+	return job{
+		ctx:      j.ctx,
+		cid:      cid,
+		sharePos: j.sharePos*2 + i,
+		depth:    j.depth + 1,
+		isAbsent: isAbsent,
+	}
 }