Integrate mss and vc (#3512)

* compact cert use VC

* use vc in MSS

* proof encoding contains leading zeros

* fix some nits

* CR fix

compactcert/worker_test.go

@@ -159,7 +159,7 @@ func (s *testWorkerStubs) CompactCertVoters(r basics.Round) (*ledgercore.VotersF
 		})
 	}
 
-	tree, err := merklearray.Build(voters.Participants, crypto.HashFactory{HashType: compactcert.HashType})
+	tree, err := merklearray.BuildVectorCommitmentTree(voters.Participants, crypto.HashFactory{HashType: compactcert.HashType})
 	if err != nil {
 		return nil, err
 	}

crypto/compactcert/builder.go

@@ -176,7 +176,7 @@ func (b *Builder) Build() (*Cert, error) {
 	b.sigsHasValidL = true
 
 	hfactory := crypto.HashFactory{HashType: HashType}
-	sigtree, err := merklearray.Build(committableSignatureSlotArray(b.sigs), hfactory)
+	sigtree, err := merklearray.BuildVectorCommitmentTree(committableSignatureSlotArray(b.sigs), hfactory)
 	if err != nil {
 		return nil, err
 	}

crypto/compactcert/builder_test.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"github.com/algoidan/falcon"
 	"hash"
+	"math/bits"
 	"testing"
 
 	"github.com/algorand/go-algorand/crypto"
@@ -110,7 +111,7 @@ func TestBuildVerify(t *testing.T) {
 		sigs = append(sigs, sig)
 	}
 
-	partcom, err := merklearray.Build(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
+	partcom, err := merklearray.BuildVectorCommitmentTree(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
 	if err != nil {
 		t.Error(err)
 	}
@@ -202,7 +203,7 @@ func TestParticipationCommitmentBinaryFormat(t *testing.T) {
 	parts = append(parts, generateRandomParticipant(a, t.Name()))
 	parts = append(parts, generateRandomParticipant(a, t.Name()))
 
-	partcom, err := merklearray.Build(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
+	partcom, err := merklearray.BuildVectorCommitmentTree(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
 	a.NoError(err)
 
 	partCommitmentRoot := partcom.Root()
@@ -212,8 +213,8 @@ func TestParticipationCommitmentBinaryFormat(t *testing.T) {
 	leaf2 := calculateHashOnPartLeaf(parts[2])
 	leaf3 := calculateHashOnPartLeaf(parts[3])
 
-	inner1 := calculateHashOnInternalNode(leaf0, leaf1)
-	inner2 := calculateHashOnInternalNode(leaf2, leaf3)
+	inner1 := calculateHashOnInternalNode(leaf0, leaf2)
+	inner2 := calculateHashOnInternalNode(leaf1, leaf3)
 
 	calcRoot := calculateHashOnInternalNode(inner1, inner2)
 
@@ -255,7 +256,7 @@ func TestSignatureCommitmentBinaryFormat(t *testing.T) {
 
 	}
 
-	partcom, err := merklearray.Build(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
+	partcom, err := merklearray.BuildVectorCommitmentTree(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
 	a.NoError(err)
 
 	b, err := MkBuilder(param, parts, partcom)
@@ -274,8 +275,9 @@ func TestSignatureCommitmentBinaryFormat(t *testing.T) {
 	leaf2 := calculateHashOnSigLeaf(t, sigs[2], findLInCert(a, sigs[2], cert))
 	leaf3 := calculateHashOnSigLeaf(t, sigs[3], findLInCert(a, sigs[3], cert))
 
-	inner1 := calculateHashOnInternalNode(leaf0, leaf1)
-	inner2 := calculateHashOnInternalNode(leaf2, leaf3)
+	// hash internal node according to the vector commitment indices
+	inner1 := calculateHashOnInternalNode(leaf0, leaf2)
+	inner2 := calculateHashOnInternalNode(leaf1, leaf3)
 
 	calcRoot := calculateHashOnInternalNode(inner1, inner2)
 
@@ -353,10 +355,11 @@ func checkSignature(a *require.Assertions, sigBytes []byte, verifier *merklekeys
 }
 
 func verifyMerklePath(idx uint64, pathLe byte, sigBytes []byte, parsedBytes int, leafHash []byte) []byte {
-	// idxDirection will indicate which sibling we should fetch LSB to MSB leaf-to-root
-	// todo when change to vector commitment this needs to be changed.
-	idxDirection := idx
+	// idxDirection will indicate which sibling we should fetch MSB to LSB leaf-to-root
+	idxDirection := bits.Reverse64(idx) >> (64 - pathLe)
+
 	// use the verification path to hash siblings up to the root
+	parsedBytes += (16 - int(pathLe)) * 64
 	for i := uint8(0); i < pathLe; i++ {
 		var innerNodeBytes []byte
 
@@ -451,7 +454,7 @@ func BenchmarkBuildVerify(b *testing.B) {
 	}
 
 	var cert *Cert
-	partcom, err := merklearray.Build(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
+	partcom, err := merklearray.BuildVectorCommitmentTree(basics.ParticipantsArray(parts), crypto.HashFactory{HashType: HashType})
 	if err != nil {
 		b.Error(err)
 	}

crypto/compactcert/commitableSignatureSlot_test.go

@@ -45,7 +45,7 @@ func TestSignatureArrayWithEmptySlot(t *testing.T) {
 	}
 
 	hfactory := crypto.HashFactory{HashType: HashType}
-	tree, err := merklearray.Build(committableSignatureSlotArray(sigs), hfactory)
+	tree, err := merklearray.BuildVectorCommitmentTree(committableSignatureSlotArray(sigs), hfactory)
 
 	leftLeafHash := calculateHashOnSigLeaf(t, sig, 60)
 	rightLeafHash := hashBytes(hfactory.NewHash(), []byte(protocol.CompactCertSig))
@@ -85,17 +85,16 @@ func calculateHashOnSigLeaf(t *testing.T, sig merklekeystore.Signature, lValue u
 
 	sigCommitment = append(sigCommitment, proofLenByte)
 
-	i := byte(0)
-	for ; i < proofLenByte; i++ {
-		sigCommitment = append(sigCommitment, sig.Proof.Path[i]...)
-	}
-
 	hash := crypto.HashFactory{HashType: HashType}.NewHash()
 	zeroDigest := make([]byte, hash.BlockSize())
-	for ; i < merklearray.MaxEncodedTreeDepth; i++ {
+	for i := byte(0); i < (merklearray.MaxEncodedTreeDepth - proofLenByte); i++ {
 		sigCommitment = append(sigCommitment, zeroDigest...)
 	}
 
+	for i := byte(0); i < proofLenByte; i++ {
+		sigCommitment = append(sigCommitment, sig.Proof.Path[i]...)
+	}
+
 	hashValue := hashBytes(hash, sigCommitment)
 	return hashValue
 }

crypto/compactcert/verifier.go

@@ -70,12 +70,12 @@ func (v *Verifier) Verify(c *Cert) error {
 	}
 
 	// verify all the reveals proofs on the signature tree.
-	if err := merklearray.Verify(crypto.GenericDigest(c.SigCommit[:]), sigs, &c.SigProofs); err != nil {
+	if err := merklearray.VerifyVectorCommitment(c.SigCommit[:], sigs, &c.SigProofs); err != nil {
 		return err
 	}
 
 	// verify all the reveals proofs on the participant tree.
-	if err := merklearray.Verify(crypto.GenericDigest(v.partcom[:]), parts, &c.PartProofs); err != nil {
+	if err := merklearray.VerifyVectorCommitment(v.partcom[:], parts, &c.PartProofs); err != nil {
 		return err
 	}
 

crypto/merklearray/proof.go

@@ -49,7 +49,7 @@ type SingleLeafProof struct {
 // The function returns a fixed length array for each hash function. which is 1 + MaxEncodedTreeDepth * digestsize
 //
 // the path is guaranteed to be less than MaxEncodedTreeDepth and if the path length is less
-// than MaxEncodedTreeDepth, array is padded with zeros.
+// than MaxEncodedTreeDepth, array will have leading zeros (to fill the array to MaxEncodedTreeDepth * digestsize).
 // more details could be found in the Algorand's spec.
 func (p *SingleLeafProof) GetFixedLengthHashableRepresentation() []byte {
 	hash := p.HashFactory.NewHash()
@@ -60,7 +60,12 @@ func (p *SingleLeafProof) GetFixedLengthHashableRepresentation() []byte {
 	binProof = append(binProof, proofLenByte)
 
 	zeroDigest := make([]byte, hash.Size())
-	for i := uint8(0); i < MaxEncodedTreeDepth; i++ {
+
+	for i := uint8(0); i < (MaxEncodedTreeDepth - proofLenByte); i++ {
+		binProof = append(binProof, zeroDigest...)
+	}
+
+	for i := uint8(0); i < proofLenByte; i++ {
 		if i < proofLenByte && p.Path[i] != nil {
 			binProof = append(binProof, p.Path[i]...)
 		} else {

crypto/merklearray/proof_test.go

@@ -25,39 +25,108 @@ import (
 
 func TestProofSerialization(t *testing.T) {
 	partitiontest.PartitionTest(t)
+	a := require.New(t)
 
-	var junk TestData
-	crypto.RandBytes(junk[:])
-
-	a := make(TestArray, 3)
+	array := make(TestArray, 3)
 	for i := uint64(0); i < 3; i++ {
-		crypto.RandBytes(a[i][:])
+		crypto.RandBytes(array[i][:])
 	}
 
-	tree, err := Build(a, crypto.HashFactory{HashType: crypto.Sha512_256})
-	require.NoError(t, err)
+	tree, err := Build(array, crypto.HashFactory{HashType: crypto.Sha512_256})
+	a.NoError(err)
 
 	// creates a proof with missing child
 	p, err := tree.ProveSingleLeaf(2)
-	require.NoError(t, err)
+	a.NoError(err)
 
 	data := p.GetFixedLengthHashableRepresentation()
-	require.Equal(t, len(data), 1+(MaxEncodedTreeDepth*crypto.Sha512_256Size))
+	a.Equal(len(data), 1+(MaxEncodedTreeDepth*crypto.Sha512_256Size))
 
 	// check the padded results
 	zeroDigest := make([]byte, crypto.Sha512_256Size)
-	require.Equal(t, data[1+((MaxEncodedTreeDepth-1)*crypto.Sha512_256Size):], zeroDigest)
-
-	var newPath []crypto.GenericDigest
-	for i := 0; i < MaxEncodedTreeDepth+1; i++ {
-		var junkDigest [crypto.Sha512_256Size]byte
-		crypto.RandBytes(junkDigest[:])
-		newPath = append(newPath, junkDigest[:])
+	i := 0
+	proofData := data[1:]
+	for ; i < (MaxEncodedTreeDepth - 2); i++ {
+		a.Equal(zeroDigest, proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
 	}
 
-	p.Path = newPath
-	p.TreeDepth = uint8(len(newPath))
+	// first proof digest is nil -> so the HashableRepresentation is zeros
+	a.Equal(crypto.GenericDigest(nil), p.Path[0])
+	a.Equal(zeroDigest, proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
+	i++
+
+	a.Equal([]byte(p.Path[1]), proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
+
+	//VC
+	tree, err = BuildVectorCommitmentTree(array, crypto.HashFactory{HashType: crypto.Sha512_256})
+	a.NoError(err)
+
+	// creates a proof with missing child
+	p, err = tree.ProveSingleLeaf(2)
+	a.NoError(err)
+
 	data = p.GetFixedLengthHashableRepresentation()
-	require.Equal(t, len(data), 1+(MaxEncodedTreeDepth*crypto.Sha512_256Size))
-	require.Equal(t, data[1+((MaxEncodedTreeDepth-1)*crypto.Sha512_256Size):], []byte(p.Path[MaxEncodedTreeDepth-1]))
+	a.Equal(len(data), 1+(MaxEncodedTreeDepth*crypto.Sha512_256Size))
+
+	// check the padded results
+	zeroDigest = make([]byte, crypto.Sha512_256Size)
+	i = 0
+	proofData = data[1:]
+	for ; i < (MaxEncodedTreeDepth - 2); i++ {
+		a.Equal(zeroDigest, proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
+	}
+
+	a.Equal([]byte(p.Path[0]), proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
+	i++
+	a.Equal([]byte(p.Path[1]), proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
+
+}
+
+func TestProofSerializationMaxTree(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	a := require.New(t)
+
+	array := make(TestArray, MaxNumLeavesOnEncodedTree)
+	for i := uint64(0); i < MaxNumLeavesOnEncodedTree; i++ {
+		crypto.RandBytes(array[i][:])
+	}
+
+	tree, err := BuildVectorCommitmentTree(array, crypto.HashFactory{HashType: crypto.Sha512_256})
+	a.NoError(err)
+
+	p, err := tree.ProveSingleLeaf(2)
+	a.NoError(err)
+
+	data := p.GetFixedLengthHashableRepresentation()
+	a.Equal(len(data), 1+(MaxEncodedTreeDepth*crypto.Sha512_256Size))
+
+	proofData := data[1:]
+	for i := 0; i < MaxEncodedTreeDepth; i++ {
+		a.Equal([]byte(p.Path[i]), proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
+	}
+}
+
+func TestProofSerializationOneLeafTree(t *testing.T) {
+	partitiontest.PartitionTest(t)
+	a := require.New(t)
+
+	array := make(TestArray, 1)
+	crypto.RandBytes(array[0][:])
+
+	tree, err := BuildVectorCommitmentTree(array, crypto.HashFactory{HashType: crypto.Sha512_256})
+	a.NoError(err)
+
+	p, err := tree.ProveSingleLeaf(0)
+	a.NoError(err)
+
+	data := p.GetFixedLengthHashableRepresentation()
+	a.Equal(len(data), 1+(MaxEncodedTreeDepth*crypto.Sha512_256Size))
+
+	zeroDigest := make([]byte, crypto.Sha512_256Size)
+
+	proofData := data[1:]
+	for i := 0; i < MaxEncodedTreeDepth; i++ {
+		a.Equal(zeroDigest, proofData[crypto.Sha512_256Size*i:crypto.Sha512_256Size*(i+1)])
+	}
+
 }

crypto/merklekeystore/committablePublicKeys_test.go

@@ -87,8 +87,9 @@ func TestEphemeralPublicKeysCommitmentBinaryFormat(t *testing.T) {
 	k3 := signer.GetSigner(8).SigningKey
 	k3hash := calculateHashOnKeyLeaf(k3, 8)
 
-	internal1 := calculateHashOnInternalNode(k0hash, k1hash)
-	internal2 := calculateHashOnInternalNode(k2hash, k3hash)
+	// hash internal node according to the vector commitment indices
+	internal1 := calculateHashOnInternalNode(k0hash, k2hash)
+	internal2 := calculateHashOnInternalNode(k1hash, k3hash)
 
 	root := calculateHashOnInternalNode(internal1, internal2)
 	a.Equal(root, signer.GetVerifier()[:])

crypto/merklekeystore/keystore.go

@@ -110,8 +110,7 @@ func New(firstValid, lastValid, interval uint64, sigAlgoType crypto.AlgorithmTyp
 	if err != nil {
 		return nil, err
 	}
-
-	tree, err := merklearray.Build(&CommittablePublicKeyArray{keys, firstValid, interval}, crypto.HashFactory{HashType: KeyStoreHashFunction})
+	tree, err := merklearray.BuildVectorCommitmentTree(&CommittablePublicKeyArray{keys, firstValid, interval}, crypto.HashFactory{HashType: KeyStoreHashFunction})
 	if err != nil {
 		return nil, err
 	}
@@ -210,7 +209,7 @@ func (v *Verifier) Verify(round uint64, msg crypto.Hashable, sig Signature) erro
 
 	// verify the merkle tree verification path using the ephemeral public key, the
 	// verification path and the index.
-	err := merklearray.Verify(
+	err := merklearray.VerifyVectorCommitment(
 		v[:],
 		map[uint64]crypto.Hashable{sig.MerkleArrayIndex: &ephkey},
 		sig.Proof.ToProof(),

crypto/merklekeystore/keystore_test.go

@@ -94,7 +94,9 @@ func TestEmptyVerifier(t *testing.T) {
 	a := require.New(t)
 
 	signer := generateTestSigner(crypto.FalconType, 8, 9, 5, a)
-	a.Equal(signer.GetVerifier().IsEmpty(), true)
+	// even if there are no keys for that period, the root is not empty
+	// (part of the vector commitment property).
+	a.Equal(signer.GetVerifier().IsEmpty(), false)
 }
 
 func TestEmptySigner(t *testing.T) {

ledger/ledgercore/votersForRound.go

@@ -115,7 +115,7 @@ func (tr *VotersForRound) LoadTree(onlineTop TopOnlineAccounts, hdr bookkeeping.
 		addrToPos[acct.Address] = uint64(i)
 	}
 
-	tree, err := merklearray.Build(participants, crypto.HashFactory{HashType: compactcert.HashType})
+	tree, err := merklearray.BuildVectorCommitmentTree(participants, crypto.HashFactory{HashType: compactcert.HashType})
 	if err != nil {
 		return err
 	}