add a Merkle commitment to transactions in block header

config/consensus.go

@@ -359,6 +359,9 @@ const (
 
 	// PaysetCommitFlat hashes the entire payset array.
 	PaysetCommitFlat
+
+	// PaysetCommitMerkle uses merklearray to commit to the payset.
+	PaysetCommitMerkle
 )
 
 // ConsensusProtocols defines a set of supported protocol versions and their
@@ -865,6 +868,9 @@ func initConsensusProtocols() {
 	vFuture.CompactCertWeightThreshold = (1 << 32) * 30 / 100
 	vFuture.CompactCertSecKQ = 128
 
+	// Enable transaction Merkle tree.
+	vFuture.PaysetCommit = PaysetCommitMerkle
+
 	Consensus[protocol.ConsensusFuture] = vFuture
 }
 

crypto/compactcert/verifier.go

@@ -49,11 +49,11 @@ func (v *Verifier) Verify(c *Cert) error {
 	}
 
 	// Verify all of the reveals
-	sigs := make(map[uint64]crypto.Hashable)
-	parts := make(map[uint64]crypto.Hashable)
+	sigs := make(map[uint64]crypto.Digest)
+	parts := make(map[uint64]crypto.Digest)
 	for pos, r := range c.Reveals {
-		sigs[pos] = r.SigSlot
-		parts[pos] = r.Part
+		sigs[pos] = crypto.HashObj(r.SigSlot)
+		parts[pos] = crypto.HashObj(r.Part)
 
 		ephID := basics.OneTimeIDForRound(v.SigRound, r.Part.KeyDilution)
 		if !r.Part.PK.Verify(ephID, v.Msg, r.SigSlot.Sig.OneTimeSignature) {

crypto/merklearray/merkle.go

@@ -167,10 +167,10 @@ func (tree *Tree) Prove(idxs []uint64) ([]crypto.Digest, error) {
 	return s.hints, nil
 }
 
-// Verify ensures that the positions in elems correspond to the hashes of their respective
-// crypto.Hashable objects in a tree with the given root hash.  The proof is expected to
-// be the proof returned by Prove().
-func Verify(root crypto.Digest, elems map[uint64]crypto.Hashable, proof []crypto.Digest) error {
+// Verify ensures that the positions in elems correspond to the respective hashes
+// in a tree with the given root hash.  The proof is expected to be the proof
+// returned by Prove().
+func Verify(root crypto.Digest, elems map[uint64]crypto.Digest, proof []crypto.Digest) error {
 	if len(elems) == 0 {
 		if len(proof) != 0 {
 			return fmt.Errorf("non-empty proof for empty set of elements")
@@ -183,7 +183,7 @@ func Verify(root crypto.Digest, elems map[uint64]crypto.Hashable, proof []crypto
 	for pos, elem := range elems {
 		pl = append(pl, layerItem{
 			pos:  pos,
-			hash: crypto.HashObj(elem),
+			hash: elem,
 		})
 	}
 

crypto/merklearray/merkle_test.go

@@ -91,26 +91,26 @@ func TestMerkle(t *testing.T) {
 		root := tree.Root()
 
 		var allpos []uint64
-		allmap := make(map[uint64]crypto.Hashable)
+		allmap := make(map[uint64]crypto.Digest)
 
 		for i := uint64(0); i < sz; i++ {
 			proof, err := tree.Prove([]uint64{i})
 			if err != nil {
 				t.Error(err)
 			}
 
-			err = Verify(root, map[uint64]crypto.Hashable{i: a[i]}, proof)
+			err = Verify(root, map[uint64]crypto.Digest{i: crypto.HashObj(a[i])}, proof)
 			if err != nil {
 				t.Error(err)
 			}
 
-			err = Verify(root, map[uint64]crypto.Hashable{i: junk}, proof)
+			err = Verify(root, map[uint64]crypto.Digest{i: crypto.HashObj(junk)}, proof)
 			if err == nil {
 				t.Errorf("no error when verifying junk")
 			}
 
 			allpos = append(allpos, i)
-			allmap[i] = a[i]
+			allmap[i] = crypto.HashObj(a[i])
 		}
 
 		proof, err := tree.Prove(allpos)
@@ -123,12 +123,12 @@ func TestMerkle(t *testing.T) {
 			t.Error(err)
 		}
 
-		err = Verify(root, map[uint64]crypto.Hashable{0: junk}, proof)
+		err = Verify(root, map[uint64]crypto.Digest{0: crypto.HashObj(junk)}, proof)
 		if err == nil {
 			t.Errorf("no error when verifying junk batch")
 		}
 
-		err = Verify(root, map[uint64]crypto.Hashable{0: junk}, nil)
+		err = Verify(root, map[uint64]crypto.Digest{0: crypto.HashObj(junk)}, nil)
 		if err == nil {
 			t.Errorf("no error when verifying junk batch")
 		}
@@ -138,18 +138,18 @@ func TestMerkle(t *testing.T) {
 			t.Errorf("no error when proving past the end")
 		}
 
-		err = Verify(root, map[uint64]crypto.Hashable{sz: junk}, nil)
+		err = Verify(root, map[uint64]crypto.Digest{sz: crypto.HashObj(junk)}, nil)
 		if err == nil {
 			t.Errorf("no error when verifying past the end")
 		}
 
 		if sz > 0 {
 			var somepos []uint64
-			somemap := make(map[uint64]crypto.Hashable)
+			somemap := make(map[uint64]crypto.Digest)
 			for i := 0; i < 10; i++ {
 				pos := crypto.RandUint64() % sz
 				somepos = append(somepos, pos)
-				somemap[pos] = a[pos]
+				somemap[pos] = crypto.HashObj(a[pos])
 			}
 
 			proof, err = tree.Prove(somepos)
@@ -234,7 +234,7 @@ func BenchmarkMerkleVerify1M(b *testing.B) {
 	b.ResetTimer()
 
 	for i := uint64(0); i < uint64(b.N); i++ {
-		err := Verify(root, map[uint64]crypto.Hashable{i % a.count: msg}, proofs[i])
+		err := Verify(root, map[uint64]crypto.Digest{i % a.count: crypto.HashObj(msg)}, proofs[i])
 		if err != nil {
 			b.Error(err)
 		}

daemon/algod/api/Makefile

@@ -1,20 +1,23 @@
+GOPATH	:= $(shell go env GOPATH)
+GOPATH1	:= $(firstword $(subst :, ,$(GOPATH)))
+
 # `make all` or just `make` should be appropriate for dev work
 all:	server/v2/generated/types.go server/v2/generated/routes.go server/v2/generated/private/types.go server/v2/generated/private/routes.go
 
 # `make generate` should be able to replace old `generate.sh` script and be appropriate for build system use
 generate:	oapi-codegen all
 
 server/v2/generated/types.go:	algod.oas3.yml
-	oapi-codegen -package generated -type-mappings integer=uint64 -generate types -exclude-tags=private,common -o ./server/v2/generated/types.go algod.oas3.yml
+	$(GOPATH1)/bin/oapi-codegen -package generated -type-mappings integer=uint64 -generate types -exclude-tags=private,common -o ./server/v2/generated/types.go algod.oas3.yml
 
 server/v2/generated/routes.go:	algod.oas3.yml
-	oapi-codegen -package generated -type-mappings integer=uint64 -generate server,spec -exclude-tags=private,common -o ./server/v2/generated/routes.go algod.oas3.yml
+	$(GOPATH1)/bin/oapi-codegen -package generated -type-mappings integer=uint64 -generate server,spec -exclude-tags=private,common -o ./server/v2/generated/routes.go algod.oas3.yml
 
 server/v2/generated/private/types.go:	algod.oas3.yml
-	oapi-codegen -package private -type-mappings integer=uint64 -generate types -include-tags=private -o ./server/v2/generated/private/types.go algod.oas3.yml
+	$(GOPATH1)/bin/oapi-codegen -package private -type-mappings integer=uint64 -generate types -include-tags=private -o ./server/v2/generated/private/types.go algod.oas3.yml
 
 server/v2/generated/private/routes.go:	algod.oas3.yml
-	oapi-codegen -package private -type-mappings integer=uint64 -generate server,spec -include-tags=private -o ./server/v2/generated/private/routes.go algod.oas3.yml
+	$(GOPATH1)/bin/oapi-codegen -package private -type-mappings integer=uint64 -generate server,spec -include-tags=private -o ./server/v2/generated/private/routes.go algod.oas3.yml
 
 algod.oas3.yml:	algod.oas2.json
 	curl -s -X POST "https://converter.swagger.io/api/convert" -H "accept: application/json" -H "Content-Type: application/json" -d @./algod.oas2.json -o .3tmp.json

daemon/algod/api/algod.oas2.json

@@ -358,6 +358,84 @@
         }
       ]
     },
+    "/v2/blocks/{round}/transactions/{txid}/proof": {
+      "get": {
+        "produces": [
+          "application/json"
+        ],
+        "schemes": [
+          "http"
+        ],
+        "summary": "Get a Merkle proof for a transaction in a block.",
+        "operationId": "GetProof",
+        "parameters": [
+          {
+            "type": "integer",
+            "description": "The round in which the transaction appears.",
+            "name": "round",
+            "in": "path",
+            "required": true
+          },
+          {
+            "type": "string",
+            "pattern": "[A-Z0-9]+",
+            "description": "The transaction ID for which to generate a proof.",
+            "name": "txid",
+            "in": "path",
+            "required": true
+          },
+          {
+            "$ref": "#/parameters/format"
+          }
+        ],
+        "responses": {
+          "200": {
+            "$ref": "#/responses/ProofResponse"
+          },
+          "400": {
+            "description": "Malformed round number or transaction ID",
+            "schema": {
+              "$ref": "#/definitions/ErrorResponse"
+            }
+          },
+          "401": {
+            "description": "Invalid API token",
+            "schema": {
+              "$ref": "#/definitions/ErrorResponse"
+            }
+          },
+          "404": {
+            "description": "Non-existent block or transaction",
+            "schema": {
+              "$ref": "#/definitions/ErrorResponse"
+            }
+          },
+          "500": {
+            "description": "Internal error, including protocol not supporting Merkle proofs.",
+            "schema": {
+              "$ref": "#/definitions/ErrorResponse"
+            }
+          },
+          "default": {
+            "description": "Unknown error"
+          }
+        }
+      },
+      "parameters": [
+        {
+          "type": "integer",
+          "name": "round",
+          "in": "path",
+          "required": true
+        },
+        {
+          "type": "string",
+          "name": "txid",
+          "in": "path",
+          "required": true
+        }
+      ]
+    },
     "/v2/ledger/supply": {
       "get": {
         "produces": [
@@ -2008,6 +2086,33 @@
         }
       }
     },
+    "ProofResponse": {
+      "description": "Proof of transaction in a block.",
+      "schema": {
+        "type": "object",
+        "required": [
+          "proof",
+          "stibhash",
+          "idx"
+        ],
+        "properties": {
+          "proof": {
+            "description": "Merkle proof of transaction membership.",
+            "type": "string",
+            "format": "byte"
+          },
+          "stibhash": {
+            "description": "Hash of SignedTxnInBlock for verifying proof.",
+            "type": "string",
+            "format": "byte"
+          },
+          "idx": {
+            "description": "Index of the transaction in the block's payset.",
+            "type": "integer"
+          }
+        }
+      }
+    },
     "CatchpointStartResponse": {
       "tags": [
         "private"