Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

les, light: fix CHT trie retrievals #16039

Merged
merged 5 commits into from
Feb 11, 2018
Merged

les, light: fix CHT trie retrievals #16039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cbd605c
les, light: fix CHT trie retrievals
karalabe Feb 8, 2018
ef58aaf
les, light: minor polishes, test remote CHT retrievals
karalabe Feb 9, 2018
390d54c
les, light: deterministic nodeset rlp, bloombits test skeleton
karalabe Feb 10, 2018
4b56783
les: add an event emission to the les bloombits test
karalabe Feb 10, 2018
6e78ee3
les: drop dead tester code
karalabe Feb 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 16 additions & 28 deletions les/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,10 +790,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
break
}
}
proofs := nodes.NodeList()
bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost)
pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost)
return p.SendProofsV2(req.ReqID, bv, proofs)
return p.SendProofsV2(req.ReqID, bv, nodes.NodeList())

case ProofsV1Msg:
if pm.odr == nil {
Expand Down Expand Up @@ -856,15 +855,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) {
return errResp(ErrRequestRejected, "")
}
trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix))
for _, req := range req.Reqs {
if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil {
sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.ChtV1Frequency-1)
sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1)
if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) {
statedb, err := pm.blockchain.State()
if err != nil {
continue
}
trie, err := statedb.Database().OpenTrie(root)
trie, err := trie.New(root, trieDb)
if err != nil {
continue
}
Expand All @@ -878,7 +874,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit {
break
}

}
}
}
Expand Down Expand Up @@ -910,20 +905,16 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
lastIdx uint64
lastType uint
root common.Hash
statedb *state.StateDB
trie state.Trie
auxTrie *trie.Trie
)

nodes := light.NewNodeSet()

for _, req := range req.Reqs {
if trie == nil || req.HelperTrieType != lastType || req.TrieIdx != lastIdx {
statedb, trie, lastType, lastIdx = nil, nil, req.HelperTrieType, req.TrieIdx
if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx {
auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx

if root, _ = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx); root != (common.Hash{}) {
if statedb, _ = pm.blockchain.State(); statedb != nil {
trie, _ = statedb.Database().OpenTrie(root)
}
var prefix string
if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) {
auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix)))
}
}
if req.AuxReq == auxRoot {
Expand All @@ -934,8 +925,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
auxData = append(auxData, data)
auxBytes += len(data)
} else {
if trie != nil {
trie.Prove(req.Key, req.FromLevel, nodes)
if auxTrie != nil {
auxTrie.Prove(req.Key, req.FromLevel, nodes)
}
if req.AuxReq != 0 {
data := pm.getHelperTrieAuxData(req)
Expand All @@ -947,10 +938,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
break
}
}
proofs := nodes.NodeList()
bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost)
pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost)
return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: proofs, AuxData: auxData})
return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData})

case HeaderProofsMsg:
if pm.odr == nil {
Expand Down Expand Up @@ -1123,7 +1113,7 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common.
func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) {
switch id {
case htCanonical:
sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.ChtFrequency-1)
sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1)
return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix
case htBloomBits:
sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1)
Expand All @@ -1134,10 +1124,8 @@ func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, stri

// getHelperTrieAuxData returns requested auxiliary data for the given HelperTrie request
func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte {
if req.HelperTrieType == htCanonical && req.AuxReq == auxHeader {
if len(req.Key) != 8 {
return nil
}
switch {
case req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8:
blockNum := binary.BigEndian.Uint64(req.Key)
hash := core.GetCanonicalHash(pm.chainDb, blockNum)
return core.GetHeaderRLP(pm.chainDb, hash, blockNum)
Expand Down
158 changes: 117 additions & 41 deletions les/handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package les

import (
"bytes"
"encoding/binary"
"math/big"
"math/rand"
"testing"
Expand Down Expand Up @@ -45,27 +45,8 @@ func expectResponse(r p2p.MsgReader, msgcode, reqID, bv uint64, data interface{}
return p2p.ExpectMsg(r, msgcode, resp{reqID, bv, data})
}

func testCheckProof(t *testing.T, exp *light.NodeSet, got light.NodeList) {
if exp.KeyCount() > len(got) {
t.Errorf("proof has fewer nodes than expected")
return
}
if exp.KeyCount() < len(got) {
t.Errorf("proof has more nodes than expected")
return
}
for _, node := range got {
n, _ := exp.Get(crypto.Keccak256(node))
if !bytes.Equal(n, node) {
t.Errorf("proof contents mismatch")
return
}
}
}

// Tests that block headers can be retrieved from a remote chain based on user queries.
func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) }

func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) }

func testGetBlockHeaders(t *testing.T, protocol int) {
Expand Down Expand Up @@ -196,7 +177,6 @@ func testGetBlockHeaders(t *testing.T, protocol int) {

// Tests that block contents can be retrieved from a remote chain based on their hashes.
func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) }

func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) }

func testGetBlockBodies(t *testing.T, protocol int) {
Expand Down Expand Up @@ -274,7 +254,6 @@ func testGetBlockBodies(t *testing.T, protocol int) {

// Tests that the contract codes can be retrieved based on account addresses.
func TestGetCodeLes1(t *testing.T) { testGetCode(t, 1) }

func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) }

func testGetCode(t *testing.T, protocol int) {
Expand Down Expand Up @@ -309,7 +288,6 @@ func testGetCode(t *testing.T, protocol int) {

// Tests that the transaction receipts can be retrieved based on hashes.
func TestGetReceiptLes1(t *testing.T) { testGetReceipt(t, 1) }

func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) }

func testGetReceipt(t *testing.T, protocol int) {
Expand Down Expand Up @@ -338,7 +316,6 @@ func testGetReceipt(t *testing.T, protocol int) {

// Tests that trie merkle proofs can be retrieved
func TestGetProofsLes1(t *testing.T) { testGetProofs(t, 1) }

func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) }

func testGetProofs(t *testing.T, protocol int) {
Expand Down Expand Up @@ -389,27 +366,126 @@ func testGetProofs(t *testing.T, protocol int) {
case 2:
cost := peer.GetRequestCost(GetProofsV2Msg, len(proofreqs))
sendRequest(peer.app, GetProofsV2Msg, 42, cost, proofreqs)
msg, err := peer.app.ReadMsg()
if err != nil {
t.Errorf("Message read error: %v", err)
}
var resp struct {
ReqID, BV uint64
Data light.NodeList
}
if err := msg.Decode(&resp); err != nil {
t.Errorf("reply decode error: %v", err)
if err := expectResponse(peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil {
t.Errorf("proofs mismatch: %v", err)
}
if msg.Code != ProofsV2Msg {
t.Errorf("Message code mismatch")
}
}

// Tests that CHT proofs can be correctly retrieved.
func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) }
func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) }

func testGetCHTProofs(t *testing.T, protocol int) {
// Figure out the client's CHT frequency
frequency := uint64(light.CHTFrequencyClient)
if protocol == 1 {
frequency = uint64(light.CHTFrequencyServer)
}
// Assemble the test environment
db, _ := ethdb.NewMemDatabase()
pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db)
bc := pm.blockchain.(*core.BlockChain)
peer, _ := newTestPeer(t, "peer", protocol, pm, true)
defer peer.close()

// Wait a while for the CHT indexer to process the new headers
time.Sleep(100 * time.Millisecond * time.Duration(frequency/light.CHTFrequencyServer)) // Chain indexer throttling
time.Sleep(250 * time.Millisecond) // CI tester slack

// Assemble the proofs from the different protocols
header := bc.GetHeaderByNumber(frequency)
rlp, _ := rlp.EncodeToBytes(header)

key := make([]byte, 8)
binary.BigEndian.PutUint64(key, frequency)

proofsV1 := []ChtResp{{
Header: header,
}}
proofsV2 := HelperTrieResps{
AuxData: [][]byte{rlp},
}
switch protocol {
case 1:
root := light.GetChtRoot(db, 0, bc.GetHeaderByNumber(frequency-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix)))

var proof light.NodeList
trie.Prove(key, 0, &proof)
proofsV1[0].Proof = proof

case 2:
root := light.GetChtV2Root(db, 0, bc.GetHeaderByNumber(frequency-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix)))
trie.Prove(key, 0, &proofsV2.Proofs)
}
// Assemble the requests for the different protocols
requestsV1 := []ChtReq{{
ChtNum: 1,
BlockNum: frequency,
}}
requestsV2 := []HelperTrieReq{{
Type: htCanonical,
TrieIdx: 0,
Key: key,
AuxReq: auxHeader,
}}
// Send the proof request and verify the response
switch protocol {
case 1:
cost := peer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1))
sendRequest(peer.app, GetHeaderProofsMsg, 42, cost, requestsV1)
if err := expectResponse(peer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil {
t.Errorf("proofs mismatch: %v", err)
}
if resp.ReqID != 42 {
t.Errorf("ReqID mismatch")
case 2:
cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2))
sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2)
if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil {
t.Errorf("proofs mismatch: %v", err)
}
if resp.BV != testBufLimit {
t.Errorf("BV mismatch")
}
}

// Tests that bloombits proofs can be correctly retrieved.
func TestGetBloombitsProofs(t *testing.T) {
// Assemble the test environment
db, _ := ethdb.NewMemDatabase()
pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db)
bc := pm.blockchain.(*core.BlockChain)
peer, _ := newTestPeer(t, "peer", 2, pm, true)
defer peer.close()

// Wait a while for the bloombits indexer to process the new headers
time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling
time.Sleep(250 * time.Millisecond) // CI tester slack

// Request and verify each bit of the bloom bits proofs
for bit := 0; bit < 2048; bit++ {
// Assemble therequest and proofs for the bloombits
key := make([]byte, 10)

binary.BigEndian.PutUint16(key[:2], uint16(bit))
binary.BigEndian.PutUint64(key[2:], uint64(light.BloomTrieFrequency))

requests := []HelperTrieReq{{
Type: htBloomBits,
TrieIdx: 0,
Key: key,
}}
var proofs HelperTrieResps

root := light.GetBloomTrieRoot(db, 0, bc.GetHeaderByNumber(light.BloomTrieFrequency-1).Hash())
trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.BloomTrieTablePrefix)))
trie.Prove(key, 0, &proofs.Proofs)

// Send the proof request and verify the response
cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests))
sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests)
if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil {
t.Errorf("bit %d: proofs mismatch: %v", bit, err)
}
testCheckProof(t, proofsV2, resp.Data)
}
}

Expand Down
26 changes: 22 additions & 4 deletions les/helper_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/les/flowcontrol"
Expand All @@ -55,6 +56,9 @@ var (
testContractCodeDeployed = testContractCode[16:]
testContractDeployed = uint64(2)

testEventEmitterCode = common.Hex2Bytes("60606040523415600e57600080fd5b7f57050ab73f6b9ebdd9f76b8d4997793f48cf956e965ee070551b9ca0bb71584e60405160405180910390a160358060476000396000f3006060604052600080fd00a165627a7a723058203f727efcad8b5811f8cb1fc2620ce5e8c63570d697aef968172de296ea3994140029")
testEventEmitterAddr common.Address

testBufLimit = uint64(100)
)

Expand Down Expand Up @@ -85,15 +89,19 @@ func testChainGen(i int, block *core.BlockGen) {
// In block 2, the test bank sends some more ether to account #1.
// acc1Addr passes it on to account #2.
// acc1Addr creates a test contract.
tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey)
// acc1Addr creates a test event.
nonce := block.TxNonce(acc1Addr)

tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey)
tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key)
nonce++
tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key)
testContractAddr = crypto.CreateAddress(acc1Addr, nonce)
tx3, _ := types.SignTx(types.NewContractCreation(nonce+1, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key)
testContractAddr = crypto.CreateAddress(acc1Addr, nonce+1)
tx4, _ := types.SignTx(types.NewContractCreation(nonce+2, big.NewInt(0), 200000, big.NewInt(0), testEventEmitterCode), signer, acc1Key)
testEventEmitterAddr = crypto.CreateAddress(acc1Addr, nonce+2)
block.AddTx(tx1)
block.AddTx(tx2)
block.AddTx(tx3)
block.AddTx(tx4)
case 2:
// Block 3 is empty but was mined by account #2.
block.SetCoinbase(acc2Addr)
Expand Down Expand Up @@ -147,6 +155,16 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor
chain, _ = light.NewLightChain(odr, gspec.Config, engine)
} else {
blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})

chtIndexer := light.NewChtIndexer(db, false)
chtIndexer.Start(blockchain)

bbtIndexer := light.NewBloomTrieIndexer(db, false)

bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks)
bloomIndexer.AddChildIndexer(bbtIndexer)
bloomIndexer.Start(blockchain)

gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator)
if _, err := blockchain.InsertChain(gchain); err != nil {
panic(err)
Expand Down
Loading