From cbd605cfc9c50248e87085852ac284561c2e88e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 8 Feb 2018 16:24:04 +0200 Subject: [PATCH 1/5] les, light: fix CHT trie retrievals --- les/handler.go | 27 ++++++++++----------------- les/handler_test.go | 5 ----- les/server.go | 6 ++---- light/postprocess.go | 5 ++--- 4 files changed, 14 insertions(+), 29 deletions(-) diff --git a/les/handler.go b/les/handler.go index 5c93133fb73e..5ba7e940228b 100644 --- a/les/handler.go +++ b/les/handler.go @@ -856,15 +856,12 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) { return errResp(ErrRequestRejected, "") } + trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for _, req := range req.Reqs { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.ChtV1Frequency-1) if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - trie, err := statedb.Database().OpenTrie(root) + trie, err := trie.New(root, trieDb) if err != nil { continue } @@ -910,20 +907,16 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { lastIdx uint64 lastType uint root common.Hash - statedb *state.StateDB - trie state.Trie + auxTrie *trie.Trie ) - nodes := light.NewNodeSet() - for _, req := range req.Reqs { - if trie == nil || req.HelperTrieType != lastType || req.TrieIdx != lastIdx { - statedb, trie, lastType, lastIdx = nil, nil, req.HelperTrieType, req.TrieIdx + if auxTrie == nil || req.HelperTrieType != lastType || req.TrieIdx != lastIdx { + auxTrie, lastType, lastIdx = nil, req.HelperTrieType, req.TrieIdx - if root, _ = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx); root != (common.Hash{}) { - if statedb, _ = pm.blockchain.State(); statedb != nil { - trie, _ = statedb.Database().OpenTrie(root) - } + var prefix string + if root, prefix = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx); root != (common.Hash{}) { + auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) } } if req.AuxReq == auxRoot { @@ -934,8 +927,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { auxData = append(auxData, data) auxBytes += len(data) } else { - if trie != nil { - trie.Prove(req.Key, req.FromLevel, nodes) + if auxTrie != nil { + auxTrie.Prove(req.Key, req.FromLevel, nodes) } if req.AuxReq != 0 { data := pm.getHelperTrieAuxData(req) diff --git a/les/handler_test.go b/les/handler_test.go index e5446c031dd9..e7cb06f9bf66 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -65,7 +65,6 @@ func testCheckProof(t *testing.T, exp *light.NodeSet, got light.NodeList) { // Tests that block headers can be retrieved from a remote chain based on user queries. func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } - func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func testGetBlockHeaders(t *testing.T, protocol int) { @@ -196,7 +195,6 @@ func testGetBlockHeaders(t *testing.T, protocol int) { // Tests that block contents can be retrieved from a remote chain based on their hashes. func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) } - func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func testGetBlockBodies(t *testing.T, protocol int) { @@ -274,7 +272,6 @@ func testGetBlockBodies(t *testing.T, protocol int) { // Tests that the contract codes can be retrieved based on account addresses. func TestGetCodeLes1(t *testing.T) { testGetCode(t, 1) } - func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func testGetCode(t *testing.T, protocol int) { @@ -309,7 +306,6 @@ func testGetCode(t *testing.T, protocol int) { // Tests that the transaction receipts can be retrieved based on hashes. func TestGetReceiptLes1(t *testing.T) { testGetReceipt(t, 1) } - func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func testGetReceipt(t *testing.T, protocol int) { @@ -338,7 +334,6 @@ func testGetReceipt(t *testing.T, protocol int) { // Tests that trie merkle proofs can be retrieved func TestGetProofsLes1(t *testing.T) { testGetProofs(t, 1) } - func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func testGetProofs(t *testing.T, protocol int) { diff --git a/les/server.go b/les/server.go index 85ebbf898825..072fc36e5e44 100644 --- a/les/server.go +++ b/les/server.go @@ -20,7 +20,6 @@ package les import ( "crypto/ecdsa" "encoding/binary" - "fmt" "math" "sync" @@ -81,15 +80,14 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { chtLastSectionV1 := (chtLastSection+1)*(light.ChtFrequency/light.ChtV1Frequency) - 1 chtSectionHead := srv.chtIndexer.SectionHead(chtLastSectionV1) chtRoot := light.GetChtV2Root(pm.chainDb, chtLastSection, chtSectionHead) - logger.Info("CHT", "section", chtLastSection, "sectionHead", fmt.Sprintf("%064x", chtSectionHead), "root", fmt.Sprintf("%064x", chtRoot)) + logger.Info("Loaded CHT", "section", chtLastSection, "head", chtSectionHead, "root", chtRoot) } - bloomTrieSectionCount, _, _ := srv.bloomTrieIndexer.Sections() if bloomTrieSectionCount != 0 { bloomTrieLastSection := bloomTrieSectionCount - 1 bloomTrieSectionHead := srv.bloomTrieIndexer.SectionHead(bloomTrieLastSection) bloomTrieRoot := light.GetBloomTrieRoot(pm.chainDb, bloomTrieLastSection, bloomTrieSectionHead) - logger.Info("BloomTrie", "section", bloomTrieLastSection, "sectionHead", fmt.Sprintf("%064x", bloomTrieSectionHead), "root", fmt.Sprintf("%064x", bloomTrieRoot)) + logger.Info("Loaded bloom trie", "section", bloomTrieLastSection, "head", bloomTrieSectionHead, "root", bloomTrieRoot) } srv.chtIndexer.Start(eth.BlockChain()) diff --git a/light/postprocess.go b/light/postprocess.go index 160d07b175ca..8b1418800d40 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -19,7 +19,6 @@ package light import ( "encoding/binary" "errors" - "fmt" "math/big" "time" @@ -175,7 +174,7 @@ func (c *ChtIndexerBackend) Commit() error { c.triedb.Commit(root, false) if ((c.section+1)*c.sectionSize)%ChtFrequency == 0 { - log.Info("Storing CHT", "idx", c.section*c.sectionSize/ChtFrequency, "sectionHead", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) + log.Info("Storing CHT", "section", c.section*c.sectionSize/ChtFrequency, "head", c.lastHash, "root", root) } StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil @@ -294,7 +293,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.triedb.Commit(root, false) sectionHead := b.sectionHeads[b.bloomTrieRatio-1] - log.Info("Storing BloomTrie", "section", b.section, "sectionHead", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression ratio", float64(compSize)/float64(decompSize)) + log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) return nil From ef58aaf253804130518a9950d34d78be242784cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 10 Feb 2018 01:02:16 +0200 Subject: [PATCH 2/5] les, light: minor polishes, test remote CHT retrievals --- les/handler.go | 16 ++++----- les/handler_test.go | 77 ++++++++++++++++++++++++++++++++++++++++++++ les/helper_test.go | 7 ++++ les/odr_requests.go | 18 +++++------ les/peer.go | 5 ++- les/server.go | 4 +-- light/lightchain.go | 6 ++-- light/odr_util.go | 4 +-- light/postprocess.go | 20 ++++++++---- 9 files changed, 122 insertions(+), 35 deletions(-) diff --git a/les/handler.go b/les/handler.go index 5ba7e940228b..dc9bd12088c8 100644 --- a/les/handler.go +++ b/les/handler.go @@ -859,7 +859,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for _, req := range req.Reqs { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.ChtV1Frequency-1) + sectionHead := core.GetCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { trie, err := trie.New(root, trieDb) if err != nil { @@ -875,7 +875,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit { break } - } } } @@ -911,11 +910,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { ) nodes := light.NewNodeSet() for _, req := range req.Reqs { - if auxTrie == nil || req.HelperTrieType != lastType || req.TrieIdx != lastIdx { - auxTrie, lastType, lastIdx = nil, req.HelperTrieType, req.TrieIdx + if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx { + auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx var prefix string - if root, prefix = pm.getHelperTrie(req.HelperTrieType, req.TrieIdx); root != (common.Hash{}) { + if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) } } @@ -940,10 +939,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } } - proofs := nodes.NodeList() bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: proofs, AuxData: auxData}) + return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}) case HeaderProofsMsg: if pm.odr == nil { @@ -1116,7 +1114,7 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common. func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) { switch id { case htCanonical: - sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.ChtFrequency-1) + sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix case htBloomBits: sectionHead := core.GetCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) @@ -1127,7 +1125,7 @@ func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, stri // getHelperTrieAuxData returns requested auxiliary data for the given HelperTrie request func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte { - if req.HelperTrieType == htCanonical && req.AuxReq == auxHeader { + if req.Type == htCanonical && req.AuxReq == auxHeader { if len(req.Key) != 8 { return nil } diff --git a/les/handler_test.go b/les/handler_test.go index e7cb06f9bf66..df25f9097cf5 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -18,6 +18,7 @@ package les import ( "bytes" + "encoding/binary" "math/big" "math/rand" "testing" @@ -408,6 +409,82 @@ func testGetProofs(t *testing.T, protocol int) { } } +// Tests that helper trie proofs can be correctly retrieved. +func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) } +func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) } + +func testGetCHTProofs(t *testing.T, protocol int) { + // Figure out the client's CHT frequency + frequency := uint64(light.CHTFrequencyClient) + if protocol == 1 { + frequency = uint64(light.CHTFrequencyServer) + } + // Assemble the test environment + db, _ := ethdb.NewMemDatabase() + pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db) + bc := pm.blockchain.(*core.BlockChain) + peer, _ := newTestPeer(t, "peer", protocol, pm, true) + defer peer.close() + + // Wait a while for the CHT indexer to process the new headers + time.Sleep(100 * time.Millisecond * time.Duration(frequency/light.CHTFrequencyServer)) // Chain indexer throttling + time.Sleep(250 * time.Millisecond) // CI tester slack + + // Assemble the proofs from the different protocols + header := bc.GetHeaderByNumber(frequency) + rlp, _ := rlp.EncodeToBytes(header) + + key := make([]byte, 8) + binary.BigEndian.PutUint64(key, frequency) + + proofsV1 := []ChtResp{{ + Header: header, + }} + proofsV2 := HelperTrieResps{ + AuxData: [][]byte{rlp}, + } + switch protocol { + case 1: + root := light.GetChtRoot(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + + var proof light.NodeList + trie.Prove(key, 0, &proof) + proofsV1[0].Proof = proof + + case 2: + root := light.GetChtV2Root(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + trie.Prove(key, 0, &proofsV2.Proofs) + } + // Assemble the requests for the different protocols + requestsV1 := []ChtReq{{ + ChtNum: 1, + BlockNum: frequency, + }} + requestsV2 := []HelperTrieReq{{ + Type: htCanonical, + TrieIdx: 0, + Key: key, + AuxReq: auxHeader, + }} + // Send the proof request and verify the response + switch protocol { + case 1: + cost := peer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1)) + sendRequest(peer.app, GetHeaderProofsMsg, 42, cost, requestsV1) + if err := expectResponse(peer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil { + t.Errorf("proofs mismatch: %v", err) + } + case 2: + cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2)) + sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2) + if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { + t.Errorf("proofs mismatch: %v", err) + } + } +} + func TestTransactionStatusLes2(t *testing.T) { db, _ := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db) diff --git a/les/helper_test.go b/les/helper_test.go index bf08e1e2f7ab..a408d186092c 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -147,6 +147,13 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain, _ = light.NewLightChain(odr, gspec.Config, engine) } else { blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) + + chtIndexer := light.NewChtIndexer(db, false) + chtIndexer.Start(blockchain) + + bloomIndexer := light.NewBloomTrieIndexer(db, false) + bloomIndexer.Start(blockchain) + gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) diff --git a/les/odr_requests.go b/les/odr_requests.go index 937a4f1d9d3f..a11d7b396538 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -321,7 +321,7 @@ const ( ) type HelperTrieReq struct { - HelperTrieType uint + Type uint TrieIdx uint64 Key []byte FromLevel, AuxReq uint @@ -365,7 +365,7 @@ func (r *ChtRequest) CanSend(peer *peer) bool { peer.lock.RLock() defer peer.lock.RUnlock() - return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.ChtFrequency + return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.CHTFrequencyClient } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -374,10 +374,10 @@ func (r *ChtRequest) Request(reqID uint64, peer *peer) error { var encNum [8]byte binary.BigEndian.PutUint64(encNum[:], r.BlockNum) req := HelperTrieReq{ - HelperTrieType: htCanonical, - TrieIdx: r.ChtNum, - Key: encNum[:], - AuxReq: auxHeader, + Type: htCanonical, + TrieIdx: r.ChtNum, + Key: encNum[:], + AuxReq: auxHeader, } return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []HelperTrieReq{req}) } @@ -498,9 +498,9 @@ func (r *BloomRequest) Request(reqID uint64, peer *peer) error { for i, sectionIdx := range r.SectionIdxList { binary.BigEndian.PutUint64(encNumber[2:10], sectionIdx) reqs[i] = HelperTrieReq{ - HelperTrieType: htBloomBits, - TrieIdx: r.BloomTrieNum, - Key: common.CopyBytes(encNumber[:]), + Type: htBloomBits, + TrieIdx: r.BloomTrieNum, + Key: common.CopyBytes(encNumber[:]), } } return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), reqs) diff --git a/les/peer.go b/les/peer.go index b72c80d35a1d..caf568077805 100644 --- a/les/peer.go +++ b/les/peer.go @@ -281,7 +281,6 @@ func (p *peer) RequestProofs(reqID, cost uint64, reqs []ProofReq) error { default: panic(nil) } - } // RequestHelperTrieProofs fetches a batch of HelperTrie merkle proofs from a remote node. @@ -291,12 +290,12 @@ func (p *peer) RequestHelperTrieProofs(reqID, cost uint64, reqs []HelperTrieReq) case lpv1: reqsV1 := make([]ChtReq, len(reqs)) for i, req := range reqs { - if req.HelperTrieType != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { + if req.Type != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { return fmt.Errorf("Request invalid in LES/1 mode") } blockNum := binary.BigEndian.Uint64(req.Key) // convert HelperTrie request to old CHT request - reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.ChtFrequency / light.ChtV1Frequency), BlockNum: blockNum, FromLevel: req.FromLevel} + reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.CHTFrequencyClient / light.CHTFrequencyServer), BlockNum: blockNum, FromLevel: req.FromLevel} } return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqsV1) case lpv2: diff --git a/les/server.go b/les/server.go index 072fc36e5e44..621385ffac9e 100644 --- a/les/server.go +++ b/les/server.go @@ -72,12 +72,12 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { logger := log.New() chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility - chtV2SectionCount := chtV1SectionCount / (light.ChtFrequency / light.ChtV1Frequency) + chtV2SectionCount := chtV1SectionCount / (light.CHTFrequencyClient / light.CHTFrequencyServer) if chtV2SectionCount != 0 { // convert to LES/2 section chtLastSection := chtV2SectionCount - 1 // convert last LES/2 section index back to LES/1 index for chtIndexer.SectionHead - chtLastSectionV1 := (chtLastSection+1)*(light.ChtFrequency/light.ChtV1Frequency) - 1 + chtLastSectionV1 := (chtLastSection+1)*(light.CHTFrequencyClient/light.CHTFrequencyServer) - 1 chtSectionHead := srv.chtIndexer.SectionHead(chtLastSectionV1) chtRoot := light.GetChtV2Root(pm.chainDb, chtLastSection, chtSectionHead) logger.Info("Loaded CHT", "section", chtLastSection, "head", chtSectionHead, "root", chtRoot) diff --git a/light/lightchain.go b/light/lightchain.go index bc88aeb4870c..181a1c2a629b 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -127,7 +127,7 @@ func (self *LightChain) addTrustedCheckpoint(cp trustedCheckpoint) { if self.odr.BloomIndexer() != nil { self.odr.BloomIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) } - log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*ChtFrequency-1, "hash", cp.sectionHead) + log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*CHTFrequencyClient-1, "hash", cp.sectionHead) } func (self *LightChain) getProcInterrupt() bool { @@ -453,8 +453,8 @@ func (self *LightChain) SyncCht(ctx context.Context) bool { } headNum := self.CurrentHeader().Number.Uint64() chtCount, _, _ := self.odr.ChtIndexer().Sections() - if headNum+1 < chtCount*ChtFrequency { - num := chtCount*ChtFrequency - 1 + if headNum+1 < chtCount*CHTFrequencyClient { + num := chtCount*CHTFrequencyClient - 1 header, err := GetHeaderByNumber(ctx, self.odr, num) if header != nil && err == nil { self.mu.Lock() diff --git a/light/odr_util.go b/light/odr_util.go index 33a8e80ce5f5..8f92d64426ed 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -52,13 +52,13 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { chtCount-- if chtCount > 0 { - sectionHeadNum = chtCount*ChtFrequency - 1 + sectionHeadNum = chtCount*CHTFrequencyClient - 1 sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1) canonicalHash = core.GetCanonicalHash(db, sectionHeadNum) } } } - if number >= chtCount*ChtFrequency { + if number >= chtCount*CHTFrequencyClient { return nil, ErrNoTrustedCht } r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number} diff --git a/light/postprocess.go b/light/postprocess.go index 8b1418800d40..b6756de51066 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -34,8 +34,14 @@ import ( ) const ( - ChtFrequency = 32768 - ChtV1Frequency = 4096 // as long as we want to retain LES/1 compatibility, servers generate CHTs with the old, higher frequency + // CHTFrequencyClient is the block frequency for creating CHTs on the client side. + CHTFrequencyClient = 32768 + + // CHTFrequencyServer is the block frequency for creating CHTs on the server side. + // Eventually this can be merged back with the client version, but that requires a + // full database upgrade, so that should be left for a suitable moment. + CHTFrequencyServer = 4096 + HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated ) @@ -99,7 +105,7 @@ func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) c // GetChtV2Root reads the CHT root assoctiated to the given section from the database // Note that sectionIdx is specified according to LES/2 CHT section size func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { - return GetChtRoot(db, (sectionIdx+1)*(ChtFrequency/ChtV1Frequency)-1, sectionHead) + return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) } // StoreChtRoot writes the CHT root assoctiated to the given section into the database @@ -123,10 +129,10 @@ type ChtIndexerBackend struct { func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { var sectionSize, confirmReq uint64 if clientMode { - sectionSize = ChtFrequency + sectionSize = CHTFrequencyClient confirmReq = HelperTrieConfirmations } else { - sectionSize = ChtV1Frequency + sectionSize = CHTFrequencyServer confirmReq = HelperTrieProcessConfirmations } idb := ethdb.NewTable(db, "chtIndex-") @@ -173,8 +179,8 @@ func (c *ChtIndexerBackend) Commit() error { } c.triedb.Commit(root, false) - if ((c.section+1)*c.sectionSize)%ChtFrequency == 0 { - log.Info("Storing CHT", "section", c.section*c.sectionSize/ChtFrequency, "head", c.lastHash, "root", root) + if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { + log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) } StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil From 390d54c3e09043d3e58e4f5de1096d2fee8c8027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 10 Feb 2018 03:38:42 +0200 Subject: [PATCH 3/5] les, light: deterministic nodeset rlp, bloombits test skeleton --- les/handler.go | 3 +-- les/handler_test.go | 59 +++++++++++++++++++++++++++++---------------- les/helper_test.go | 6 ++++- les/odr_requests.go | 8 +++--- light/nodeset.go | 27 +++++++++++++-------- 5 files changed, 65 insertions(+), 38 deletions(-) diff --git a/les/handler.go b/les/handler.go index dc9bd12088c8..734befeeb25b 100644 --- a/les/handler.go +++ b/les/handler.go @@ -790,10 +790,9 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { break } } - proofs := nodes.NodeList() bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendProofsV2(req.ReqID, bv, proofs) + return p.SendProofsV2(req.ReqID, bv, nodes.NodeList()) case ProofsV1Msg: if pm.odr == nil { diff --git a/les/handler_test.go b/les/handler_test.go index df25f9097cf5..fa59fe66dee4 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -385,31 +385,13 @@ func testGetProofs(t *testing.T, protocol int) { case 2: cost := peer.GetRequestCost(GetProofsV2Msg, len(proofreqs)) sendRequest(peer.app, GetProofsV2Msg, 42, cost, proofreqs) - msg, err := peer.app.ReadMsg() - if err != nil { - t.Errorf("Message read error: %v", err) - } - var resp struct { - ReqID, BV uint64 - Data light.NodeList - } - if err := msg.Decode(&resp); err != nil { - t.Errorf("reply decode error: %v", err) - } - if msg.Code != ProofsV2Msg { - t.Errorf("Message code mismatch") - } - if resp.ReqID != 42 { - t.Errorf("ReqID mismatch") - } - if resp.BV != testBufLimit { - t.Errorf("BV mismatch") + if err := expectResponse(peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { + t.Errorf("proofs mismatch: %v", err) } - testCheckProof(t, proofsV2, resp.Data) } } -// Tests that helper trie proofs can be correctly retrieved. +// Tests that CHT proofs can be correctly retrieved. func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) } func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) } @@ -485,6 +467,41 @@ func testGetCHTProofs(t *testing.T, protocol int) { } } +// Tests that bloombits proofs can be correctly retrieved. +func TestGetBloombitsProofs(t *testing.T) { + // Assemble the test environment + db, _ := ethdb.NewMemDatabase() + pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) + //bc := pm.blockchain.(*core.BlockChain) + peer, _ := newTestPeer(t, "peer", 2, pm, true) + defer peer.close() + + // Wait a while for the bloombits indexer to process the new headers + time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling + time.Sleep(250 * time.Millisecond) // CI tester slack + + // Assemble therequest and proofs for the bloombits + bit := rand.Intn(2048) + key := make([]byte, 10) + + binary.BigEndian.PutUint16(key[:2], uint16(bit)) + binary.BigEndian.PutUint64(key[2:], 0) + + requests := []HelperTrieReq{{ + Type: htBloomBits, + TrieIdx: 0, + Key: key, + }} + var proofs HelperTrieResps + + // Send the proof request and verify the response + cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) + sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests) + if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { + t.Errorf("proofs mismatch: %v", err) + } +} + func TestTransactionStatusLes2(t *testing.T) { db, _ := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db) diff --git a/les/helper_test.go b/les/helper_test.go index a408d186092c..e1e311b44ba1 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/les/flowcontrol" @@ -151,7 +152,10 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chtIndexer := light.NewChtIndexer(db, false) chtIndexer.Start(blockchain) - bloomIndexer := light.NewBloomTrieIndexer(db, false) + bbtIndexer := light.NewBloomTrieIndexer(db, false) + + bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks) + bloomIndexer.AddChildIndexer(bbtIndexer) bloomIndexer.Start(blockchain) gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator) diff --git a/les/odr_requests.go b/les/odr_requests.go index a11d7b396538..34d759dd2a05 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -493,10 +493,10 @@ func (r *BloomRequest) Request(reqID uint64, peer *peer) error { reqs := make([]HelperTrieReq, len(r.SectionIdxList)) var encNumber [10]byte - binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx)) + binary.BigEndian.PutUint16(encNumber[:2], uint16(r.BitIdx)) for i, sectionIdx := range r.SectionIdxList { - binary.BigEndian.PutUint64(encNumber[2:10], sectionIdx) + binary.BigEndian.PutUint64(encNumber[2:], sectionIdx) reqs[i] = HelperTrieReq{ Type: htBloomBits, TrieIdx: r.BloomTrieNum, @@ -525,10 +525,10 @@ func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error { // Verify the proofs var encNumber [10]byte - binary.BigEndian.PutUint16(encNumber[0:2], uint16(r.BitIdx)) + binary.BigEndian.PutUint16(encNumber[:2], uint16(r.BitIdx)) for i, idx := range r.SectionIdxList { - binary.BigEndian.PutUint64(encNumber[2:10], idx) + binary.BigEndian.PutUint64(encNumber[2:], idx) value, err, _ := trie.VerifyProof(r.BloomTrieRoot, encNumber[:], reads) if err != nil { return err diff --git a/light/nodeset.go b/light/nodeset.go index ffdb71bb7927..245b5eb76671 100644 --- a/light/nodeset.go +++ b/light/nodeset.go @@ -29,7 +29,9 @@ import ( // NodeSet stores a set of trie nodes. It implements trie.Database and can also // act as a cache for another trie.Database. type NodeSet struct { - db map[string][]byte + nodes map[string][]byte + order []string + dataSize int lock sync.RWMutex } @@ -37,7 +39,7 @@ type NodeSet struct { // NewNodeSet creates an empty node set func NewNodeSet() *NodeSet { return &NodeSet{ - db: make(map[string][]byte), + nodes: make(map[string][]byte), } } @@ -46,10 +48,15 @@ func (db *NodeSet) Put(key []byte, value []byte) error { db.lock.Lock() defer db.lock.Unlock() - if _, ok := db.db[string(key)]; !ok { - db.db[string(key)] = common.CopyBytes(value) - db.dataSize += len(value) + if _, ok := db.nodes[string(key)]; ok { + return nil } + keystr := string(key) + + db.nodes[keystr] = common.CopyBytes(value) + db.order = append(db.order, keystr) + db.dataSize += len(value) + return nil } @@ -58,7 +65,7 @@ func (db *NodeSet) Get(key []byte) ([]byte, error) { db.lock.RLock() defer db.lock.RUnlock() - if entry, ok := db.db[string(key)]; ok { + if entry, ok := db.nodes[string(key)]; ok { return entry, nil } return nil, errors.New("not found") @@ -75,7 +82,7 @@ func (db *NodeSet) KeyCount() int { db.lock.RLock() defer db.lock.RUnlock() - return len(db.db) + return len(db.nodes) } // DataSize returns the aggregated data size of nodes in the set @@ -92,8 +99,8 @@ func (db *NodeSet) NodeList() NodeList { defer db.lock.RUnlock() var values NodeList - for _, value := range db.db { - values = append(values, value) + for _, key := range db.order { + values = append(values, db.nodes[key]) } return values } @@ -103,7 +110,7 @@ func (db *NodeSet) Store(target ethdb.Putter) { db.lock.RLock() defer db.lock.RUnlock() - for key, value := range db.db { + for key, value := range db.nodes { target.Put([]byte(key), value) } } From 4b567839b63baf1dad776a0f8687f56ee9a0b9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 10 Feb 2018 14:26:35 +0200 Subject: [PATCH 4/5] les: add an event emission to the les bloombits test --- les/handler.go | 6 ++---- les/handler_test.go | 46 +++++++++++++++++++++++++-------------------- les/helper_test.go | 15 +++++++++++---- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/les/handler.go b/les/handler.go index 734befeeb25b..864abe605adf 100644 --- a/les/handler.go +++ b/les/handler.go @@ -1124,10 +1124,8 @@ func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, stri // getHelperTrieAuxData returns requested auxiliary data for the given HelperTrie request func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte { - if req.Type == htCanonical && req.AuxReq == auxHeader { - if len(req.Key) != 8 { - return nil - } + switch { + case req.Type == htCanonical && req.AuxReq == auxHeader && len(req.Key) == 8: blockNum := binary.BigEndian.Uint64(req.Key) hash := core.GetCanonicalHash(pm.chainDb, blockNum) return core.GetHeaderRLP(pm.chainDb, hash, blockNum) diff --git a/les/handler_test.go b/les/handler_test.go index fa59fe66dee4..cb8a8390e413 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -472,7 +472,7 @@ func TestGetBloombitsProofs(t *testing.T) { // Assemble the test environment db, _ := ethdb.NewMemDatabase() pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) - //bc := pm.blockchain.(*core.BlockChain) + bc := pm.blockchain.(*core.BlockChain) peer, _ := newTestPeer(t, "peer", 2, pm, true) defer peer.close() @@ -480,25 +480,31 @@ func TestGetBloombitsProofs(t *testing.T) { time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling time.Sleep(250 * time.Millisecond) // CI tester slack - // Assemble therequest and proofs for the bloombits - bit := rand.Intn(2048) - key := make([]byte, 10) - - binary.BigEndian.PutUint16(key[:2], uint16(bit)) - binary.BigEndian.PutUint64(key[2:], 0) - - requests := []HelperTrieReq{{ - Type: htBloomBits, - TrieIdx: 0, - Key: key, - }} - var proofs HelperTrieResps - - // Send the proof request and verify the response - cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) - sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests) - if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { - t.Errorf("proofs mismatch: %v", err) + // Request and verify each bit of the bloom bits proofs + for bit := 0; bit < 2048; bit++ { + // Assemble therequest and proofs for the bloombits + key := make([]byte, 10) + + binary.BigEndian.PutUint16(key[:2], uint16(bit)) + binary.BigEndian.PutUint64(key[2:], uint64(light.BloomTrieFrequency)) + + requests := []HelperTrieReq{{ + Type: htBloomBits, + TrieIdx: 0, + Key: key, + }} + var proofs HelperTrieResps + + root := light.GetBloomTrieRoot(db, 0, bc.GetHeaderByNumber(light.BloomTrieFrequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.BloomTrieTablePrefix))) + trie.Prove(key, 0, &proofs.Proofs) + + // Send the proof request and verify the response + cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) + sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests) + if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { + t.Errorf("bit %d: proofs mismatch: %v", bit, err) + } } } diff --git a/les/helper_test.go b/les/helper_test.go index e1e311b44ba1..6d997a1a365b 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -56,6 +56,9 @@ var ( testContractCodeDeployed = testContractCode[16:] testContractDeployed = uint64(2) + testEventEmitterCode = common.Hex2Bytes("60606040523415600e57600080fd5b7f57050ab73f6b9ebdd9f76b8d4997793f48cf956e965ee070551b9ca0bb71584e60405160405180910390a160358060476000396000f3006060604052600080fd00a165627a7a723058203f727efcad8b5811f8cb1fc2620ce5e8c63570d697aef968172de296ea3994140029") + testEventEmitterAddr common.Address + testBufLimit = uint64(100) ) @@ -86,15 +89,19 @@ func testChainGen(i int, block *core.BlockGen) { // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. // acc1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + // acc1Addr creates a test event. nonce := block.TxNonce(acc1Addr) + + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) - nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key) - testContractAddr = crypto.CreateAddress(acc1Addr, nonce) + tx3, _ := types.SignTx(types.NewContractCreation(nonce+1, big.NewInt(0), 200000, big.NewInt(0), testContractCode), signer, acc1Key) + testContractAddr = crypto.CreateAddress(acc1Addr, nonce+1) + tx4, _ := types.SignTx(types.NewContractCreation(nonce+2, big.NewInt(0), 200000, big.NewInt(0), testEventEmitterCode), signer, acc1Key) + testEventEmitterAddr = crypto.CreateAddress(acc1Addr, nonce+2) block.AddTx(tx1) block.AddTx(tx2) block.AddTx(tx3) + block.AddTx(tx4) case 2: // Block 3 is empty but was mined by account #2. block.SetCoinbase(acc2Addr) From 6e78ee396c8b2ff589a384caa003017dab39ee6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 10 Feb 2018 14:30:30 +0200 Subject: [PATCH 5/5] les: drop dead tester code --- les/handler_test.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/les/handler_test.go b/les/handler_test.go index cb8a8390e413..9468032f67f0 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -17,7 +17,6 @@ package les import ( - "bytes" "encoding/binary" "math/big" "math/rand" @@ -46,24 +45,6 @@ func expectResponse(r p2p.MsgReader, msgcode, reqID, bv uint64, data interface{} return p2p.ExpectMsg(r, msgcode, resp{reqID, bv, data}) } -func testCheckProof(t *testing.T, exp *light.NodeSet, got light.NodeList) { - if exp.KeyCount() > len(got) { - t.Errorf("proof has fewer nodes than expected") - return - } - if exp.KeyCount() < len(got) { - t.Errorf("proof has more nodes than expected") - return - } - for _, node := range got { - n, _ := exp.Get(crypto.Keccak256(node)) - if !bytes.Equal(n, node) { - t.Errorf("proof contents mismatch") - return - } - } -} - // Tests that block headers can be retrieved from a remote chain based on user queries. func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) }