From fc00bfd08d2e68eefb4c27c491691bbe7aed71a7 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Tue, 1 Sep 2020 17:50:20 -0700 Subject: [PATCH 01/19] add failing snapshot tests --- test/contracts/forking/Snapshot.sol | 13 +++ test/forking/snapshot.js | 119 ++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 test/contracts/forking/Snapshot.sol create mode 100644 test/forking/snapshot.js diff --git a/test/contracts/forking/Snapshot.sol b/test/contracts/forking/Snapshot.sol new file mode 100644 index 0000000000..52c4d460be --- /dev/null +++ b/test/contracts/forking/Snapshot.sol @@ -0,0 +1,13 @@ +pragma solidity ^0.6.0; + +contract Snapshot { + uint public value; + + constructor() public { + value = 0; + } + + function test() public { + value = value + 1; + } +} diff --git a/test/forking/snapshot.js b/test/forking/snapshot.js new file mode 100644 index 0000000000..6dab6e16b6 --- /dev/null +++ b/test/forking/snapshot.js @@ -0,0 +1,119 @@ +const assert = require("assert"); +const bootstrap = require("../helpers/contract/bootstrap"); +const intializeTestProvider = require("../helpers/web3/initializeTestProvider"); + +/** + * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" + * is the main chain the tests interact with; and the "forked chain" is the + * chain that _was forked_. This is in contrast to general naming, where the + * main chain represents the main chain to be forked (like the Ethereum live + * network) and the fork chaing being "the fork". + */ + +async function takeSnapshot(web3) { + return new Promise((resolve, reject) => { + web3.currentProvider.send( + { + jsonrpc: "2.0", + method: "evm_snapshot", + id: new Date().getTime() + }, + (err, result) => { + if (err) { + return reject(err); + } + return resolve(result.id); + } + ); + }); +} + +async function revertToSnapShot(web3, stateId) { + await new Promise((resolve, reject) => { + web3.currentProvider.send( + { + jsonrpc: "2.0", + method: "evm_revert", + params: [stateId], + id: new Date().getTime() + }, + (err, result) => { + if (err) { + return reject(err); + } + return resolve(result); + } + ); + }); +} + +describe("Forking Snapshots", () => { + let forkedContext; + let mainContext; + const logger = { + log: function(msg) {} + }; + + before("Set up forked provider with web3 instance and deploy a contract", async function() { + this.timeout(5000); + + const contractRef = { + contractFiles: ["Snapshot"], + contractSubdirectory: "forking" + }; + + const ganacheProviderOptions = { + logger, + seed: "main provider" + }; + + forkedContext = await bootstrap(contractRef, ganacheProviderOptions); + }); + + before("Set up main provider and web3 instance", async function() { + const { provider: forkedProvider } = forkedContext; + mainContext = await intializeTestProvider({ + fork: forkedProvider, + logger, + seed: "forked provider" + }); + }); + + it("successfully manages storage slot deletion", async() => { + const { instance: forkedInstance, abi } = forkedContext; + const { web3: mainWeb3 } = mainContext; + + const accounts = await mainWeb3.eth.getAccounts(); + const instance = new mainWeb3.eth.Contract(abi, forkedInstance._address); + const txParams = { + from: accounts[0] + }; + + let value; + + value = await instance.methods.value().call(); + assert.equal(value, 0); + + await instance.methods.test().send(txParams); + + value = await instance.methods.value().call(); + assert.equal(value, 1); + + const snapshotId = await takeSnapshot(mainWeb3); + + await instance.methods.test().send(txParams); + + value = await instance.methods.value().call(); + assert.equal(value, 2); + + await revertToSnapShot(mainWeb3, snapshotId); + + value = await instance.methods.value().call(); + assert.equal(value, 0); + + await instance.methods.test().send(txParams); + + value = await instance.methods.value().call(); + assert.equal(value, 1); + }); +}); From 4f67ac19a34b8c87957cf57db2ab1675d5609d77 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Thu, 3 Sep 2020 18:16:15 -0700 Subject: [PATCH 02/19] move ForkedStorageBaseTrie to not be block number aware --- lib/blockchain_double.js | 2 +- lib/forking/forked_blockchain.js | 97 ++++++++++-------- lib/forking/forked_storage_trie.js | 153 ++++++++++------------------- 3 files changed, 110 insertions(+), 142 deletions(-) diff --git a/lib/blockchain_double.js b/lib/blockchain_double.js index 84cd841aea..301e7a046c 100644 --- a/lib/blockchain_double.js +++ b/lib/blockchain_double.js @@ -539,7 +539,7 @@ BlockchainDouble.prototype.readyCall = function(tx, emulateParent, blockNumber, callback(err); return; } - const stateTrie = this.createStateTrie(this.data.trie_db, stateRoot); + const stateTrie = this.createStateTrie(this.data.trie_db, stateRoot, { persist: false }); const vm = this.createVMFromStateTrie(stateTrie); callback(null, vm, runArgs); }); diff --git a/lib/forking/forked_blockchain.js b/lib/forking/forked_blockchain.js index 0c9c79142f..6472eb85ad 100644 --- a/lib/forking/forked_blockchain.js +++ b/lib/forking/forked_blockchain.js @@ -424,36 +424,60 @@ ForkedBlockchain.prototype.getBlock = function(number, callback) { }; ForkedBlockchain.prototype.getStorage = function(address, key, number, callback) { - this.getLookupStorageTrie(this.stateTrie)(address, (err, trie) => { + var self = this; + + this.getEffectiveBlockNumber(number, (err, blockNumber) => { if (err) { return callback(err); } - this.getEffectiveBlockNumber(number, (err, blockNumber) => { - if (err) { - return callback(err); - } - if (blockNumber > this.forkBlockNumber) { - // only hit the ForkedStorageTrieBase if we're not looking - // for something that's on the forked chain - trie.get(utils.setLengthLeft(utils.toBuffer(key), 32), blockNumber, callback); - } else { - // we're looking for something prior to forking, so let's - // hit eth_getStorageAt - this.web3.eth.getStorageAt(to.rpcDataHexString(address), to.rpcDataHexString(key), blockNumber, function( - err, - value - ) { + if (blockNumber > self.forkBlockNumber) { + // we should have this block + + self.getBlock(blockNumber, function(err, block) { + if (err) { + return callback(err); + } + + const trie = self.stateTrie; + + // Manipulate the state root in place to maintain checkpoints + const currentStateRoot = trie.root; + self.stateTrie.root = block.header.stateRoot; + + self.getLookupStorageTrie(self.stateTrie)(address, (err, trie) => { if (err) { return callback(err); } - value = utils.rlp.encode(value); + trie.get(utils.setLengthLeft(utils.toBuffer(key), 32), function(err, value) { + // Finally, put the stateRoot back for good + trie.root = currentStateRoot; + + if (err != null) { + return callback(err); + } - callback(null, value); + callback(null, value); + }); }); - } - }); + }); + } else { + // we're looking for something prior to forking, so let's + // hit eth_getStorageAt + self.web3.eth.getStorageAt(to.rpcDataHexString(address), to.rpcDataHexString(key), blockNumber, function( + err, + value + ) { + if (err) { + return callback(err); + } + + value = utils.rlp.encode(value); + + callback(null, value); + }); + } }); }; @@ -475,34 +499,23 @@ ForkedBlockchain.prototype.getCode = function(address, number, callback) { } number = effective; - self.stateTrie.keyExists(address, function(err, exists) { + self.stateTrie.getTouchedAt(address, (err, touchedAt) => { if (err) { return callback(err); } - // If we've stored the value and we're looking at one of our stored blocks, - // get it from our stored data. - if (exists && number > to.number(self.forkBlockNumber)) { + + if (typeof touchedAt !== "undefined" && touchedAt <= number) { BlockchainDouble.prototype.getCode.call(self, address, number, callback); } else { - self.stateTrie.keyIsDeleted(address, (err, deleted) => { - if (err) { - return callback(err); - } - if (deleted) { - return callback(null, Buffer.allocUnsafe(0)); - } - // Else, we need to fetch it from web3. If our number is greater than - // the fork, let's just use "latest". - if (number > to.number(self.forkBlockNumber)) { - number = "latest"; - } + if (number > to.number(self.forkBlockNumber)) { + number = "latest"; + } - self.fetchCodeFromFallback(address, number, function(err, code) { - if (code) { - code = utils.toBuffer(code); - } - callback(err, code); - }); + self.fetchCodeFromFallback(address, number, function(err, code) { + if (code) { + code = utils.toBuffer(code); + } + callback(err, code); }); } }); diff --git a/lib/forking/forked_storage_trie.js b/lib/forking/forked_storage_trie.js index 6e9c34f682..33b8194729 100644 --- a/lib/forking/forked_storage_trie.js +++ b/lib/forking/forked_storage_trie.js @@ -2,7 +2,6 @@ const Sublevel = require("level-sublevel"); const MerklePatriciaTree = require("merkle-patricia-tree"); const BaseTrie = require("merkle-patricia-tree/baseTrie"); const checkpointInterface = require("merkle-patricia-tree/checkpoint-interface"); -const Account = require("ethereumjs-account").default; var utils = require("ethereumjs-util"); var inherits = require("util").inherits; var Web3 = require("web3"); @@ -12,7 +11,7 @@ inherits(ForkedStorageBaseTrie, BaseTrie); function ForkedStorageBaseTrie(db, root, options) { BaseTrie.call(this, db, root); - this._deleted = Sublevel(this.db).sublevel("deleted"); + this._touched = Sublevel(this.db).sublevel("touched"); this.options = options; this.address = options.address; @@ -20,101 +19,40 @@ function ForkedStorageBaseTrie(db, root, options) { this.blockchain = options.blockchain; this.fork = options.fork; this.web3 = new Web3(this.fork); + this.persist = typeof options.persist === "undefined" ? true : options.persist; } // Note: This overrides a standard method whereas the other methods do not. -ForkedStorageBaseTrie.prototype.get = function(key, blockNumber, callback) { +ForkedStorageBaseTrie.prototype.get = function(key, callback) { var self = this; - let blockNumberProvided = true; - - // Allow an optional blockNumber - if (typeof blockNumber === "function") { - callback = blockNumber; - blockNumber = this.forkBlockNumber; - blockNumberProvided = false; - } key = utils.toBuffer(key); - // If the account/key doesn't exist in our state trie, get it off the wire. - this.keyExists(key, function(err, exists) { + this.getTouchedAt(key, function(err, touchedAt) { if (err) { return callback(err); } - if (exists) { - // I'm checking to see if a blockNumber is provided because the below - // logic breaks for things like nonce lookup, in which we should just - // use the root trie as is. I'm guessing there's a cleaner architecture - // that doesn't require such checks - if (blockNumberProvided) { - // this logic is heavily influenced by BlockchainDouble.prototype.getStorage - // but some adjustments were necessary due to the ForkedStorageTrieBase context - self.blockchain.getBlock(blockNumber, function(err, block) { + if (typeof touchedAt !== "undefined") { + MerklePatriciaTree.prototype.get.call(self, key, function(err, r) { + callback(err, r); + }); + } else { + // If this is the main trie, get the whole account. + if (self.address == null) { + self.blockchain.fetchAccountFromFallback(key, self.forkBlockNumber, function(err, account) { if (err) { return callback(err); } - // Manipulate the state root in place to maintain checkpoints - const currentStateRoot = self.root; - self.root = block.header.stateRoot; - - MerklePatriciaTree.prototype.get.call(self, utils.toBuffer(self.address), function(err, data) { - if (err != null) { - // Put the stateRoot back if there's an error - self.root = currentStateRoot; - return callback(err); - } - - const account = new Account(data); - - self.root = account.stateRoot; - MerklePatriciaTree.prototype.get.call(self, key, function(err, value) { - // Finally, put the stateRoot back for good - self.root = currentStateRoot; - - if (err != null) { - return callback(err, value); - } - - callback(null, value); - }); - }); + callback(null, account.serialize()); }); } else { - MerklePatriciaTree.prototype.get.call(self, key, function(err, r) { - callback(err, r); - }); - } - } else { - self.keyIsDeleted(key, (err, deleted) => { - if (err) { - return callback(err); - } - - if (deleted) { - // it was deleted. return nothing. - callback(null, Buffer.allocUnsafe(0)); - return; - } - - // If this is the main trie, get the whole account. - if (self.address == null) { - self.blockchain.fetchAccountFromFallback(key, blockNumber, function(err, account) { - if (err) { - return callback(err); - } - - callback(null, account.serialize()); - }); - } else { - if (to.number(blockNumber) > to.number(self.forkBlockNumber)) { - blockNumber = self.forkBlockNumber; - } - self.web3.eth.getStorageAt(to.rpcDataHexString(self.address), to.rpcDataHexString(key), blockNumber, function( - err, - value - ) { + self.web3.eth.getStorageAt( + to.rpcDataHexString(self.address), + to.rpcDataHexString(key), + self.forkBlockNumber, + function(err, value) { if (err) { return callback(err); } @@ -122,9 +60,9 @@ ForkedStorageBaseTrie.prototype.get = function(key, blockNumber, callback) { value = utils.rlp.encode(value); callback(null, value); - }); - } - }); + } + ); + } } }); }; @@ -137,41 +75,58 @@ ForkedStorageBaseTrie.prototype.keyExists = function(key, callback) { }); }; -const originalPut = ForkedStorageBaseTrie.prototype.put; -ForkedStorageBaseTrie.prototype.put = function(key, value, callback) { +ForkedStorageBaseTrie.prototype.touch = function(key, callback) { + const self = this; let rpcKey = to.rpcDataHexString(key); if (this.address) { rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; } - this._deleted.get(rpcKey, (_, result) => { - if (result === 1) { - this._deleted.put(rpcKey, 0, () => { - originalPut.call(this, key, value, callback); + + this._touched.get(rpcKey, (_, result) => { + if (typeof result === "undefined") { + // key doesnt exist + this.blockchain.data.blocks.last((err, lastBlock) => { + if (err) { + console.log(new Error("shouldn't happen")); + callback(); + return; + } + + const number = lastBlock === null ? self.forkBlockNumber : to.number(lastBlock.header.number); + if (this.persist) { + this._touched.put(rpcKey, number + 1); + } + callback(); }); } else { - originalPut.call(this, key, value, callback); + callback(); } }); }; -ForkedStorageBaseTrie.prototype.keyIsDeleted = function(key, callback) { +const originalPut = ForkedStorageBaseTrie.prototype.put; +ForkedStorageBaseTrie.prototype.put = function(key, value, callback) { + const self = this; + this.touch(key, function() { + originalPut.call(self, key, value, callback); + }); +}; + +ForkedStorageBaseTrie.prototype.getTouchedAt = function(key, callback) { let rpcKey = to.rpcDataHexString(key); if (this.address) { rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; } - this._deleted.get(rpcKey, (_, result) => { - callback(null, result === 1); + this._touched.get(rpcKey, (_, result) => { + callback(null, result); }); }; const originalDelete = ForkedStorageBaseTrie.prototype.del; ForkedStorageBaseTrie.prototype.del = function(key, callback) { - let rpcKey = to.rpcDataHexString(key); - if (this.address) { - rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; - } - this._deleted.put(rpcKey, 1, () => { - originalDelete.call(this, key, callback); + const self = this; + this.touch(key, function() { + originalDelete.call(self, key, callback); }); }; From 7adc292d502f21a32e034663e64448b5f8c179fb Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Thu, 3 Sep 2020 18:42:28 -0700 Subject: [PATCH 03/19] fix snapshot test --- test/forking/snapshot.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/forking/snapshot.js b/test/forking/snapshot.js index 6dab6e16b6..ab8b83fe40 100644 --- a/test/forking/snapshot.js +++ b/test/forking/snapshot.js @@ -22,7 +22,7 @@ async function takeSnapshot(web3) { if (err) { return reject(err); } - return resolve(result.id); + return resolve(result.result); } ); }); @@ -109,11 +109,11 @@ describe("Forking Snapshots", () => { await revertToSnapShot(mainWeb3, snapshotId); value = await instance.methods.value().call(); - assert.equal(value, 0); + assert.equal(value, 1); await instance.methods.test().send(txParams); value = await instance.methods.value().call(); - assert.equal(value, 1); + assert.equal(value, 2); }); }); From 41d1f9519c6a6e4875a270824c150d54e72e3422 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 4 Sep 2020 19:21:31 -0700 Subject: [PATCH 04/19] add tx count checking to test --- test/forking/snapshot.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/forking/snapshot.js b/test/forking/snapshot.js index ab8b83fe40..b249e80db3 100644 --- a/test/forking/snapshot.js +++ b/test/forking/snapshot.js @@ -79,7 +79,7 @@ describe("Forking Snapshots", () => { }); }); - it("successfully manages storage slot deletion", async() => { + it("successfully handles snapshot/revert scenarios", async() => { const { instance: forkedInstance, abi } = forkedContext; const { web3: mainWeb3 } = mainContext; @@ -99,6 +99,7 @@ describe("Forking Snapshots", () => { value = await instance.methods.value().call(); assert.equal(value, 1); + const beforeSnapshotNonce = await mainWeb3.eth.getTransactionCount(accounts[0]); const snapshotId = await takeSnapshot(mainWeb3); await instance.methods.test().send(txParams); @@ -106,8 +107,14 @@ describe("Forking Snapshots", () => { value = await instance.methods.value().call(); assert.equal(value, 2); + const beforeRevertNonce = await mainWeb3.eth.getTransactionCount(accounts[0]); + assert.equal(beforeRevertNonce, beforeSnapshotNonce + 1); + await revertToSnapShot(mainWeb3, snapshotId); + const afterRevertNonce = await mainWeb3.eth.getTransactionCount(accounts[0]); + assert.equal(afterRevertNonce, beforeSnapshotNonce); + value = await instance.methods.value().call(); assert.equal(value, 1); From b450cf0435a04e0f5c4752ef7dd8c55609f360b3 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 4 Sep 2020 19:22:13 -0700 Subject: [PATCH 05/19] add forked blockchain overload for getQueuedNonce --- lib/forking/forked_blockchain.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/lib/forking/forked_blockchain.js b/lib/forking/forked_blockchain.js index 6472eb85ad..a1133a95ef 100644 --- a/lib/forking/forked_blockchain.js +++ b/lib/forking/forked_blockchain.js @@ -858,6 +858,37 @@ ForkedBlockchain.prototype.getBlockLogs = function(number, callback) { }); }; +ForkedBlockchain.prototype.getQueuedNonce = function(address, callback) { + var nonce = null; + var addressBuffer = to.buffer(address); + this.pending_transactions.forEach(function(tx) { + if (!tx.from.equals(addressBuffer)) { + return; + } + + var pendingNonce = new BN(tx.nonce); + // If this is the first queued nonce for this address we found, + // or it's higher than the previous highest, note it. + if (nonce === null || pendingNonce.gt(nonce)) { + nonce = pendingNonce; + } + }); + + // If we found a queued transaction nonce, return one higher + // than the highest we found + if (nonce != null) { + return callback(null, nonce.iaddn(1).toArrayLike(Buffer)); + } + this.getLookupAccount(this.stateTrie)(addressBuffer, function(err, account) { + if (err) { + return callback(err); + } + + // nonces are initialized as an empty buffer, which isn't what we want. + callback(null, account.nonce.length === 0 ? Buffer.from([0]) : account.nonce); + }); +}; + ForkedBlockchain.prototype.close = function(callback) { if (this.fork.disconnect) { this.fork.disconnect(); From 9af6aba59c865f095e1159330efffddc2731d1dc Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 4 Sep 2020 19:23:04 -0700 Subject: [PATCH 06/19] clear appropriate "touched" db entries when reverting --- lib/forking/forked_blockchain.js | 70 ++++++++++++++++++++++++++++++ lib/forking/forked_storage_trie.js | 1 + 2 files changed, 71 insertions(+) diff --git a/lib/forking/forked_blockchain.js b/lib/forking/forked_blockchain.js index a1133a95ef..31941f0105 100644 --- a/lib/forking/forked_blockchain.js +++ b/lib/forking/forked_blockchain.js @@ -10,6 +10,7 @@ var to = require("../utils/to.js"); var Transaction = require("../utils/transaction"); var async = require("async"); var LRUCache = require("lru-cache"); +const Sublevel = require("level-sublevel"); const BN = utils.BN; var inherits = require("util").inherits; @@ -162,6 +163,7 @@ function ForkedBlockchain(options) { }; this.web3 = new Web3(this.fork); + this._touchedKeys = []; } ForkedBlockchain.prototype.initialize = async function(accounts, callback) { @@ -888,6 +890,74 @@ ForkedBlockchain.prototype.getQueuedNonce = function(address, callback) { callback(null, account.nonce.length === 0 ? Buffer.from([0]) : account.nonce); }); }; +ForkedBlockchain.prototype.processBlock = async function(vm, block, commit, callback) { + const self = this; + + self._touchedKeys = []; + BlockchainDouble.prototype.processBlock.call(self, vm, block, commit, callback); +}; + +ForkedBlockchain.prototype.putBlock = function(block, logs, receipts, callback) { + const self = this; + const touched = Sublevel(self.data.trie_db).sublevel("touched"); + const blockKey = `block-${to.number(block.header.number)}`; + + BlockchainDouble.prototype.putBlock.call(self, block, logs, receipts, function(err, result) { + if (err) { + return callback(err); + } + + touched.put(blockKey, JSON.stringify(self._touchedKeys), (err) => { + if (err) { + return callback(err); + } + + callback(null, result); + }); + }); +}; + +ForkedBlockchain.prototype.popBlock = function(callback) { + const self = this; + const touched = Sublevel(this.data.trie_db).sublevel("touched"); + + this.data.blocks.last(function(err, block) { + if (err) { + return callback(err); + } + if (block == null) { + return callback(null, null); + } + + const blockKey = `block-${to.number(block.header.number)}`; + touched.get(blockKey, function(err, value) { + if (err) { + return callback(err); + } + + const touchedKeys = value ? JSON.parse(value) : []; + async.eachSeries( + touchedKeys, + function(touchedKey, finished) { + touched.del(touchedKey, finished); + }, + function(err) { + if (err) { + return callback(err); + } + + touched.del(blockKey, function(err) { + if (err) { + return callback(err); + } + + BlockchainDouble.prototype.popBlock.call(self, callback); + }); + } + ); + }); + }); +}; ForkedBlockchain.prototype.close = function(callback) { if (this.fork.disconnect) { diff --git a/lib/forking/forked_storage_trie.js b/lib/forking/forked_storage_trie.js index 33b8194729..2ab7fbfd61 100644 --- a/lib/forking/forked_storage_trie.js +++ b/lib/forking/forked_storage_trie.js @@ -95,6 +95,7 @@ ForkedStorageBaseTrie.prototype.touch = function(key, callback) { const number = lastBlock === null ? self.forkBlockNumber : to.number(lastBlock.header.number); if (this.persist) { this._touched.put(rpcKey, number + 1); + this.blockchain._touchedKeys.push(rpcKey); } callback(); }); From db55a414a070a7fdf562bcafe6a8555b4fc23ed4 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Tue, 8 Sep 2020 18:05:15 -0700 Subject: [PATCH 07/19] sanitize user input to be lower case --- lib/subproviders/geth_api_double.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/subproviders/geth_api_double.js b/lib/subproviders/geth_api_double.js index 55bcad80ea..dd1734e857 100644 --- a/lib/subproviders/geth_api_double.js +++ b/lib/subproviders/geth_api_double.js @@ -169,11 +169,11 @@ GethApiDouble.prototype.eth_gasPrice = function(callback) { }; GethApiDouble.prototype.eth_getBalance = function(address, blockNumber, callback) { - this.state.getBalance(address, blockNumber, callback); + this.state.getBalance(address.toLowerCase(), blockNumber, callback); }; GethApiDouble.prototype.eth_getCode = function(address, blockNumber, callback) { - this.state.getCode(address, blockNumber, callback); + this.state.getCode(address.toLowerCase(), blockNumber, callback); }; GethApiDouble.prototype.eth_getBlockByNumber = function(blockNumber, includeFullTransactions, callback) { @@ -287,7 +287,7 @@ GethApiDouble.prototype.eth_getTransactionByBlockNumberAndIndex = function(hashO }; GethApiDouble.prototype.eth_getTransactionCount = function(address, blockNumber, callback) { - this.state.getTransactionCount(address, blockNumber, (err, count) => { + this.state.getTransactionCount(address.toLowerCase(), blockNumber, (err, count) => { if (err instanceof BlockOutOfRangeError) { return callback(null, null); } @@ -300,7 +300,7 @@ GethApiDouble.prototype.eth_sign = function(address, dataToSign, callback) { var error; try { - result = this.state.sign(address, dataToSign); + result = this.state.sign(address.toLowerCase(), dataToSign); } catch (e) { error = e; } @@ -313,7 +313,7 @@ GethApiDouble.prototype.eth_signTypedData = function(address, typedDataToSign, c var error; try { - result = this.state.signTypedData(address, typedDataToSign); + result = this.state.signTypedData(address.toLowerCase(), typedDataToSign); } catch (e) { error = e; } @@ -364,7 +364,7 @@ GethApiDouble.prototype.eth_estimateGas = function(txData, blockNumber, callback }; GethApiDouble.prototype.eth_getStorageAt = function(address, position, blockNumber, callback) { - this.state.queueStorage(address, position, blockNumber, callback); + this.state.queueStorage(address.toLowerCase(), position.toLowerCase(), blockNumber, callback); }; GethApiDouble.prototype.eth_newBlockFilter = function(callback) { From f0d09c1f9f521a6a23ae0fb9ccf2fbed382d468f Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 11 Sep 2020 16:20:12 -0700 Subject: [PATCH 08/19] apply suggestions from PR review --- lib/forking/forked_storage_trie.js | 39 +++++++++++++++++++----------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/lib/forking/forked_storage_trie.js b/lib/forking/forked_storage_trie.js index 2ab7fbfd61..367610fa65 100644 --- a/lib/forking/forked_storage_trie.js +++ b/lib/forking/forked_storage_trie.js @@ -76,27 +76,31 @@ ForkedStorageBaseTrie.prototype.keyExists = function(key, callback) { }; ForkedStorageBaseTrie.prototype.touch = function(key, callback) { + if (this.persist) { + return callback(); + } + const self = this; let rpcKey = to.rpcDataHexString(key); if (this.address) { rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; } - this._touched.get(rpcKey, (_, result) => { + this._touched.get(rpcKey, (err, result) => { + if (err) { + return callback(err); + } + if (typeof result === "undefined") { - // key doesnt exist + // key doesn't exist this.blockchain.data.blocks.last((err, lastBlock) => { if (err) { - console.log(new Error("shouldn't happen")); - callback(); - return; + return callback(err); } const number = lastBlock === null ? self.forkBlockNumber : to.number(lastBlock.header.number); - if (this.persist) { - this._touched.put(rpcKey, number + 1); - this.blockchain._touchedKeys.push(rpcKey); - } + this._touched.put(rpcKey, number + 1); + this.blockchain._touchedKeys.push(rpcKey); callback(); }); } else { @@ -108,7 +112,11 @@ ForkedStorageBaseTrie.prototype.touch = function(key, callback) { const originalPut = ForkedStorageBaseTrie.prototype.put; ForkedStorageBaseTrie.prototype.put = function(key, value, callback) { const self = this; - this.touch(key, function() { + this.touch(key, function(err) { + if (err) { + return callback(err); + } + originalPut.call(self, key, value, callback); }); }; @@ -118,15 +126,18 @@ ForkedStorageBaseTrie.prototype.getTouchedAt = function(key, callback) { if (this.address) { rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; } - this._touched.get(rpcKey, (_, result) => { - callback(null, result); - }); + + this._touched.get(rpcKey, callback); }; const originalDelete = ForkedStorageBaseTrie.prototype.del; ForkedStorageBaseTrie.prototype.del = function(key, callback) { const self = this; - this.touch(key, function() { + this.touch(key, function(err) { + if (err) { + return callback(err); + } + originalDelete.call(self, key, callback); }); }; From ebc0cea116b79ca5153d4407e9514b692d0e25df Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 11 Sep 2020 16:28:32 -0700 Subject: [PATCH 09/19] move forking tests to the local directory (bad merge) --- test/{ => local}/forking/delete.js | 4 ++-- test/{ => local}/forking/snapshot.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename test/{ => local}/forking/delete.js (91%) rename test/{ => local}/forking/snapshot.js (95%) diff --git a/test/forking/delete.js b/test/local/forking/delete.js similarity index 91% rename from test/forking/delete.js rename to test/local/forking/delete.js index 6be8048d51..57e6e88bf2 100644 --- a/test/forking/delete.js +++ b/test/local/forking/delete.js @@ -1,6 +1,6 @@ const assert = require("assert"); -const bootstrap = require("../helpers/contract/bootstrap"); -const intializeTestProvider = require("../helpers/web3/initializeTestProvider"); +const bootstrap = require("../../helpers/contract/bootstrap"); +const intializeTestProvider = require("../../helpers/web3/initializeTestProvider"); /** * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" diff --git a/test/forking/snapshot.js b/test/local/forking/snapshot.js similarity index 95% rename from test/forking/snapshot.js rename to test/local/forking/snapshot.js index b249e80db3..f3faf84a67 100644 --- a/test/forking/snapshot.js +++ b/test/local/forking/snapshot.js @@ -1,6 +1,6 @@ const assert = require("assert"); -const bootstrap = require("../helpers/contract/bootstrap"); -const intializeTestProvider = require("../helpers/web3/initializeTestProvider"); +const bootstrap = require("../../helpers/contract/bootstrap"); +const intializeTestProvider = require("../../helpers/web3/initializeTestProvider"); /** * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" From 3a8253b1fe3af08302133de228ee1182ca9745aa Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 11 Sep 2020 16:41:23 -0700 Subject: [PATCH 10/19] fix bugs introduced in my implementation of fixing pr review --- lib/forking/forked_storage_trie.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/forking/forked_storage_trie.js b/lib/forking/forked_storage_trie.js index 367610fa65..5c4f681811 100644 --- a/lib/forking/forked_storage_trie.js +++ b/lib/forking/forked_storage_trie.js @@ -76,7 +76,7 @@ ForkedStorageBaseTrie.prototype.keyExists = function(key, callback) { }; ForkedStorageBaseTrie.prototype.touch = function(key, callback) { - if (this.persist) { + if (!this.persist) { return callback(); } @@ -87,7 +87,7 @@ ForkedStorageBaseTrie.prototype.touch = function(key, callback) { } this._touched.get(rpcKey, (err, result) => { - if (err) { + if (err && err.type !== "NotFoundError") { return callback(err); } @@ -127,7 +127,13 @@ ForkedStorageBaseTrie.prototype.getTouchedAt = function(key, callback) { rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; } - this._touched.get(rpcKey, callback); + this._touched.get(rpcKey, function(err, result) { + if (err && err.type !== "NotFoundError") { + return callback(err); + } + + callback(null, result); + }); }; const originalDelete = ForkedStorageBaseTrie.prototype.del; From 4d36fddec1e23618f0cb15734e6be0ac9526bade Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 11 Sep 2020 16:42:19 -0700 Subject: [PATCH 11/19] stop using deprecated assert.equal --- test/local/forking/snapshot.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/local/forking/snapshot.js b/test/local/forking/snapshot.js index f3faf84a67..0b8f468f0c 100644 --- a/test/local/forking/snapshot.js +++ b/test/local/forking/snapshot.js @@ -92,12 +92,12 @@ describe("Forking Snapshots", () => { let value; value = await instance.methods.value().call(); - assert.equal(value, 0); + assert.strictEqual(value, "0"); await instance.methods.test().send(txParams); value = await instance.methods.value().call(); - assert.equal(value, 1); + assert.strictEqual(value, "1"); const beforeSnapshotNonce = await mainWeb3.eth.getTransactionCount(accounts[0]); const snapshotId = await takeSnapshot(mainWeb3); @@ -105,22 +105,22 @@ describe("Forking Snapshots", () => { await instance.methods.test().send(txParams); value = await instance.methods.value().call(); - assert.equal(value, 2); + assert.strictEqual(value, "2"); const beforeRevertNonce = await mainWeb3.eth.getTransactionCount(accounts[0]); - assert.equal(beforeRevertNonce, beforeSnapshotNonce + 1); + assert.strictEqual(beforeRevertNonce, beforeSnapshotNonce + 1); await revertToSnapShot(mainWeb3, snapshotId); const afterRevertNonce = await mainWeb3.eth.getTransactionCount(accounts[0]); - assert.equal(afterRevertNonce, beforeSnapshotNonce); + assert.strictEqual(afterRevertNonce, beforeSnapshotNonce); value = await instance.methods.value().call(); - assert.equal(value, 1); + assert.strictEqual(value, "1"); await instance.methods.test().send(txParams); value = await instance.methods.value().call(); - assert.equal(value, 2); + assert.strictEqual(value, "2"); }); }); From 9818f54dcf2a9c62e6ac9218ea9db1c264ff30da Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Fri, 11 Sep 2020 17:17:32 -0700 Subject: [PATCH 12/19] handle eth_call in a forking context properly --- lib/forking/forked_blockchain.js | 31 ++++++++++++++++ test/local/forking/call.js | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 test/local/forking/call.js diff --git a/lib/forking/forked_blockchain.js b/lib/forking/forked_blockchain.js index 31941f0105..68f175a937 100644 --- a/lib/forking/forked_blockchain.js +++ b/lib/forking/forked_blockchain.js @@ -890,6 +890,37 @@ ForkedBlockchain.prototype.getQueuedNonce = function(address, callback) { callback(null, account.nonce.length === 0 ? Buffer.from([0]) : account.nonce); }); }; + +ForkedBlockchain.prototype.processCall = function(tx, blockNumber, callback) { + const self = this; + + this.getEffectiveBlockNumber(blockNumber, function(err, effectiveBlockNumber) { + if (err) { + return callback(err); + } + + if (effectiveBlockNumber > self.forkBlockNumber) { + BlockchainDouble.prototype.processCall.call(self, tx, blockNumber, callback); + } else { + self.web3.eth.call({ + from: to.rpcDataHexString(tx.from), + to: to.nullableRpcDataHexString(tx.to), + data: to.rpcDataHexString(tx.data) + }, effectiveBlockNumber, function(err, result) { + if (err) { + return callback(err); + } + + callback(null, { + execResult: { + returnValue: result + } + }); + }); + } + }); +}; + ForkedBlockchain.prototype.processBlock = async function(vm, block, commit, callback) { const self = this; diff --git a/test/local/forking/call.js b/test/local/forking/call.js new file mode 100644 index 0000000000..7c6a81260e --- /dev/null +++ b/test/local/forking/call.js @@ -0,0 +1,62 @@ +const assert = require("assert"); +const bootstrap = require("../../helpers/contract/bootstrap"); +const intializeTestProvider = require("../../helpers/web3/initializeTestProvider"); + +describe("Forking eth_call", () => { + let forkedContext; + const logger = { + log: function(msg) {} + }; + + before("Set up forked provider with web3 instance and deploy a contract", async function() { + this.timeout(5000); + + const contractRef = { + contractFiles: ["Snapshot"], + contractSubdirectory: "forking" + }; + + const ganacheProviderOptions = { + logger, + seed: "main provider" + }; + + forkedContext = await bootstrap(contractRef, ganacheProviderOptions); + }); + + it("gets values at specified blocks on the original change", async function() { + const { + send, + accounts: [from], + abi, + web3: originalWeb3, + instance: { _address: contractAddress }, + provider: originalProvider + } = forkedContext; + + const originalContract = new originalWeb3.eth.Contract(abi, contractAddress); + + const txParams = { from }; + + await originalContract.methods.test().send(txParams); + const initialValue = await originalContract.methods.value().call(); + await send("evm_mine", null); + const { result: preForkBlockNumber } = await send("eth_blockNumber"); + + // Fork the "original" chain _now_ + const { web3 } = await intializeTestProvider({ + fork: originalProvider, + logger, + seed: "forked provider" + }); + + const instance = new web3.eth.Contract(abi, contractAddress); + + // get the value as it was before we forked + instance.defaultBlock = preForkBlockNumber; + // there is a bug possibly unrelated to this PR that prevents this from + // returning the wrong value (it crashes with a `pop of undefined` instead). + const finalValue = await instance.methods.value().call(); + assert.strictEqual(finalValue, initialValue); + }); +}); From 3bfb36597f0b7bff276d0671974d6e654f5e3394 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Tue, 15 Sep 2020 16:29:40 -0700 Subject: [PATCH 13/19] Revert "sanitize user input to be lower case" This reverts commit 5d97074698302547a3cfb6df684bd82c87c1485e. --- lib/subproviders/geth_api_double.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/subproviders/geth_api_double.js b/lib/subproviders/geth_api_double.js index dd1734e857..55bcad80ea 100644 --- a/lib/subproviders/geth_api_double.js +++ b/lib/subproviders/geth_api_double.js @@ -169,11 +169,11 @@ GethApiDouble.prototype.eth_gasPrice = function(callback) { }; GethApiDouble.prototype.eth_getBalance = function(address, blockNumber, callback) { - this.state.getBalance(address.toLowerCase(), blockNumber, callback); + this.state.getBalance(address, blockNumber, callback); }; GethApiDouble.prototype.eth_getCode = function(address, blockNumber, callback) { - this.state.getCode(address.toLowerCase(), blockNumber, callback); + this.state.getCode(address, blockNumber, callback); }; GethApiDouble.prototype.eth_getBlockByNumber = function(blockNumber, includeFullTransactions, callback) { @@ -287,7 +287,7 @@ GethApiDouble.prototype.eth_getTransactionByBlockNumberAndIndex = function(hashO }; GethApiDouble.prototype.eth_getTransactionCount = function(address, blockNumber, callback) { - this.state.getTransactionCount(address.toLowerCase(), blockNumber, (err, count) => { + this.state.getTransactionCount(address, blockNumber, (err, count) => { if (err instanceof BlockOutOfRangeError) { return callback(null, null); } @@ -300,7 +300,7 @@ GethApiDouble.prototype.eth_sign = function(address, dataToSign, callback) { var error; try { - result = this.state.sign(address.toLowerCase(), dataToSign); + result = this.state.sign(address, dataToSign); } catch (e) { error = e; } @@ -313,7 +313,7 @@ GethApiDouble.prototype.eth_signTypedData = function(address, typedDataToSign, c var error; try { - result = this.state.signTypedData(address.toLowerCase(), typedDataToSign); + result = this.state.signTypedData(address, typedDataToSign); } catch (e) { error = e; } @@ -364,7 +364,7 @@ GethApiDouble.prototype.eth_estimateGas = function(txData, blockNumber, callback }; GethApiDouble.prototype.eth_getStorageAt = function(address, position, blockNumber, callback) { - this.state.queueStorage(address.toLowerCase(), position.toLowerCase(), blockNumber, callback); + this.state.queueStorage(address, position, blockNumber, callback); }; GethApiDouble.prototype.eth_newBlockFilter = function(callback) { From 94c727bbb2133eea065b26c2b03b6422c24ad150 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Tue, 15 Sep 2020 19:04:12 -0700 Subject: [PATCH 14/19] add failing case senstivity test --- test/local/ethers.js | 4 +- test/local/forking/call.js | 4 +- test/local/forking/caseSensitivity.js | 195 ++++++++++++++++++++++++ test/local/forking/delete.js | 4 +- test/local/forking/forkingAsProvider.js | 4 +- test/local/forking/snapshot.js | 4 +- 6 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 test/local/forking/caseSensitivity.js diff --git a/test/local/ethers.js b/test/local/ethers.js index d39df74ac9..a8474339a9 100644 --- a/test/local/ethers.js +++ b/test/local/ethers.js @@ -1,7 +1,7 @@ const assert = require("assert"); const { BN } = require("ethereumjs-util"); const ethers = require("ethers"); -const intializeTestProvider = require("../helpers/web3/initializeTestProvider"); +const initializeTestProvider = require("../helpers/web3/initializeTestProvider"); describe("ethers", async() => { let ethersProvider, wallet, gasPrice, value; @@ -18,7 +18,7 @@ describe("ethers", async() => { ] }; - const { provider } = await intializeTestProvider(ganacheOptions); + const { provider } = await initializeTestProvider(ganacheOptions); ethersProvider = new ethers.providers.Web3Provider(provider); const privateKey = Buffer.from(secretKey, "hex"); diff --git a/test/local/forking/call.js b/test/local/forking/call.js index 7c6a81260e..67d2900ba2 100644 --- a/test/local/forking/call.js +++ b/test/local/forking/call.js @@ -1,6 +1,6 @@ const assert = require("assert"); const bootstrap = require("../../helpers/contract/bootstrap"); -const intializeTestProvider = require("../../helpers/web3/initializeTestProvider"); +const initializeTestProvider = require("../../helpers/web3/initializeTestProvider"); describe("Forking eth_call", () => { let forkedContext; @@ -44,7 +44,7 @@ describe("Forking eth_call", () => { const { result: preForkBlockNumber } = await send("eth_blockNumber"); // Fork the "original" chain _now_ - const { web3 } = await intializeTestProvider({ + const { web3 } = await initializeTestProvider({ fork: originalProvider, logger, seed: "forked provider" diff --git a/test/local/forking/caseSensitivity.js b/test/local/forking/caseSensitivity.js new file mode 100644 index 0000000000..153d93bbd8 --- /dev/null +++ b/test/local/forking/caseSensitivity.js @@ -0,0 +1,195 @@ +const assert = require("assert"); +const bootstrap = require("../../helpers/contract/bootstrap"); +const initializeTestProvider = require("../../helpers/web3/initializeTestProvider"); + +/** + * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" + * is the main chain the tests interact with; and the "forked chain" is the + * chain that _was forked_. This is in contrast to general naming, where the + * main chain represents the main chain to be forked (like the Ethereum live + * network) and the fork chain being "the fork". + */ + +// Defining our own functions to send raw rpc calls because web3 +// does a toLower on the address + +async function getBalance(web3, id, address, blockNumber) { + return new Promise(function(resolve, reject) { + web3.currentProvider.send( + { + jsonrpc: "2.0", + method: "eth_getBalance", + params: [address, blockNumber], + id + }, + function(err, result) { + if (err) { + reject(err); + } else { + resolve(web3.utils.hexToNumberString(result.result)); + } + } + ); + }); +} + +async function getCode(web3, id, address, blockNumber) { + return new Promise(function(resolve, reject) { + web3.currentProvider.send( + { + jsonrpc: "2.0", + method: "eth_getCode", + params: [address, blockNumber], + id + }, + function(err, result) { + if (err) { + reject(err); + } else { + resolve(result.result); + } + } + ); + }); +} + +async function getStorageAt(web3, id, address, position, blockNumber) { + return new Promise(function(resolve, reject) { + web3.currentProvider.send( + { + jsonrpc: "2.0", + method: "eth_getStorageAt", + params: [address, position, blockNumber], + id + }, + function(err, result) { + if (err) { + reject(err); + } else { + resolve(result.result); + } + } + ); + }); +} + +describe("Forking methods are Case Insensitive", () => { + let forkedContext; + let forkedAccounts; + let forkedBlockNumber; + let mainContext; + let mainAccounts; + let instance; + let id = 0; + const logger = { + log: function(msg) {} + }; + + before("Set up forked provider with web3 instance and deploy a contract", async function() { + this.timeout(5000); + + const contractRef = { + contractFiles: ["Snapshot"], + contractSubdirectory: "forking" + }; + + const ganacheProviderOptions = { + logger, + seed: "main provider" + }; + + forkedContext = await bootstrap(contractRef, ganacheProviderOptions); + forkedAccounts = await forkedContext.web3.eth.getAccounts(); + forkedBlockNumber = await forkedContext.web3.eth.getBlockNumber(); + }); + + before("Set up main provider and web3 instance", async function() { + const { provider: forkedProvider } = forkedContext; + mainContext = await initializeTestProvider({ + fork: forkedProvider, + logger, + seed: "forked provider" + }); + mainAccounts = await mainContext.web3.eth.getAccounts(); + }); + + before("Make transaction to a forked account", async function() { + await mainContext.web3.eth.sendTransaction({ + from: mainAccounts[1], + to: forkedAccounts[1], + value: mainContext.web3.utils.toWei("1", "ether") + }); + }); + + before("Make a transaction to a forked contract that will modify storage", async function() { + const { instance: forkedInstance, abi } = forkedContext; + instance = new mainContext.web3.eth.Contract(abi, forkedInstance._address); + await instance.methods.test().send({ from: mainAccounts[0] }); + }); + + it("eth_getBalance", async function() { + const mainWeb3 = mainContext.web3; + + const addressLower = forkedAccounts[1].toLowerCase(); + const balanceBeforeForkLower = await getBalance(mainWeb3, id++, addressLower, forkedBlockNumber); + const balanceNowLower = await getBalance(mainWeb3, id++, addressLower, "latest"); + assert.strictEqual(balanceBeforeForkLower, mainWeb3.utils.toWei("100", "ether")); + assert.strictEqual(balanceNowLower, mainWeb3.utils.toWei("101", "ether")); + + const addressUpper = forkedAccounts[1].toUpperCase().replace(/^0X/, "0x"); + const balanceBeforeForkUpper = await getBalance(mainWeb3, id++, addressUpper, forkedBlockNumber); + const balanceNowUpper = await getBalance(mainWeb3, id++, addressUpper, "latest"); + assert.strictEqual(balanceBeforeForkUpper, mainWeb3.utils.toWei("100", "ether")); + assert.strictEqual(balanceNowUpper, mainWeb3.utils.toWei("101", "ether")); + + // ensure nothing got changed in these calls + const balanceNowLower2 = await getBalance(mainWeb3, id++, addressLower, "latest"); + assert.strictEqual(balanceNowLower2, balanceNowLower); + }); + + it("eth_getCode", async function() { + const mainWeb3 = mainContext.web3; + + const addressLower = forkedContext.instance._address.toLowerCase(); + const codeBeforeDeployLower = await getCode(mainWeb3, id++, addressLower, "earliest"); + const codeBeforeForkLower = await getCode(mainWeb3, id++, addressLower, forkedBlockNumber); + const codeNowLower = await getCode(mainWeb3, id++, addressLower, "latest"); + assert.strictEqual(codeBeforeDeployLower, "0x"); + assert.strictEqual(codeBeforeForkLower.length > 2, true); + assert.strictEqual(codeNowLower.length > 2, true); + assert.strictEqual(codeBeforeForkLower, codeNowLower); + + const addressUpper = forkedContext.instance._address.toUpperCase().replace(/^0X/, "0x"); + const codeBeforeDeployUpper = await getCode(mainWeb3, id++, addressUpper, "earliest"); + const codeBeforeForkUpper = await getCode(mainWeb3, id++, addressUpper, forkedBlockNumber); + const codeNowUpper = await getCode(mainWeb3, id++, addressUpper, "latest"); + assert.strictEqual(codeBeforeDeployUpper, "0x"); + assert.strictEqual(codeBeforeForkUpper.length > 2, true); + assert.strictEqual(codeNowUpper.length > 2, true); + assert.strictEqual(codeBeforeForkUpper, codeNowUpper); + + // ensure nothing got changed in these calls + const codeNowLower2 = await getCode(mainWeb3, id++, addressLower, "latest"); + assert.strictEqual(codeNowLower2, codeNowLower); + }); + + it("eth_getStorageAt", async function() { + const mainWeb3 = mainContext.web3; + + const addressLower = forkedContext.instance._address.toLowerCase(); + const valueBeforeForkLower = await getStorageAt(mainWeb3, id++, addressLower, 0, forkedBlockNumber); + const valueNowLower = await getStorageAt(mainWeb3, id++, addressLower, 0, "latest"); + assert.strictEqual(valueBeforeForkLower, "0x00"); + assert.strictEqual(valueNowLower, "0x01"); + + const addressUpper = forkedContext.instance._address.toUpperCase().replace(/^0X/, "0x"); + const valueBeforeForkUpper = await getStorageAt(mainWeb3, id++, addressUpper, 0, forkedBlockNumber); + const valueNowUpper = await getStorageAt(mainWeb3, id++, addressUpper, 0, "latest"); + assert.strictEqual(valueBeforeForkUpper, "0x00"); + assert.strictEqual(valueNowUpper, "0x01"); + + // ensure nothing got changed in these calls + const valueNowLower2 = await getStorageAt(mainWeb3, id++, addressLower, 0, "latest"); + assert.strictEqual(valueNowLower2, valueNowLower); + }); +}); diff --git a/test/local/forking/delete.js b/test/local/forking/delete.js index 57e6e88bf2..265f4bd5be 100644 --- a/test/local/forking/delete.js +++ b/test/local/forking/delete.js @@ -1,6 +1,6 @@ const assert = require("assert"); const bootstrap = require("../../helpers/contract/bootstrap"); -const intializeTestProvider = require("../../helpers/web3/initializeTestProvider"); +const initializeTestProvider = require("../../helpers/web3/initializeTestProvider"); /** * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" @@ -35,7 +35,7 @@ describe("Forking Deletion", () => { before("Set up main provider and web3 instance", async function() { const { provider: forkedProvider } = forkedContext; - mainContext = await intializeTestProvider({ + mainContext = await initializeTestProvider({ fork: forkedProvider, logger, seed: "forked provider" diff --git a/test/local/forking/forkingAsProvider.js b/test/local/forking/forkingAsProvider.js index 7ed9d2b380..07862e2408 100644 --- a/test/local/forking/forkingAsProvider.js +++ b/test/local/forking/forkingAsProvider.js @@ -1,6 +1,6 @@ const assert = require("assert"); const bootstrap = require("../../helpers/contract/bootstrap"); -const intializeTestProvider = require("../../helpers/web3/initializeTestProvider"); +const initializeTestProvider = require("../../helpers/web3/initializeTestProvider"); /** * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" @@ -35,7 +35,7 @@ describe("Forking using a Provider", () => { before("Set up main provider and web3 instance", async function() { const { provider: forkedProvider } = forkedContext; - mainContext = await intializeTestProvider({ + mainContext = await initializeTestProvider({ fork: forkedProvider, logger, seed: "forked provider" diff --git a/test/local/forking/snapshot.js b/test/local/forking/snapshot.js index 0b8f468f0c..54d0da3b3c 100644 --- a/test/local/forking/snapshot.js +++ b/test/local/forking/snapshot.js @@ -1,6 +1,6 @@ const assert = require("assert"); const bootstrap = require("../../helpers/contract/bootstrap"); -const intializeTestProvider = require("../../helpers/web3/initializeTestProvider"); +const initializeTestProvider = require("../../helpers/web3/initializeTestProvider"); /** * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" @@ -72,7 +72,7 @@ describe("Forking Snapshots", () => { before("Set up main provider and web3 instance", async function() { const { provider: forkedProvider } = forkedContext; - mainContext = await intializeTestProvider({ + mainContext = await initializeTestProvider({ fork: forkedProvider, logger, seed: "forked provider" From b4981bf442ad86b651448f2cded9b26d7ea5fa45 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Tue, 15 Sep 2020 19:14:54 -0700 Subject: [PATCH 15/19] add toLowerCase to ForkedStorageTrie touch functions --- lib/forking/forked_storage_trie.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/forking/forked_storage_trie.js b/lib/forking/forked_storage_trie.js index 5c4f681811..4963b5fd93 100644 --- a/lib/forking/forked_storage_trie.js +++ b/lib/forking/forked_storage_trie.js @@ -85,6 +85,7 @@ ForkedStorageBaseTrie.prototype.touch = function(key, callback) { if (this.address) { rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; } + rpcKey = rpcKey.toLowerCase(); this._touched.get(rpcKey, (err, result) => { if (err && err.type !== "NotFoundError") { @@ -126,6 +127,7 @@ ForkedStorageBaseTrie.prototype.getTouchedAt = function(key, callback) { if (this.address) { rpcKey = `${to.rpcDataHexString(this.address)};${rpcKey}`; } + rpcKey = rpcKey.toLowerCase(); this._touched.get(rpcKey, function(err, result) { if (err && err.type !== "NotFoundError") { From 392516af56e24a4cb666aa471748d7ed856b4a80 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Wed, 16 Sep 2020 17:00:24 -0700 Subject: [PATCH 16/19] use rpc helper instead of our own functions --- test/local/forking/caseSensitivity.js | 118 +++++++------------------- 1 file changed, 30 insertions(+), 88 deletions(-) diff --git a/test/local/forking/caseSensitivity.js b/test/local/forking/caseSensitivity.js index 153d93bbd8..aac90d2b25 100644 --- a/test/local/forking/caseSensitivity.js +++ b/test/local/forking/caseSensitivity.js @@ -1,5 +1,6 @@ const assert = require("assert"); const bootstrap = require("../../helpers/contract/bootstrap"); +const generateSend = require("../../helpers/utils/rpc"); const initializeTestProvider = require("../../helpers/web3/initializeTestProvider"); /** @@ -10,69 +11,6 @@ const initializeTestProvider = require("../../helpers/web3/initializeTestProvide * network) and the fork chain being "the fork". */ -// Defining our own functions to send raw rpc calls because web3 -// does a toLower on the address - -async function getBalance(web3, id, address, blockNumber) { - return new Promise(function(resolve, reject) { - web3.currentProvider.send( - { - jsonrpc: "2.0", - method: "eth_getBalance", - params: [address, blockNumber], - id - }, - function(err, result) { - if (err) { - reject(err); - } else { - resolve(web3.utils.hexToNumberString(result.result)); - } - } - ); - }); -} - -async function getCode(web3, id, address, blockNumber) { - return new Promise(function(resolve, reject) { - web3.currentProvider.send( - { - jsonrpc: "2.0", - method: "eth_getCode", - params: [address, blockNumber], - id - }, - function(err, result) { - if (err) { - reject(err); - } else { - resolve(result.result); - } - } - ); - }); -} - -async function getStorageAt(web3, id, address, position, blockNumber) { - return new Promise(function(resolve, reject) { - web3.currentProvider.send( - { - jsonrpc: "2.0", - method: "eth_getStorageAt", - params: [address, position, blockNumber], - id - }, - function(err, result) { - if (err) { - reject(err); - } else { - resolve(result.result); - } - } - ); - }); -} - describe("Forking methods are Case Insensitive", () => { let forkedContext; let forkedAccounts; @@ -80,7 +18,6 @@ describe("Forking methods are Case Insensitive", () => { let mainContext; let mainAccounts; let instance; - let id = 0; const logger = { log: function(msg) {} }; @@ -110,6 +47,11 @@ describe("Forking methods are Case Insensitive", () => { logger, seed: "forked provider" }); + const send = generateSend(mainContext.web3.currentProvider); + mainContext.send = async function() { + const result = await send(...arguments); + return result.result; + }; mainAccounts = await mainContext.web3.eth.getAccounts(); }); @@ -128,68 +70,68 @@ describe("Forking methods are Case Insensitive", () => { }); it("eth_getBalance", async function() { - const mainWeb3 = mainContext.web3; + const { web3, send } = mainContext; const addressLower = forkedAccounts[1].toLowerCase(); - const balanceBeforeForkLower = await getBalance(mainWeb3, id++, addressLower, forkedBlockNumber); - const balanceNowLower = await getBalance(mainWeb3, id++, addressLower, "latest"); - assert.strictEqual(balanceBeforeForkLower, mainWeb3.utils.toWei("100", "ether")); - assert.strictEqual(balanceNowLower, mainWeb3.utils.toWei("101", "ether")); + const balanceBeforeForkLower = await send("eth_getBalance", addressLower, forkedBlockNumber); + const balanceNowLower = await send("eth_getBalance", addressLower, "latest"); + assert.strictEqual(balanceBeforeForkLower, web3.utils.toHex(web3.utils.toWei("100", "ether"))); + assert.strictEqual(balanceNowLower, web3.utils.toHex(web3.utils.toWei("101", "ether"))); const addressUpper = forkedAccounts[1].toUpperCase().replace(/^0X/, "0x"); - const balanceBeforeForkUpper = await getBalance(mainWeb3, id++, addressUpper, forkedBlockNumber); - const balanceNowUpper = await getBalance(mainWeb3, id++, addressUpper, "latest"); - assert.strictEqual(balanceBeforeForkUpper, mainWeb3.utils.toWei("100", "ether")); - assert.strictEqual(balanceNowUpper, mainWeb3.utils.toWei("101", "ether")); + const balanceBeforeForkUpper = await send("eth_getBalance", addressUpper, forkedBlockNumber); + const balanceNowUpper = await send("eth_getBalance", addressUpper, "latest"); + assert.strictEqual(balanceBeforeForkUpper, web3.utils.toHex(web3.utils.toWei("100", "ether"))); + assert.strictEqual(balanceNowUpper, web3.utils.toHex(web3.utils.toWei("101", "ether"))); // ensure nothing got changed in these calls - const balanceNowLower2 = await getBalance(mainWeb3, id++, addressLower, "latest"); + const balanceNowLower2 = await send("eth_getBalance", addressLower, "latest"); assert.strictEqual(balanceNowLower2, balanceNowLower); }); it("eth_getCode", async function() { - const mainWeb3 = mainContext.web3; + const { send } = mainContext; const addressLower = forkedContext.instance._address.toLowerCase(); - const codeBeforeDeployLower = await getCode(mainWeb3, id++, addressLower, "earliest"); - const codeBeforeForkLower = await getCode(mainWeb3, id++, addressLower, forkedBlockNumber); - const codeNowLower = await getCode(mainWeb3, id++, addressLower, "latest"); + const codeBeforeDeployLower = await send("eth_getCode", addressLower, "earliest"); + const codeBeforeForkLower = await send("eth_getCode", addressLower, forkedBlockNumber); + const codeNowLower = await send("eth_getCode", addressLower, "latest"); assert.strictEqual(codeBeforeDeployLower, "0x"); assert.strictEqual(codeBeforeForkLower.length > 2, true); assert.strictEqual(codeNowLower.length > 2, true); assert.strictEqual(codeBeforeForkLower, codeNowLower); const addressUpper = forkedContext.instance._address.toUpperCase().replace(/^0X/, "0x"); - const codeBeforeDeployUpper = await getCode(mainWeb3, id++, addressUpper, "earliest"); - const codeBeforeForkUpper = await getCode(mainWeb3, id++, addressUpper, forkedBlockNumber); - const codeNowUpper = await getCode(mainWeb3, id++, addressUpper, "latest"); + const codeBeforeDeployUpper = await send("eth_getCode", addressUpper, "earliest"); + const codeBeforeForkUpper = await send("eth_getCode", addressUpper, forkedBlockNumber); + const codeNowUpper = await send("eth_getCode", addressUpper, "latest"); assert.strictEqual(codeBeforeDeployUpper, "0x"); assert.strictEqual(codeBeforeForkUpper.length > 2, true); assert.strictEqual(codeNowUpper.length > 2, true); assert.strictEqual(codeBeforeForkUpper, codeNowUpper); // ensure nothing got changed in these calls - const codeNowLower2 = await getCode(mainWeb3, id++, addressLower, "latest"); + const codeNowLower2 = await send("eth_getCode", addressLower, "latest"); assert.strictEqual(codeNowLower2, codeNowLower); }); it("eth_getStorageAt", async function() { - const mainWeb3 = mainContext.web3; + const { send } = mainContext; const addressLower = forkedContext.instance._address.toLowerCase(); - const valueBeforeForkLower = await getStorageAt(mainWeb3, id++, addressLower, 0, forkedBlockNumber); - const valueNowLower = await getStorageAt(mainWeb3, id++, addressLower, 0, "latest"); + const valueBeforeForkLower = await send("eth_getStorageAt", addressLower, 0, forkedBlockNumber); + const valueNowLower = await send("eth_getStorageAt", addressLower, 0, "latest"); assert.strictEqual(valueBeforeForkLower, "0x00"); assert.strictEqual(valueNowLower, "0x01"); const addressUpper = forkedContext.instance._address.toUpperCase().replace(/^0X/, "0x"); - const valueBeforeForkUpper = await getStorageAt(mainWeb3, id++, addressUpper, 0, forkedBlockNumber); - const valueNowUpper = await getStorageAt(mainWeb3, id++, addressUpper, 0, "latest"); + const valueBeforeForkUpper = await send("eth_getStorageAt", addressUpper, 0, forkedBlockNumber); + const valueNowUpper = await send("eth_getStorageAt", addressUpper, 0, "latest"); assert.strictEqual(valueBeforeForkUpper, "0x00"); assert.strictEqual(valueNowUpper, "0x01"); // ensure nothing got changed in these calls - const valueNowLower2 = await getStorageAt(mainWeb3, id++, addressLower, 0, "latest"); + const valueNowLower2 = await send("eth_getStorageAt", addressLower, 0, "latest"); assert.strictEqual(valueNowLower2, valueNowLower); }); }); From aa0f51367d3129fc470a4892711461f2ea48ccc2 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Mon, 21 Sep 2020 18:24:58 -0700 Subject: [PATCH 17/19] add failing test that gets wrong return value when debugging a forked tx --- test/contracts/forking/Debug.sol | 19 +++++++++ test/local/forking/debug.js | 73 ++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 test/contracts/forking/Debug.sol create mode 100644 test/local/forking/debug.js diff --git a/test/contracts/forking/Debug.sol b/test/contracts/forking/Debug.sol new file mode 100644 index 0000000000..19b12f24a0 --- /dev/null +++ b/test/contracts/forking/Debug.sol @@ -0,0 +1,19 @@ +pragma solidity ^0.6.0; + +contract Debug { + uint public value; + + constructor() public { + value = 1; + } + + function test() public mod() returns (uint) { + value = value + 1; + return value; + } + + modifier mod() { + require(value < 2); + _; + } +} diff --git a/test/local/forking/debug.js b/test/local/forking/debug.js new file mode 100644 index 0000000000..880d895d05 --- /dev/null +++ b/test/local/forking/debug.js @@ -0,0 +1,73 @@ +const assert = require("assert"); +const bootstrap = require("../../helpers/contract/bootstrap"); +const generateSend = require("../../helpers/utils/rpc"); +const initializeTestProvider = require("../../helpers/web3/initializeTestProvider"); + +/** + * NOTE: Naming in these tests is a bit confusing. Here, the "main chain" + * is the main chain the tests interact with; and the "forked chain" is the + * chain that _was forked_. This is in contrast to general naming, where the + * main chain represents the main chain to be forked (like the Ethereum live + * network) and the fork chaing being "the fork". + */ + +describe("Forking Debugging", () => { + let forkedContext; + let mainContext; + let mainAccounts; + const logger = { + log: function(msg) {} + }; + + before("Set up forked provider with web3 instance and deploy a contract", async function() { + this.timeout(5000); + + const contractRef = { + contractFiles: ["Debug"], + contractSubdirectory: "forking" + }; + + const ganacheProviderOptions = { + logger, + seed: "main provider" + }; + + forkedContext = await bootstrap(contractRef, ganacheProviderOptions); + }); + + before("Set up main provider and web3 instance", async function() { + const { provider: forkedProvider } = forkedContext; + mainContext = await initializeTestProvider({ + fork: forkedProvider, + logger, + seed: "forked provider" + }); + mainAccounts = await mainContext.web3.eth.getAccounts(); + }); + + it("successfully manages storage slot deletion", async() => { + const { instance: forkedInstance, abi } = forkedContext; + const { web3: mainWeb3 } = mainContext; + let value; + + const instance = new mainWeb3.eth.Contract(abi, forkedInstance._address); + + value = await instance.methods.value().call(); + assert.strictEqual(value, "1"); + + const tx = await instance.methods.test().send({ + from: mainAccounts[0] + }); + value = await instance.methods.value().call(); + assert.strictEqual(value, "2"); + + const send = generateSend(mainWeb3.currentProvider); + + const result = await send("debug_traceTransaction", tx.transactionHash, {}); + + const txStructLogs = result.result.structLogs; + const txMemory = txStructLogs[txStructLogs.length - 1].memory; + const txReturnValue = parseInt(txMemory[txMemory.length - 1], 16); + assert.strictEqual(txReturnValue, 2); + }); +}); From 301872fa075ea37d80dfd102f59594f69ff7e488 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Mon, 21 Sep 2020 18:35:03 -0700 Subject: [PATCH 18/19] add back check to check if the keyExists to support 'past' lookups --- lib/forking/forked_storage_trie.js | 56 +++++++++++++++++------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/lib/forking/forked_storage_trie.js b/lib/forking/forked_storage_trie.js index 4963b5fd93..dc2c0e4789 100644 --- a/lib/forking/forked_storage_trie.js +++ b/lib/forking/forked_storage_trie.js @@ -28,42 +28,48 @@ ForkedStorageBaseTrie.prototype.get = function(key, callback) { key = utils.toBuffer(key); - this.getTouchedAt(key, function(err, touchedAt) { + self.keyExists(key, function(err, keyExists) { if (err) { return callback(err); } - if (typeof touchedAt !== "undefined") { - MerklePatriciaTree.prototype.get.call(self, key, function(err, r) { - callback(err, r); - }); - } else { - // If this is the main trie, get the whole account. - if (self.address == null) { - self.blockchain.fetchAccountFromFallback(key, self.forkBlockNumber, function(err, account) { - if (err) { - return callback(err); - } - - callback(null, account.serialize()); + self.getTouchedAt(key, function(err, touchedAt) { + if (err) { + return callback(err); + } + + if (keyExists && typeof touchedAt !== "undefined") { + MerklePatriciaTree.prototype.get.call(self, key, function(err, r) { + callback(err, r); }); } else { - self.web3.eth.getStorageAt( - to.rpcDataHexString(self.address), - to.rpcDataHexString(key), - self.forkBlockNumber, - function(err, value) { + // If this is the main trie, get the whole account. + if (self.address == null) { + self.blockchain.fetchAccountFromFallback(key, self.forkBlockNumber, function(err, account) { if (err) { return callback(err); } - value = utils.rlp.encode(value); - - callback(null, value); - } - ); + callback(null, account.serialize()); + }); + } else { + self.web3.eth.getStorageAt( + to.rpcDataHexString(self.address), + to.rpcDataHexString(key), + self.forkBlockNumber, + function(err, value) { + if (err) { + return callback(err); + } + + value = utils.rlp.encode(value); + + callback(null, value); + } + ); + } } - } + }); }); }; From e7e50b87b2d220b61616a65d301195b33cde3ce4 Mon Sep 17 00:00:00 2001 From: Mike Seese Date: Tue, 22 Sep 2020 14:21:49 -0700 Subject: [PATCH 19/19] use `put(0)` for forked deletion to make sure the key stays in the trie --- lib/forking/forked_storage_trie.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/forking/forked_storage_trie.js b/lib/forking/forked_storage_trie.js index dc2c0e4789..6d0c8226e7 100644 --- a/lib/forking/forked_storage_trie.js +++ b/lib/forking/forked_storage_trie.js @@ -144,16 +144,8 @@ ForkedStorageBaseTrie.prototype.getTouchedAt = function(key, callback) { }); }; -const originalDelete = ForkedStorageBaseTrie.prototype.del; ForkedStorageBaseTrie.prototype.del = function(key, callback) { - const self = this; - this.touch(key, function(err) { - if (err) { - return callback(err); - } - - originalDelete.call(self, key, callback); - }); + this.put(key, 0, callback); }; ForkedStorageBaseTrie.prototype.copy = function() {