Add changes for stats collector (idena-network#1141)

* Collect balance updates in wasm env

* Collect used gas

* Collect contract balance update

* Collect contract balance update

* Collect burnt coins

* Add wrong grade reason to stats

* Collect deployed contracts

* Fix KillTx stake transfer amount

blockchain/blockchain.go

@@ -1388,6 +1388,7 @@ func (chain *Blockchain) processTxs(txs []*types.Transaction, context *txsExecut
 					return nil, nil, nil, nil, 0, errors.New("block can't contain skipped tx")
 				}
 			}
+			collector.AddTxGas(context.statsCollector, tx, gas)
 			if !chain.config.Consensus.EnableUpgrade10 {
 				if usedGas+gas > types.MaxBlockSize(chain.config.Consensus.EnableUpgrade11) {
 					return nil, nil, nil, nil, 0, errors.New("block exceeds gas limit")
@@ -1410,6 +1411,7 @@ func (chain *Blockchain) processTxs(txs []*types.Transaction, context *txsExecut
 			}
 		}
 	}
+	collector.AddBlockGas(context.statsCollector, usedGas)
 	return totalFee, totalTips, receipts, tasks, usedGas, nil
 }
 
@@ -1582,7 +1584,7 @@ func (chain *Blockchain) applyTxOnState(tx *types.Transaction, context *txExecut
 		stateDB.SubLockedStake(sender, stakeToBurn)
 		stateDB.SubReplenishedStake(sender, stateDB.GetReplenishedStakeBalance(sender))
 		stateDB.AddBalance(sender, stakeToBalance)
-		collector.AddKillTxStakeTransfer(statsCollector, tx, stake)
+		collector.AddKillTxStakeTransfer(statsCollector, tx, stakeToBalance)
 	case types.KillInviteeTx:
 		collector.BeginTxBalanceUpdate(statsCollector, tx, appState)
 		defer collector.CompleteBalanceUpdate(statsCollector, appState)
@@ -1676,7 +1678,7 @@ func (chain *Blockchain) applyTxOnState(tx *types.Transaction, context *txExecut
 		contractAddr := context.vm.ContractAddr(tx, &sender)
 
 		if shouldAddPayAmount {
-			collector.BeginTxBalanceUpdate(statsCollector, tx, appState)
+			collector.BeginTxBalanceUpdate(statsCollector, tx, appState, contractAddr)
 			stateDB.SubBalance(sender, amount)
 			stateDB.AddBalance(contractAddr, amount)
 			collector.CompleteBalanceUpdate(statsCollector, appState)
@@ -1685,7 +1687,7 @@ func (chain *Blockchain) applyTxOnState(tx *types.Transaction, context *txExecut
 		if receipt.Error != nil {
 			chain.log.Error("contract err", "err", receipt.Error)
 		}
-		collector.BeginTxBalanceUpdate(statsCollector, tx, appState)
+		collector.BeginTxBalanceUpdate(statsCollector, tx, appState, contractAddr)
 		defer collector.CompleteBalanceUpdate(statsCollector, appState)
 
 		if !receipt.Success && shouldAddPayAmount {

core/ceremony/ceremony.go

@@ -1089,7 +1089,8 @@ func (vc *ValidationCeremony) ApplyNewEpoch(height uint64, appState *appstate.Ap
 
 		totalFlipsCount := len(shard.flips)
 
-		flipQualification, reportersToReward := vc.qualification.qualifyFlips(uint(totalFlipsCount), shard.candidates, shard.longFlipsPerCandidate)
+		flipQualification, reportersToReward, wrongGradeReasons := vc.qualification.qualifyFlips(uint(totalFlipsCount), shard.candidates, shard.longFlipsPerCandidate)
+		stats.WrongGradeReasons = wrongGradeReasons
 
 		flipQualificationMap := make(map[int]FlipQualification)
 		for i, item := range flipQualification {

core/ceremony/qualification.go

@@ -114,7 +114,7 @@ func (q *qualification) restore() {
 	}
 }
 
-func (q *qualification) qualifyFlips(totalFlipsCount uint, candidates []*candidate, flipsPerCandidate [][]int) ([]FlipQualification, *reportersToReward) {
+func (q *qualification) qualifyFlips(totalFlipsCount uint, candidates []*candidate, flipsPerCandidate [][]int) ([]FlipQualification, *reportersToReward, map[common.Address]statsTypes.WrongGradeReason) {
 
 	q.lock.RLock()
 	defer q.lock.RUnlock()
@@ -126,6 +126,8 @@ func (q *qualification) qualifyFlips(totalFlipsCount uint, candidates []*candida
 		reportCommitteeSize int
 	}, totalFlipsCount)
 
+	wrongGradeReasons := make(map[common.Address]statsTypes.WrongGradeReason)
+
 	reportersToReward := newReportersToReward()
 	grades := newGrades()
 
@@ -173,6 +175,17 @@ func (q *qualification) qualifyFlips(totalFlipsCount uint, candidates []*candida
 				if ignoreGrades {
 					reportersToReward.deleteReporter(candidate.Address)
 					grades.deleteGrades(candidateIdx)
+					var reason statsTypes.WrongGradeReason
+					if ignoreReports {
+						reason |= statsTypes.TooManyReports
+					}
+					if !hasApprove {
+						reason |= statsTypes.NoApproves
+					}
+					if increasedGradeCnt > 1 {
+						reason |= statsTypes.TooManyIncreasedApproves
+					}
+					wrongGradeReasons[candidate.Address] = reason
 				}
 			} else if ignoreReports {
 				reportersToReward.deleteReporter(candidate.Address)
@@ -202,7 +215,7 @@ func (q *qualification) qualifyFlips(totalFlipsCount uint, candidates []*candida
 		}
 	}
 
-	return result, reportersToReward
+	return result, reportersToReward, wrongGradeReasons
 }
 
 func (q *qualification) qualifyCandidate(candidate common.Address, flipQualificationMap map[int]FlipQualification,

core/ceremony/qualification_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/idena-network/idena-go/crypto/ecies"
 	"github.com/idena-network/idena-go/crypto/vrf/p256"
 	"github.com/idena-network/idena-go/database"
+	types2 "github.com/idena-network/idena-go/stats/types"
 	"github.com/idena-network/idena-go/tests"
 	"github.com/shopspring/decimal"
 	"github.com/stretchr/testify/require"
@@ -510,7 +511,7 @@ func TestQualification_qualifyFlips(t *testing.T) {
 		},
 	}
 
-	flipQualifications, reportersToReward := q.qualifyFlips(11, candidates, flipsPerCandidate)
+	flipQualifications, reportersToReward, wrongGradeReasons := q.qualifyFlips(11, candidates, flipsPerCandidate)
 
 	require.Equal(t, 11, len(flipQualifications))
 	require.NotNil(t, reportersToReward)
@@ -558,4 +559,7 @@ func TestQualification_qualifyFlips(t *testing.T) {
 	require.NotNil(t, reportersToReward.reportersByFlip[reportedFlipIdx][addr1])
 	require.NotNil(t, reportersToReward.reportersByFlip[reportedFlipIdx][addr2])
 	require.NotNil(t, reportersToReward.reportersByFlip[reportedFlipIdx][addr3])
+
+	require.Len(t, wrongGradeReasons, 1)
+	require.Equal(t, wrongGradeReasons[addrWithIgnoredReports], types2.TooManyReports)
 }

stats/collector/collector.go

@@ -63,7 +63,7 @@ type StatsCollector interface {
 	AddKillTxStakeTransfer(tx *types.Transaction, amount *big.Int)
 
 	BeginVerifiedStakeTransferBalanceUpdate(addrFrom, addrTo common.Address, appState *appstate.AppState)
-	BeginTxBalanceUpdate(tx *types.Transaction, appState *appstate.AppState)
+	BeginTxBalanceUpdate(tx *types.Transaction, appState *appstate.AppState, additionalAddresses ...common.Address)
 	BeginProposerRewardBalanceUpdate(balanceDest, stakeDest common.Address, potentialPenaltyPayment *big.Int, appState *appstate.AppState)
 	BeginCommitteeRewardBalanceUpdate(balanceDest, stakeDest common.Address, potentialPenaltyPayment *big.Int, appState *appstate.AppState)
 	BeginEpochRewardBalanceUpdate(balanceDest, stakeDest common.Address, appState *appstate.AppState)
@@ -78,11 +78,15 @@ type StatsCollector interface {
 	BeginApplyingTx(tx *types.Transaction, appState *appstate.AppState)
 	CompleteApplyingTx(appState *appstate.AppState)
 	AddTxFee(feeAmount *big.Int)
+	AddTxGas(tx *types.Transaction, gas uint64)
+	AddBlockGas(gas uint64)
 
 	AddContractStake(amount *big.Int)
-	AddContractBalanceUpdate(address common.Address, getCurrentBalance GetBalanceFunc, newBalance *big.Int, appState *appstate.AppState)
-	AddContractBurntCoins(address common.Address, getAmount GetBalanceFunc)
+	AddContractBalanceUpdate(contractAddress *common.Address, address common.Address, getCurrentBalance GetBalanceFunc, newBalance *big.Int, appState *appstate.AppState, balancesCache *map[common.Address]*big.Int)
+	ApplyContractBalanceUpdates(balancesCache, parentBalancesCache *map[common.Address]*big.Int)
+	AddContractBurntCoins(address common.Address, getAmount GetBalanceFunc, balancesCache *map[common.Address]*big.Int)
 	AddContractTerminationBurntCoins(address common.Address, stake, refund *big.Int)
+	AddWasmContract(address common.Address, code []byte)
 
 	AddOracleVotingDeploy(contractAddress common.Address, startTime uint64, votingMinPayment *big.Int,
 		fact []byte, state byte, votingDuration, publicVotingDuration uint64, winnerThreshold, quorum byte,
@@ -591,15 +595,15 @@ func BeginVerifiedStakeTransferBalanceUpdate(c StatsCollector, addrFrom, addrTo
 	c.BeginVerifiedStakeTransferBalanceUpdate(addrFrom, addrTo, appState)
 }
 
-func (c *collectorStub) BeginTxBalanceUpdate(tx *types.Transaction, appState *appstate.AppState) {
+func (c *collectorStub) BeginTxBalanceUpdate(tx *types.Transaction, appState *appstate.AppState, additionalAddresses ...common.Address) {
 	// do nothing
 }
 
-func BeginTxBalanceUpdate(c StatsCollector, tx *types.Transaction, appState *appstate.AppState) {
+func BeginTxBalanceUpdate(c StatsCollector, tx *types.Transaction, appState *appstate.AppState, additionalAddresses ...common.Address) {
 	if c == nil {
 		return
 	}
-	c.BeginTxBalanceUpdate(tx, appState)
+	c.BeginTxBalanceUpdate(tx, appState, additionalAddresses...)
 }
 
 func (c *collectorStub) BeginProposerRewardBalanceUpdate(balanceDest, stakeDest common.Address, potentialPenaltyPayment *big.Int, appState *appstate.AppState) {
@@ -745,6 +749,28 @@ func AddTxFee(c StatsCollector, feeAmount *big.Int) {
 	c.AddTxFee(feeAmount)
 }
 
+func (c *collectorStub) AddTxGas(tx *types.Transaction, gas uint64) {
+	// do nothing
+}
+
+func AddTxGas(c StatsCollector, tx *types.Transaction, gas uint64) {
+	if c == nil {
+		return
+	}
+	c.AddTxGas(tx, gas)
+}
+
+func (c *collectorStub) AddBlockGas(gas uint64) {
+	// do nothing
+}
+
+func AddBlockGas(c StatsCollector, gas uint64) {
+	if c == nil {
+		return
+	}
+	c.AddBlockGas(gas)
+}
+
 func (c *collectorStub) AddContractStake(amount *big.Int) {
 	// do nothing
 }
@@ -756,26 +782,37 @@ func AddContractStake(c StatsCollector, amount *big.Int) {
 	c.AddContractStake(amount)
 }
 
-func (c *collectorStub) AddContractBalanceUpdate(address common.Address, getCurrentBalance GetBalanceFunc, newBalance *big.Int, appState *appstate.AppState) {
+func (c *collectorStub) AddContractBalanceUpdate(contractAddress *common.Address, address common.Address, getCurrentBalance GetBalanceFunc, newBalance *big.Int, appState *appstate.AppState, balancesCache *map[common.Address]*big.Int) {
 	// do nothing
 }
 
-func AddContractBalanceUpdate(c StatsCollector, address common.Address, getCurrentBalance GetBalanceFunc, newBalance *big.Int, appState *appstate.AppState) {
+func AddContractBalanceUpdate(c StatsCollector, contractAddress *common.Address, address common.Address, getCurrentBalance GetBalanceFunc, newBalance *big.Int, appState *appstate.AppState, balancesCache *map[common.Address]*big.Int) {
 	if c == nil {
 		return
 	}
-	c.AddContractBalanceUpdate(address, getCurrentBalance, newBalance, appState)
+	c.AddContractBalanceUpdate(contractAddress, address, getCurrentBalance, newBalance, appState, balancesCache)
 }
 
-func (c *collectorStub) AddContractBurntCoins(address common.Address, getAmount GetBalanceFunc) {
+func (c *collectorStub) ApplyContractBalanceUpdates(balancesCache, parentBalancesCache *map[common.Address]*big.Int) {
 	// do nothing
 }
 
-func AddContractBurntCoins(c StatsCollector, address common.Address, getAmount GetBalanceFunc) {
+func ApplyContractBalanceUpdates(c StatsCollector, balancesCache, parentBalancesCache *map[common.Address]*big.Int) {
 	if c == nil {
 		return
 	}
-	c.AddContractBurntCoins(address, getAmount)
+	c.ApplyContractBalanceUpdates(balancesCache, parentBalancesCache)
+}
+
+func (c *collectorStub) AddContractBurntCoins(address common.Address, getAmount GetBalanceFunc, balancesCache *map[common.Address]*big.Int) {
+	// do nothing
+}
+
+func AddContractBurntCoins(c StatsCollector, address common.Address, getAmount GetBalanceFunc, balancesCache *map[common.Address]*big.Int) {
+	if c == nil {
+		return
+	}
+	c.AddContractBurntCoins(address, getAmount, balancesCache)
 }
 
 func (c *collectorStub) AddContractTerminationBurntCoins(address common.Address, stake, refund *big.Int) {
@@ -789,6 +826,17 @@ func AddContractTerminationBurntCoins(c StatsCollector, address common.Address,
 	c.AddContractTerminationBurntCoins(address, stake, refund)
 }
 
+func (c *collectorStub) AddWasmContract(address common.Address, code []byte) {
+	// do nothing
+}
+
+func AddWasmContract(c StatsCollector, address common.Address, code []byte) {
+	if c == nil {
+		return
+	}
+	c.AddWasmContract(address, code)
+}
+
 func (c *collectorStub) AddOracleVotingDeploy(contractAddress common.Address, startTime uint64,
 	votingMinPayment *big.Int, fact []byte, state byte, votingDuration, publicVotingDuration uint64, winnerThreshold, quorum byte,
 	committeeSize, networkSize uint64, ownerFee byte, ownerDeposit, oracleRewardFund *big.Int, refundRecipient *common.Address, hash []byte) {

stats/types/types.go

@@ -15,6 +15,7 @@ type ValidationShardStats struct {
 	IdentitiesPerAddr map[common.Address]*IdentityStats
 	FlipsPerIdx       map[int]*FlipStats
 	FlipCids          [][]byte
+	WrongGradeReasons map[common.Address]WrongGradeReason
 }
 
 type IdentityStats struct {
@@ -52,3 +53,11 @@ func NewValidationStats() *ValidationShardStats {
 		FlipsPerIdx:       make(map[int]*FlipStats),
 	}
 }
+
+type WrongGradeReason uint32
+
+const (
+	TooManyReports WrongGradeReason = 1 << iota
+	NoApproves
+	TooManyIncreasedApproves
+)

vm/env/env.go

@@ -100,7 +100,7 @@ func (e *EnvImp) subBalance(address common.Address, amount *big.Int) {
 }
 
 func (e *EnvImp) setBalance(address common.Address, amount *big.Int) {
-	collector.AddContractBalanceUpdate(e.statsCollector, address, e.getBalance, amount, e.state)
+	collector.AddContractBalanceUpdate(e.statsCollector, nil, address, e.getBalance, amount, e.state, &e.balancesCache)
 	e.balancesCache[address] = amount
 }
 
@@ -247,7 +247,7 @@ func (e *EnvImp) Iterate(ctx CallContext, minKey []byte, maxKey []byte, f func(k
 func (e *EnvImp) BurnAll(ctx CallContext) {
 	e.gasCounter.AddGas(costs.BurnAllGas)
 	address := ctx.ContractAddr()
-	collector.AddContractBurntCoins(e.statsCollector, address, e.getBalance)
+	collector.AddContractBurntCoins(e.statsCollector, address, e.getBalance, &e.balancesCache)
 	e.setBalance(address, common.Big0)
 }
 
@@ -314,6 +314,7 @@ func (e *EnvImp) Commit() []*types.TxEvent {
 	for contract, stake := range e.contractStakeCache {
 		e.state.State.SetContractStake(contract, stake)
 	}
+	collector.ApplyContractBalanceUpdates(e.statsCollector, &e.balancesCache, nil)
 
 	return e.events
 }

vm/vm.go

@@ -179,7 +179,7 @@ func (vm *VmImpl) Run(tx *types.Transaction, from *common.Address, gasLimit int6
 	}
 
 	if vm.IsWasm(tx) {
-		wasmVm := wasm.NewWasmVM(vm.appState, vm.blockHeaderProvider, vm.head, vm.cfg, commitToState)
+		wasmVm := wasm.NewWasmVM(vm.appState, vm.blockHeaderProvider, vm.head, vm.cfg, commitToState, vm.statsCollector)
 		return wasmVm.Run(tx, costs.GasToWasmGas(uint64(gasLimit)))
 	}
 

vm/wasm/vm.go

@@ -8,6 +8,7 @@ import (
 	"github.com/idena-network/idena-go/common/math"
 	"github.com/idena-network/idena-go/config"
 	"github.com/idena-network/idena-go/core/appstate"
+	"github.com/idena-network/idena-go/stats/collector"
 	"github.com/idena-network/idena-go/vm/costs"
 	"github.com/idena-network/idena-wasm-binding/lib"
 )
@@ -18,11 +19,12 @@ type WasmVM struct {
 	head                *types.Header
 	cfg                 *config.Config
 	commitToState       bool
+	statsCollector      collector.StatsCollector
 }
 
 func (vm *WasmVM) deploy(tx *types.Transaction, limit uint64) (env *WasmEnv, gasUsed uint64, actionResult []byte, err error) {
 	ctx := NewContractContext(tx)
-	env = NewWasmEnv(vm.appState, vm.blockHeaderProvider, ctx, vm.head, "deploy", vm.cfg.IsDebug, vm.commitToState, vm.cfg.Consensus.EnableUpgrade12)
+	env = NewWasmEnv(vm.appState, vm.blockHeaderProvider, ctx, vm.head, "deploy", vm.cfg.IsDebug, vm.commitToState, vm.cfg.Consensus.EnableUpgrade12, vm.statsCollector)
 	attach := attachments.ParseDeployContractAttachment(tx)
 	actionResult = []byte{}
 	if attach == nil {
@@ -50,7 +52,7 @@ func (vm *WasmVM) call(tx *types.Transaction, limit uint64) (env *WasmEnv, gasUs
 		method = attachment.Method
 	}
 	ctx := NewContractContext(tx)
-	env = NewWasmEnv(vm.appState, vm.blockHeaderProvider, ctx, vm.head, method, vm.cfg.IsDebug, vm.commitToState, vm.cfg.Consensus.EnableUpgrade12)
+	env = NewWasmEnv(vm.appState, vm.blockHeaderProvider, ctx, vm.head, method, vm.cfg.IsDebug, vm.commitToState, vm.cfg.Consensus.EnableUpgrade12, vm.statsCollector)
 	contract := *tx.To
 	code := vm.appState.State.GetContractCode(contract)
 	actionResult = []byte{}
@@ -111,6 +113,6 @@ func (vm *WasmVM) Run(tx *types.Transaction, wasmGasLimit uint64) *types.TxRecei
 	}
 }
 
-func NewWasmVM(appState *appstate.AppState, blockHeaderProvider BlockHeaderProvider, head *types.Header, cfg *config.Config, commitToState bool) *WasmVM {
-	return &WasmVM{appState: appState, blockHeaderProvider: blockHeaderProvider, head: head, cfg: cfg, commitToState: commitToState}
+func NewWasmVM(appState *appstate.AppState, blockHeaderProvider BlockHeaderProvider, head *types.Header, cfg *config.Config, commitToState bool, statsCollector collector.StatsCollector) *WasmVM {
+	return &WasmVM{appState: appState, blockHeaderProvider: blockHeaderProvider, head: head, cfg: cfg, commitToState: commitToState, statsCollector: statsCollector}
 }

vm/wasm/vm_test.go

@@ -46,7 +46,7 @@ func TestVm_Erc20(t *testing.T) {
 	appState, _ := appstate.NewAppState(db, eventbus.New())
 	appState.Initialize(0)
 
-	vm := NewWasmVM(appState, nil, createHeader(1, 1), getLatestConfig(), true)
+	vm := NewWasmVM(appState, nil, createHeader(1, 1), getLatestConfig(), true, nil)
 	rnd := rand.New(rand.NewSource(1))
 	key, _ := crypto.GenerateKeyFromSeed(rnd)
 
@@ -121,7 +121,7 @@ func TestVm_Erc20(t *testing.T) {
 var nonce = uint32(1)
 
 func deployContract(key *ecdsa.PrivateKey, appState *appstate.AppState, code []byte, args ...[]byte) *types.TxReceipt {
-	vm := NewWasmVM(appState, nil, createHeader(1, 1), getLatestConfig(), true)
+	vm := NewWasmVM(appState, nil, createHeader(1, 1), getLatestConfig(), true, nil)
 	deployAttach := attachments.CreateDeployContractAttachment(common.Hash{}, code, nil, args...)
 	payload, _ := deployAttach.ToBytes()
 
@@ -138,7 +138,7 @@ func deployContract(key *ecdsa.PrivateKey, appState *appstate.AppState, code []b
 }
 
 func callContract(key *ecdsa.PrivateKey, appState *appstate.AppState, contract common.Address, method string, args ...[]byte) *types.TxReceipt {
-	vm := NewWasmVM(appState, nil, createHeader(1, 1), getLatestConfig(), true)
+	vm := NewWasmVM(appState, nil, createHeader(1, 1), getLatestConfig(), true, nil)
 	callAttach := attachments.CreateCallContractAttachment(method, args...)
 	payload, _ := callAttach.ToBytes()