diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000000..f722032b16 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,27 @@ +name: Build and Test +on: [push, pull_request] +jobs: + build: + name: Go CI + runs-on: ubuntu-latest + strategy: + matrix: + go: [1.13, 1.14] + steps: + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + - name: Check out source + uses: actions/checkout@v2 + - name: Install Linters + run: "curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.26.0" + - name: Build + env: + GO111MODULE: "on" + run: go build ./... + - name: Test + env: + GO111MODULE: "on" + run: | + sh ./goclean.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5a611c3960..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: go -cache: - directories: - - $GOCACHE - - $GOPATH/pkg/mod - - $GOPATH/github.com/golang - - $GOPATH/gopkg.in/alecthomas -go: - - "1.13.x" -sudo: false -install: - - export PATH=$PATH:$PWD/linux-amd64/ - - GO111MODULE=on go install . ./cmd/... - - GO111MODULE=off go get -u gopkg.in/alecthomas/gometalinter.v2 - - GO111MODULE=off gometalinter.v2 --install -script: - - export PATH=$PATH:$HOME/gopath/bin - - ./goclean.sh diff --git a/CHANGES b/CHANGES index 6f7013e922..d38897b339 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,161 @@ User visible changes for btcd A full-node bitcoin implementation written in Go ============================================================================ +Changes in 0.21.0 (Thu Aug 27 2020) + - Network-related changes: + - Handle notfound messages from peers in netsync package (#1603) + - RPC changes: + - Add compatibility for getblock RPC changes in bitcoind 0.15.0 (#1529) + - Add new optional Params field to rpcclient.ConnConfig (#1467) + - Add new error code ErrRPCInWarmup in btcjson (#1541) + - Add compatibility for changes to getmempoolentry response in bitcoind + 0.19.0 (#1524) + - Add rpcclient methods for estimatesmartfee and generatetoaddress + commands (#1500) + - Add rpcclient method for getblockstats command (#1500) + - Parse serialized transaction from createrawtransaction command using + both segwit, and legacy format (#1502) + - Support cookie-based authentication in rpcclient (#1460) + - Add rpcclient method for getchaintxstats command (#1571) + - Add rpcclient method for fundrawtransaction command (#1553) + - Add rpcclient method for getbalances command (#1595) + - Add new method rpcclient.GetTransactionWatchOnly (#1592) + - Crypto changes: + - Fix panic in fieldVal.SetByteSlice when called with large values, and + improve the method to be 35% faster (#1602) + - btcctl changes: + - Added -regtest mode to btcctl (#1556) + - Misc changes: + - Fix a bug due to a deadlock in connmgr's dynamic ban scoring (#1509) + - Add blockchain.NewUtxoEntry() to directly create entries for + UtxoViewpoint (#1588) + - Replace LRU cache implementation in peer package with a generic one + from decred/dcrd (#1599) + - Contributors (alphabetical order): + - Anirudha Bose + - Antonin Hildebrand + - Dan Cline + - Daniel McNally + - David Hill + - Federico Bond + - George Tankersley + - Henry + - Henry Harder + - Iskander Sharipov + - Ivan Kuznetsov + - Jake Sylvestre + - Javed Khan + - JeremyRand + - Jin + - John C. Vernaleo + - Kulpreet Singh + - Mikael Lindlof + - Murray Nesbitt + - Nisen + - Olaoluwa Osuntokun + - Oliver Gugger + - Steven Roose + - Torkel Rogstad + - Tyler Chambers + - Wilmer Paulino + - Yash Bhutwala + - adiabat + - jalavosus + - mohanson + - qqjettkgjzhxmwj + - qshuai + - shuai.qi + - tpkeeper + +Changes in v0.20.1 (Wed Nov 13 2019) + - RPC changes: + - Add compatibility for bitcoind v0.19.0 in rpcclient and btcjson + packages (#1484) + - Contributors (alphabetical order): + - Eugene Zeigel + - Olaoluwa Osuntokun + - Wilmer Paulino + +Changes in v0.20.0 (Tue Oct 15 2019) + - Significant changes made since 0.12.0. See git log or refer to release + notes on GitHub for full details. + - Contributors (alphabetical order): + - Albert Puigsech Galicia + - Alex Akselrod + - Alex Bosworth + - Alex Manuskin + - Alok Menghrajani + - Anatoli Babenia + - Andy Weidenbaum + - Calvin McAnarney + - Chris Martin + - Chris Pacia + - Chris Shepherd + - Conner Fromknecht + - Craig Sturdy + - Cédric Félizard + - Daniel Krawisz + - Daniel Martí + - Daniel McNally + - Dario Nieuwenhuis + - Dave Collins + - David Hill + - David de Kloet + - GeertJohan + - Grace Noah + - Gregory Trubetskoy + - Hector Jusforgues + - Iskander (Alex) Sharipov + - Janus Troelsen + - Jasper + - Javed Khan + - Jeremiah Goyette + - Jim Posen + - Jimmy Song + - Johan T. Halseth + - John C. Vernaleo + - Jonathan Gillham + - Josh Rickmar + - Jon Underwood + - Jonathan Zeppettini + - Jouke Hofman + - Julian Meyer + - Kai + - Kamil Slowikowski + - Kefkius + - Leonardo Lazzaro + - Marco Peereboom + - Marko Bencun + - Mawueli Kofi Adzoe + - Michail Kargakis + - Mitchell Paull + - Nathan Bass + - Nicola 'tekNico' Larosa + - Olaoluwa Osuntokun + - Pedro Martelletto + - Ricardo Velhote + - Roei Erez + - Ruben de Vries + - Rune T. Aune + - Sad Pencil + - Shuai Qi + - Steven Roose + - Tadge Dryja + - Tibor Bősze + - Tomás Senart + - Tzu-Jung Lee + - Vadym Popov + - Waldir Pimenta + - Wilmer Paulino + - benma + - danda + - dskloet + - esemplastic + - jadeblaquiere + - nakagawa + - preminem + - qshuai + Changes in 0.12.0 (Fri Nov 20 2015) - Protocol and network related changes: - Add a new checkpoint at block height 382320 (#555) diff --git a/README.md b/README.md index a270fd2e97..8c9d252ad4 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ btcd ==== -[![Build Status](https://travis-ci.org/btcsuite/btcd.png?branch=master)](https://travis-ci.org/btcsuite/btcd) -[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) -[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/btcsuite/btcd) +[![Build Status](https://github.com/btcsuite/btcd/workflows/Build%20and%20Test/badge.svg)](https://github.com/btcsuite/btcd/actions) +[![ISC License](https://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](https://godoc.org/github.com/btcsuite/btcd) btcd is an alternative full node bitcoin implementation written in Go (golang). diff --git a/blockchain/chainio.go b/blockchain/chainio.go index 8d76d3ca94..c456c006f3 100644 --- a/blockchain/chainio.go +++ b/blockchain/chainio.go @@ -1149,18 +1149,9 @@ func (b *BlockChain) initChainState() error { blockIndexBucket := dbTx.Metadata().Bucket(blockIndexBucketName) - // Determine how many blocks will be loaded into the index so we can - // allocate the right amount. - var blockCount int32 - cursor := blockIndexBucket.Cursor() - for ok := cursor.First(); ok; ok = cursor.Next() { - blockCount++ - } - blockNodes := make([]blockNode, blockCount) - var i int32 var lastNode *blockNode - cursor = blockIndexBucket.Cursor() + cursor := blockIndexBucket.Cursor() for ok := cursor.First(); ok; ok = cursor.Next() { header, status, err := deserializeBlockRow(cursor.Value()) if err != nil { @@ -1193,7 +1184,7 @@ func (b *BlockChain) initChainState() error { // Initialize the block node for the block, connect it, // and add it to the block index. - node := &blockNodes[i] + node := new(blockNode) initBlockNode(node, header, parent) node.status = status b.index.addNode(node) diff --git a/blockchain/difficulty_test.go b/blockchain/difficulty_test.go index b42b7c730d..6fed37f136 100644 --- a/blockchain/difficulty_test.go +++ b/blockchain/difficulty_test.go @@ -63,7 +63,7 @@ func TestCalcWork(t *testing.T) { } for x, test := range tests { - bits := uint32(test.in) + bits := test.in r := CalcWork(bits) if r.Int64() != test.out { diff --git a/blockchain/indexers/addrindex_test.go b/blockchain/indexers/addrindex_test.go index 135ef19196..e545887f8b 100644 --- a/blockchain/indexers/addrindex_test.go +++ b/blockchain/indexers/addrindex_test.go @@ -68,7 +68,7 @@ func (b *addrIndexBucket) printLevels(addrKey [addrKeySize]byte) string { if !bytes.Equal(k[:levelOffset], addrKey[:]) { continue } - level := uint8(k[levelOffset]) + level := k[levelOffset] if level > highestLevel { highestLevel = level } @@ -105,7 +105,7 @@ func (b *addrIndexBucket) sanityCheck(addrKey [addrKeySize]byte, expectedTotal i if !bytes.Equal(k[:levelOffset], addrKey[:]) { continue } - level := uint8(k[levelOffset]) + level := k[levelOffset] if level > highestLevel { highestLevel = level } diff --git a/blockchain/utxoviewpoint.go b/blockchain/utxoviewpoint.go index 08005d27a1..b85765814d 100644 --- a/blockchain/utxoviewpoint.go +++ b/blockchain/utxoviewpoint.go @@ -111,6 +111,22 @@ func (entry *UtxoEntry) Clone() *UtxoEntry { } } +// NewUtxoEntry returns a new UtxoEntry built from the arguments. +func NewUtxoEntry( + txOut *wire.TxOut, blockHeight int32, isCoinbase bool) *UtxoEntry { + var cbFlag txoFlags + if isCoinbase { + cbFlag |= tfCoinBase + } + + return &UtxoEntry{ + amount: txOut.Value, + pkScript: txOut.PkScript, + blockHeight: blockHeight, + packedFlags: cbFlag, + } +} + // UtxoViewpoint represents a view into the set of unspent transaction outputs // from a specific point of view in the chain. For example, it could be for // the end of the main chain, some point in the history of the main chain, or diff --git a/btcec/field.go b/btcec/field.go index c2bb84b3fe..98105ed8e4 100644 --- a/btcec/field.go +++ b/btcec/field.go @@ -226,20 +226,24 @@ func (f *fieldVal) SetBytes(b *[32]byte) *fieldVal { return f } -// SetByteSlice packs the passed big-endian value into the internal field value -// representation. Only the first 32-bytes are used. As a result, it is up to -// the caller to ensure numbers of the appropriate size are used or the value -// will be truncated. +// SetByteSlice interprets the provided slice as a 256-bit big-endian unsigned +// integer (meaning it is truncated to the first 32 bytes), packs it into the +// internal field value representation, and returns the updated field value. +// +// Note that since passing a slice with more than 32 bytes is truncated, it is +// possible that the truncated value is less than the field prime. It is up to +// the caller to decide whether it needs to provide numbers of the appropriate +// size or if it is acceptable to use this function with the described +// truncation behavior. // // The field value is returned to support chaining. This enables syntax like: // f := new(fieldVal).SetByteSlice(byteSlice) func (f *fieldVal) SetByteSlice(b []byte) *fieldVal { var b32 [32]byte - for i := 0; i < len(b); i++ { - if i < 32 { - b32[i+(32-len(b))] = b[i] - } + if len(b) > 32 { + b = b[:32] } + copy(b32[32-len(b):], b) return f.SetBytes(&b32) } diff --git a/btcec/field_test.go b/btcec/field_test.go index 27b9730f65..4226ba55f9 100644 --- a/btcec/field_test.go +++ b/btcec/field_test.go @@ -7,6 +7,7 @@ package btcec import ( "crypto/rand" + "encoding/hex" "fmt" "reflect" "testing" @@ -965,3 +966,156 @@ func testSqrt(t *testing.T, test sqrtTest) { } } } + +// TestFieldSetBytes ensures that setting a field value to a 256-bit big-endian +// unsigned integer via both the slice and array methods works as expected for +// edge cases. Random cases are tested via the various other tests. +func TestFieldSetBytes(t *testing.T) { + tests := []struct { + name string // test description + in string // hex encoded test value + expected [10]uint32 // expected raw ints + }{{ + name: "zero", + in: "00", + expected: [10]uint32{0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, { + name: "field prime", + in: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x03ffffbf, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x003fffff, + }, + }, { + name: "field prime - 1", + in: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2e", + expected: [10]uint32{ + 0x03fffc2e, 0x03ffffbf, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x003fffff, + }, + }, { + name: "field prime + 1 (overflow in word zero)", + in: "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc30", + expected: [10]uint32{ + 0x03fffc30, 0x03ffffbf, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x003fffff, + }, + }, { + name: "field prime first 32 bits", + in: "fffffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x00000003f, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + }, + }, { + name: "field prime word zero", + in: "03fffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + }, + }, { + name: "field prime first 64 bits", + in: "fffffffefffffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x03ffffbf, 0x00000fff, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + }, + }, { + name: "field prime word zero and one", + in: "0ffffefffffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x03ffffbf, 0x00000000, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + }, + }, { + name: "field prime first 96 bits", + in: "fffffffffffffffefffffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x03ffffbf, 0x03ffffff, 0x0003ffff, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + }, + }, { + name: "field prime word zero, one, and two", + in: "3ffffffffffefffffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x03ffffbf, 0x03ffffff, 0x00000000, 0x00000000, + 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, + }, + }, { + name: "overflow in word one (prime + 1<<26)", + in: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffff03fffc2f", + expected: [10]uint32{ + 0x03fffc2f, 0x03ffffc0, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x003fffff, + }, + }, { + name: "(field prime - 1) * 2 NOT mod P, truncated >32 bytes", + in: "01fffffffffffffffffffffffffffffffffffffffffffffffffffffffdfffff85c", + expected: [10]uint32{ + 0x01fffff8, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x00007fff, + }, + }, { + name: "2^256 - 1", + in: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + expected: [10]uint32{ + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, + 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x03ffffff, 0x003fffff, + }, + }, { + name: "alternating bits", + in: "a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5", + expected: [10]uint32{ + 0x01a5a5a5, 0x01696969, 0x025a5a5a, 0x02969696, 0x01a5a5a5, + 0x01696969, 0x025a5a5a, 0x02969696, 0x01a5a5a5, 0x00296969, + }, + }, { + name: "alternating bits 2", + in: "5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + expected: [10]uint32{ + 0x025a5a5a, 0x02969696, 0x01a5a5a5, 0x01696969, 0x025a5a5a, + 0x02969696, 0x01a5a5a5, 0x01696969, 0x025a5a5a, 0x00169696, + }, + }} + + for _, test := range tests { + inBytes := hexToBytes(test.in) + + // Ensure setting the bytes via the slice method works as expected. + var f fieldVal + f.SetByteSlice(inBytes) + if !reflect.DeepEqual(f.n, test.expected) { + t.Errorf("%s: unexpected result\ngot: %x\nwant: %x", test.name, f.n, + test.expected) + continue + } + + // Ensure setting the bytes via the array method works as expected. + var f2 fieldVal + var b32 [32]byte + truncatedInBytes := inBytes + if len(truncatedInBytes) > 32 { + truncatedInBytes = truncatedInBytes[:32] + } + copy(b32[32-len(truncatedInBytes):], truncatedInBytes) + f2.SetBytes(&b32) + if !reflect.DeepEqual(f2.n, test.expected) { + t.Errorf("%s: unexpected result\ngot: %x\nwant: %x", test.name, + f2.n, test.expected) + continue + } + } +} + +// hexToBytes converts the passed hex string into bytes and will panic if there +// is an error. This is only provided for the hard-coded constants so errors in +// the source code can be detected. It will only (and must only) be called with +// hard-coded values. +func hexToBytes(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic("invalid hex in source file: " + s) + } + return b +} diff --git a/btcec/pubkey_test.go b/btcec/pubkey_test.go index 0a45f1c01d..68b61de10b 100644 --- a/btcec/pubkey_test.go +++ b/btcec/pubkey_test.go @@ -232,11 +232,11 @@ func TestPubKeys(t *testing.T) { var pkStr []byte switch test.format { case pubkeyUncompressed: - pkStr = (*PublicKey)(pk).SerializeUncompressed() + pkStr = pk.SerializeUncompressed() case pubkeyCompressed: - pkStr = (*PublicKey)(pk).SerializeCompressed() + pkStr = pk.SerializeCompressed() case pubkeyHybrid: - pkStr = (*PublicKey)(pk).SerializeHybrid() + pkStr = pk.SerializeHybrid() } if !bytes.Equal(test.key, pkStr) { t.Errorf("%s pubkey: serialized keys do not match.", diff --git a/btcec/signature_test.go b/btcec/signature_test.go index 19ab772ef9..d238741455 100644 --- a/btcec/signature_test.go +++ b/btcec/signature_test.go @@ -464,8 +464,7 @@ func TestSignatureSerialize(t *testing.T) { func testSignCompact(t *testing.T, tag string, curve *KoblitzCurve, data []byte, isCompressed bool) { - tmp, _ := NewPrivateKey(curve) - priv := (*PrivateKey)(tmp) + priv, _ := NewPrivateKey(curve) hashed := []byte("testing") sig, err := SignCompact(curve, priv, hashed, isCompressed) diff --git a/btcjson/chainsvrcmds.go b/btcjson/chainsvrcmds.go index 90ab70ece7..d66478c305 100644 --- a/btcjson/chainsvrcmds.go +++ b/btcjson/chainsvrcmds.go @@ -8,6 +8,7 @@ package btcjson import ( + "encoding/hex" "encoding/json" "fmt" @@ -63,10 +64,15 @@ type CreateRawTransactionCmd struct { // NewCreateRawTransactionCmd returns a new instance which can be used to issue // a createrawtransaction JSON-RPC command. // -// Amounts are in BTC. +// Amounts are in BTC. Passing in nil and the empty slice as inputs is equivalent, +// both gets interpreted as the empty slice. func NewCreateRawTransactionCmd(inputs []TransactionInput, amounts map[string]float64, lockTime *int64) *CreateRawTransactionCmd { - + // to make sure we're serializing this to the empty list and not null, we + // explicitly initialize the list + if inputs == nil { + inputs = []TransactionInput{} + } return &CreateRawTransactionCmd{ Inputs: inputs, Amounts: amounts, @@ -74,6 +80,37 @@ func NewCreateRawTransactionCmd(inputs []TransactionInput, amounts map[string]fl } } +// FundRawTransactionOpts are the different options that can be passed to rawtransaction +type FundRawTransactionOpts struct { + ChangeAddress *string `json:"changeAddress,omitempty"` + ChangePosition *int `json:"changePosition,omitempty"` + ChangeType *string `json:"change_type,omitempty"` + IncludeWatching *bool `json:"includeWatching,omitempty"` + LockUnspents *bool `json:"lockUnspents,omitempty"` + FeeRate *float64 `json:"feeRate,omitempty"` // BTC/kB + SubtractFeeFromOutputs []int `json:"subtractFeeFromOutputs,omitempty"` + Replaceable *bool `json:"replaceable,omitempty"` + ConfTarget *int `json:"conf_target,omitempty"` + EstimateMode *EstimateSmartFeeMode `json:"estimate_mode,omitempty"` +} + +// FundRawTransactionCmd defines the fundrawtransaction JSON-RPC command +type FundRawTransactionCmd struct { + HexTx string + Options FundRawTransactionOpts + IsWitness *bool +} + +// NewFundRawTransactionCmd returns a new instance which can be used to issue +// a fundrawtransaction JSON-RPC command +func NewFundRawTransactionCmd(serializedTx []byte, opts FundRawTransactionOpts, isWitness *bool) *FundRawTransactionCmd { + return &FundRawTransactionCmd{ + HexTx: hex.EncodeToString(serializedTx), + Options: opts, + IsWitness: isWitness, + } +} + // DecodeRawTransactionCmd defines the decoderawtransaction JSON-RPC command. type DecodeRawTransactionCmd struct { HexTx string @@ -130,7 +167,7 @@ func NewGetBestBlockHashCmd() *GetBestBlockHashCmd { // GetBlockCmd defines the getblock JSON-RPC command. type GetBlockCmd struct { Hash string - Verbosity *int `jsonrpcdefault:"0"` + Verbosity *int `jsonrpcdefault:"1"` } // NewGetBlockCmd returns a new instance which can be used to issue a getblock @@ -363,6 +400,24 @@ func NewGetChainTipsCmd() *GetChainTipsCmd { return &GetChainTipsCmd{} } +// GetChainTxStatsCmd defines the getchaintxstats JSON-RPC command. +type GetChainTxStatsCmd struct { + NBlocks *int32 + BlockHash *string +} + +// NewGetChainTxStatsCmd returns a new instance which can be used to issue a +// getchaintxstats JSON-RPC command. +// +// The parameters which are pointers indicate they are optional. Passing nil +// for optional parameters will use the default value. +func NewGetChainTxStatsCmd(nBlocks *int32, blockHash *string) *GetChainTxStatsCmd { + return &GetChainTxStatsCmd{ + NBlocks: nBlocks, + BlockHash: blockHash, + } +} + // GetConnectionCountCmd defines the getconnectioncount JSON-RPC command. type GetConnectionCountCmd struct{} @@ -833,6 +888,7 @@ func init() { MustRegisterCmd("addnode", (*AddNodeCmd)(nil), flags) MustRegisterCmd("createrawtransaction", (*CreateRawTransactionCmd)(nil), flags) + MustRegisterCmd("fundrawtransaction", (*FundRawTransactionCmd)(nil), flags) MustRegisterCmd("decoderawtransaction", (*DecodeRawTransactionCmd)(nil), flags) MustRegisterCmd("decodescript", (*DecodeScriptCmd)(nil), flags) MustRegisterCmd("getaddednodeinfo", (*GetAddedNodeInfoCmd)(nil), flags) @@ -847,6 +903,7 @@ func init() { MustRegisterCmd("getcfilter", (*GetCFilterCmd)(nil), flags) MustRegisterCmd("getcfilterheader", (*GetCFilterHeaderCmd)(nil), flags) MustRegisterCmd("getchaintips", (*GetChainTipsCmd)(nil), flags) + MustRegisterCmd("getchaintxstats", (*GetChainTxStatsCmd)(nil), flags) MustRegisterCmd("getconnectioncount", (*GetConnectionCountCmd)(nil), flags) MustRegisterCmd("getdifficulty", (*GetDifficultyCmd)(nil), flags) MustRegisterCmd("getgenerate", (*GetGenerateCmd)(nil), flags) diff --git a/btcjson/chainsvrcmds_test.go b/btcjson/chainsvrcmds_test.go index dca2332860..e2b5025727 100644 --- a/btcjson/chainsvrcmds_test.go +++ b/btcjson/chainsvrcmds_test.go @@ -6,6 +6,7 @@ package btcjson_test import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "reflect" @@ -60,6 +61,21 @@ func TestChainSvrCmds(t *testing.T) { Amounts: map[string]float64{"456": .0123}, }, }, + { + name: "createrawtransaction - no inputs", + newCmd: func() (interface{}, error) { + return btcjson.NewCmd("createrawtransaction", `[]`, `{"456":0.0123}`) + }, + staticCmd: func() interface{} { + amounts := map[string]float64{"456": .0123} + return btcjson.NewCreateRawTransactionCmd(nil, amounts, nil) + }, + marshalled: `{"jsonrpc":"1.0","method":"createrawtransaction","params":[[],{"456":0.0123}],"id":1}`, + unmarshalled: &btcjson.CreateRawTransactionCmd{ + Inputs: []btcjson.TransactionInput{}, + Amounts: map[string]float64{"456": .0123}, + }, + }, { name: "createrawtransaction optional", newCmd: func() (interface{}, error) { @@ -80,7 +96,108 @@ func TestChainSvrCmds(t *testing.T) { LockTime: btcjson.Int64(12312333333), }, }, + { + name: "fundrawtransaction - empty opts", + newCmd: func() (i interface{}, e error) { + return btcjson.NewCmd("fundrawtransaction", "deadbeef", "{}") + }, + staticCmd: func() interface{} { + deadbeef, err := hex.DecodeString("deadbeef") + if err != nil { + panic(err) + } + return btcjson.NewFundRawTransactionCmd(deadbeef, btcjson.FundRawTransactionOpts{}, nil) + }, + marshalled: `{"jsonrpc":"1.0","method":"fundrawtransaction","params":["deadbeef",{}],"id":1}`, + unmarshalled: &btcjson.FundRawTransactionCmd{ + HexTx: "deadbeef", + Options: btcjson.FundRawTransactionOpts{}, + IsWitness: nil, + }, + }, + { + name: "fundrawtransaction - full opts", + newCmd: func() (i interface{}, e error) { + return btcjson.NewCmd("fundrawtransaction", "deadbeef", `{"changeAddress":"bcrt1qeeuctq9wutlcl5zatge7rjgx0k45228cxez655","changePosition":1,"change_type":"legacy","includeWatching":true,"lockUnspents":true,"feeRate":0.7,"subtractFeeFromOutputs":[0],"replaceable":true,"conf_target":8,"estimate_mode":"ECONOMICAL"}`) + }, + staticCmd: func() interface{} { + deadbeef, err := hex.DecodeString("deadbeef") + if err != nil { + panic(err) + } + changeAddress := "bcrt1qeeuctq9wutlcl5zatge7rjgx0k45228cxez655" + change := 1 + changeType := "legacy" + watching := true + lockUnspents := true + feeRate := 0.7 + replaceable := true + confTarget := 8 + return btcjson.NewFundRawTransactionCmd(deadbeef, btcjson.FundRawTransactionOpts{ + ChangeAddress: &changeAddress, + ChangePosition: &change, + ChangeType: &changeType, + IncludeWatching: &watching, + LockUnspents: &lockUnspents, + FeeRate: &feeRate, + SubtractFeeFromOutputs: []int{0}, + Replaceable: &replaceable, + ConfTarget: &confTarget, + EstimateMode: &btcjson.EstimateModeEconomical, + }, nil) + }, + marshalled: `{"jsonrpc":"1.0","method":"fundrawtransaction","params":["deadbeef",{"changeAddress":"bcrt1qeeuctq9wutlcl5zatge7rjgx0k45228cxez655","changePosition":1,"change_type":"legacy","includeWatching":true,"lockUnspents":true,"feeRate":0.7,"subtractFeeFromOutputs":[0],"replaceable":true,"conf_target":8,"estimate_mode":"ECONOMICAL"}],"id":1}`, + unmarshalled: func() interface{} { + changeAddress := "bcrt1qeeuctq9wutlcl5zatge7rjgx0k45228cxez655" + change := 1 + changeType := "legacy" + watching := true + lockUnspents := true + feeRate := 0.7 + replaceable := true + confTarget := 8 + return &btcjson.FundRawTransactionCmd{ + HexTx: "deadbeef", + Options: btcjson.FundRawTransactionOpts{ + ChangeAddress: &changeAddress, + ChangePosition: &change, + ChangeType: &changeType, + IncludeWatching: &watching, + LockUnspents: &lockUnspents, + FeeRate: &feeRate, + SubtractFeeFromOutputs: []int{0}, + Replaceable: &replaceable, + ConfTarget: &confTarget, + EstimateMode: &btcjson.EstimateModeEconomical, + }, + IsWitness: nil, + } + }(), + }, + { + name: "fundrawtransaction - iswitness", + newCmd: func() (i interface{}, e error) { + return btcjson.NewCmd("fundrawtransaction", "deadbeef", "{}", true) + }, + staticCmd: func() interface{} { + deadbeef, err := hex.DecodeString("deadbeef") + if err != nil { + panic(err) + } + t := true + return btcjson.NewFundRawTransactionCmd(deadbeef, btcjson.FundRawTransactionOpts{}, &t) + }, + marshalled: `{"jsonrpc":"1.0","method":"fundrawtransaction","params":["deadbeef",{},true],"id":1}`, + unmarshalled: &btcjson.FundRawTransactionCmd{ + HexTx: "deadbeef", + Options: btcjson.FundRawTransactionOpts{}, + IsWitness: func() *bool { + t := true + return &t + }(), + }, + }, { name: "decoderawtransaction", newCmd: func() (interface{}, error) { @@ -153,6 +270,20 @@ func TestChainSvrCmds(t *testing.T) { Verbosity: btcjson.Int(0), }, }, + { + name: "getblock default verbosity", + newCmd: func() (interface{}, error) { + return btcjson.NewCmd("getblock", "123") + }, + staticCmd: func() interface{} { + return btcjson.NewGetBlockCmd("123", nil) + }, + marshalled: `{"jsonrpc":"1.0","method":"getblock","params":["123"],"id":1}`, + unmarshalled: &btcjson.GetBlockCmd{ + Hash: "123", + Verbosity: btcjson.Int(1), + }, + }, { name: "getblock required optional1", newCmd: func() (interface{}, error) { @@ -408,6 +539,44 @@ func TestChainSvrCmds(t *testing.T) { marshalled: `{"jsonrpc":"1.0","method":"getchaintips","params":[],"id":1}`, unmarshalled: &btcjson.GetChainTipsCmd{}, }, + { + name: "getchaintxstats", + newCmd: func() (interface{}, error) { + return btcjson.NewCmd("getchaintxstats") + }, + staticCmd: func() interface{} { + return btcjson.NewGetChainTxStatsCmd(nil, nil) + }, + marshalled: `{"jsonrpc":"1.0","method":"getchaintxstats","params":[],"id":1}`, + unmarshalled: &btcjson.GetChainTxStatsCmd{}, + }, + { + name: "getchaintxstats optional nblocks", + newCmd: func() (interface{}, error) { + return btcjson.NewCmd("getchaintxstats", btcjson.Int32(1000)) + }, + staticCmd: func() interface{} { + return btcjson.NewGetChainTxStatsCmd(btcjson.Int32(1000), nil) + }, + marshalled: `{"jsonrpc":"1.0","method":"getchaintxstats","params":[1000],"id":1}`, + unmarshalled: &btcjson.GetChainTxStatsCmd{ + NBlocks: btcjson.Int32(1000), + }, + }, + { + name: "getchaintxstats optional nblocks and blockhash", + newCmd: func() (interface{}, error) { + return btcjson.NewCmd("getchaintxstats", btcjson.Int32(1000), btcjson.String("0000afaf")) + }, + staticCmd: func() interface{} { + return btcjson.NewGetChainTxStatsCmd(btcjson.Int32(1000), btcjson.String("0000afaf")) + }, + marshalled: `{"jsonrpc":"1.0","method":"getchaintxstats","params":[1000,"0000afaf"],"id":1}`, + unmarshalled: &btcjson.GetChainTxStatsCmd{ + NBlocks: btcjson.Int32(1000), + BlockHash: btcjson.String("0000afaf"), + }, + }, { name: "getconnectioncount", newCmd: func() (interface{}, error) { diff --git a/btcjson/chainsvrresults.go b/btcjson/chainsvrresults.go index 10490fc977..0bbd8679d7 100644 --- a/btcjson/chainsvrresults.go +++ b/btcjson/chainsvrresults.go @@ -4,7 +4,14 @@ package btcjson -import "encoding/json" +import ( + "bytes" + "encoding/hex" + "encoding/json" + + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" +) // GetBlockHeaderVerboseResult models the data from the getblockheader command when // the verbose flag is set. When the verbose flag is not set, getblockheader @@ -107,6 +114,18 @@ type GetBlockVerboseTxResult struct { NextHash string `json:"nextblockhash,omitempty"` } +// GetChainTxStatsResult models the data from the getchaintxstats command. +type GetChainTxStatsResult struct { + Time int64 `json:"time"` + TxCount int64 `json:"txcount"` + WindowFinalBlockHash string `json:"window_final_block_hash"` + WindowFinalBlockHeight int32 `json:"window_final_block_height"` + WindowBlockCount int32 `json:"window_block_count"` + WindowTxCount int32 `json:"window_tx_count"` + WindowInterval int32 `json:"window_interval"` + TxRate float64 `json:"txrate"` +} + // CreateMultiSigResult models the data returned from the createmultisig // command. type CreateMultiSigResult struct { @@ -271,7 +290,7 @@ type GetBlockTemplateResult struct { type MempoolFees struct { Base float64 `json:"base"` - Modified float64 `json:"base"` + Modified float64 `json:"modified"` Ancestor float64 `json:"ancestor"` Descendant float64 `json:"descendant"` } @@ -664,3 +683,50 @@ type EstimateSmartFeeResult struct { Errors []string `json:"errors,omitempty"` Blocks int64 `json:"blocks"` } + +var _ json.Unmarshaler = &FundRawTransactionResult{} + +type rawFundRawTransactionResult struct { + Transaction string `json:"hex"` + Fee float64 `json:"fee"` + ChangePosition int `json:"changepos"` +} + +// FundRawTransactionResult is the result of the fundrawtransaction JSON-RPC call +type FundRawTransactionResult struct { + Transaction *wire.MsgTx + Fee btcutil.Amount + ChangePosition int // the position of the added change output, or -1 +} + +// UnmarshalJSON unmarshals the result of the fundrawtransaction JSON-RPC call +func (f *FundRawTransactionResult) UnmarshalJSON(data []byte) error { + var rawRes rawFundRawTransactionResult + if err := json.Unmarshal(data, &rawRes); err != nil { + return err + } + + txBytes, err := hex.DecodeString(rawRes.Transaction) + if err != nil { + return err + } + + var msgTx wire.MsgTx + witnessErr := msgTx.Deserialize(bytes.NewReader(txBytes)) + if witnessErr != nil { + legacyErr := msgTx.DeserializeNoWitness(bytes.NewReader(txBytes)) + if legacyErr != nil { + return legacyErr + } + } + + fee, err := btcutil.NewAmount(rawRes.Fee) + if err != nil { + return err + } + + f.Transaction = &msgTx + f.Fee = fee + f.ChangePosition = rawRes.ChangePosition + return nil +} diff --git a/btcjson/cmdinfo_test.go b/btcjson/cmdinfo_test.go index 471fccfc77..61a693e404 100644 --- a/btcjson/cmdinfo_test.go +++ b/btcjson/cmdinfo_test.go @@ -151,7 +151,7 @@ func TestMethodUsageText(t *testing.T) { { name: "getblock", method: "getblock", - expected: `getblock "hash" (verbosity=0)`, + expected: `getblock "hash" (verbosity=1)`, }, } diff --git a/btcjson/register.go b/btcjson/register.go index 5de001c91e..10cd0f9a86 100644 --- a/btcjson/register.go +++ b/btcjson/register.go @@ -287,6 +287,6 @@ func RegisteredCmdMethods() []string { methods = append(methods, k) } - sort.Sort(sort.StringSlice(methods)) + sort.Strings(methods) return methods } diff --git a/btcjson/register_test.go b/btcjson/register_test.go index 3832678aaf..2d3ab10f3e 100644 --- a/btcjson/register_test.go +++ b/btcjson/register_test.go @@ -256,7 +256,7 @@ func TestRegisteredCmdMethods(t *testing.T) { // Ensure the returned methods are sorted. sortedMethods := make([]string, len(methods)) copy(sortedMethods, methods) - sort.Sort(sort.StringSlice(sortedMethods)) + sort.Strings(sortedMethods) if !reflect.DeepEqual(sortedMethods, methods) { t.Fatal("RegisteredCmdMethods: methods are not sorted") } diff --git a/btcjson/walletsvrcmds.go b/btcjson/walletsvrcmds.go index 53bb93da14..be1a67fbf0 100644 --- a/btcjson/walletsvrcmds.go +++ b/btcjson/walletsvrcmds.go @@ -188,6 +188,15 @@ func NewGetBalanceCmd(account *string, minConf *int) *GetBalanceCmd { } } +// GetBalancesCmd defines the getbalances JSON-RPC command. +type GetBalancesCmd struct{} + +// NewGetBalancesCmd returns a new instance which can be used to issue a +// getbalances JSON-RPC command. +func NewGetBalancesCmd() *GetBalancesCmd { + return &GetBalancesCmd{} +} + // GetNewAddressCmd defines the getnewaddress JSON-RPC command. type GetNewAddressCmd struct { Account *string @@ -693,6 +702,7 @@ func init() { MustRegisterCmd("getaccountaddress", (*GetAccountAddressCmd)(nil), flags) MustRegisterCmd("getaddressesbyaccount", (*GetAddressesByAccountCmd)(nil), flags) MustRegisterCmd("getbalance", (*GetBalanceCmd)(nil), flags) + MustRegisterCmd("getbalances", (*GetBalancesCmd)(nil), flags) MustRegisterCmd("getnewaddress", (*GetNewAddressCmd)(nil), flags) MustRegisterCmd("getrawchangeaddress", (*GetRawChangeAddressCmd)(nil), flags) MustRegisterCmd("getreceivedbyaccount", (*GetReceivedByAccountCmd)(nil), flags) diff --git a/btcjson/walletsvrcmds_test.go b/btcjson/walletsvrcmds_test.go index 2e0f1d0f8e..554a87413a 100644 --- a/btcjson/walletsvrcmds_test.go +++ b/btcjson/walletsvrcmds_test.go @@ -250,6 +250,17 @@ func TestWalletSvrCmds(t *testing.T) { MinConf: btcjson.Int(6), }, }, + { + name: "getbalances", + newCmd: func() (interface{}, error) { + return btcjson.NewCmd("getbalances") + }, + staticCmd: func() interface{} { + return btcjson.NewGetBalancesCmd() + }, + marshalled: `{"jsonrpc":"1.0","method":"getbalances","params":[],"id":1}`, + unmarshalled: &btcjson.GetBalancesCmd{}, + }, { name: "getnewaddress", newCmd: func() (interface{}, error) { diff --git a/btcjson/walletsvrresults.go b/btcjson/walletsvrresults.go index 9246d131ae..6e69ed9066 100644 --- a/btcjson/walletsvrresults.go +++ b/btcjson/walletsvrresults.go @@ -159,3 +159,17 @@ type GetBestBlockResult struct { Hash string `json:"hash"` Height int32 `json:"height"` } + +// BalanceDetailsResult models the details data from the `getbalances` command. +type BalanceDetailsResult struct { + Trusted float64 `json:"trusted"` + UntrustedPending float64 `json:"untrusted_pending"` + Immature float64 `json:"immature"` + Used *float64 `json:"used"` +} + +// GetBalancesResult models the data returned from the getbalances command. +type GetBalancesResult struct { + Mine BalanceDetailsResult `json:"mine"` + WatchOnly *BalanceDetailsResult `json:"watchonly"` +} diff --git a/cmd/btcctl/config.go b/cmd/btcctl/config.go index 8ed85f352b..939b6a8699 100644 --- a/cmd/btcctl/config.go +++ b/cmd/btcctl/config.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/btcsuite/btcd/btcjson" + "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcutil" flags "github.com/jessevdk/go-flags" ) @@ -92,42 +93,51 @@ func listCommands() { // // See loadConfig for details on the configuration load process. type config struct { - ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` - ListCommands bool `short:"l" long:"listcommands" description:"List all of the supported commands and exit"` - NoTLS bool `long:"notls" description:"Disable TLS"` - Proxy string `long:"proxy" description:"Connect via SOCKS5 proxy (eg. 127.0.0.1:9050)"` - ProxyPass string `long:"proxypass" default-mask:"-" description:"Password for proxy server"` - ProxyUser string `long:"proxyuser" description:"Username for proxy server"` - RPCCert string `short:"c" long:"rpccert" description:"RPC server certificate chain for validation"` - RPCPassword string `short:"P" long:"rpcpass" default-mask:"-" description:"RPC password"` - RPCServer string `short:"s" long:"rpcserver" description:"RPC server to connect to"` - RPCUser string `short:"u" long:"rpcuser" description:"RPC username"` - SimNet bool `long:"simnet" description:"Connect to the simulation test network"` - TLSSkipVerify bool `long:"skipverify" description:"Do not verify tls certificates (not recommended!)"` - TestNet3 bool `long:"testnet" description:"Connect to testnet"` - ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` - Wallet bool `long:"wallet" description:"Connect to wallet"` + ConfigFile string `short:"C" long:"configfile" description:"Path to configuration file"` + ListCommands bool `short:"l" long:"listcommands" description:"List all of the supported commands and exit"` + NoTLS bool `long:"notls" description:"Disable TLS"` + Proxy string `long:"proxy" description:"Connect via SOCKS5 proxy (eg. 127.0.0.1:9050)"` + ProxyPass string `long:"proxypass" default-mask:"-" description:"Password for proxy server"` + ProxyUser string `long:"proxyuser" description:"Username for proxy server"` + RegressionTest bool `long:"regtest" description:"Connect to the regression test network"` + RPCCert string `short:"c" long:"rpccert" description:"RPC server certificate chain for validation"` + RPCPassword string `short:"P" long:"rpcpass" default-mask:"-" description:"RPC password"` + RPCServer string `short:"s" long:"rpcserver" description:"RPC server to connect to"` + RPCUser string `short:"u" long:"rpcuser" description:"RPC username"` + SimNet bool `long:"simnet" description:"Connect to the simulation test network"` + TLSSkipVerify bool `long:"skipverify" description:"Do not verify tls certificates (not recommended!)"` + TestNet3 bool `long:"testnet" description:"Connect to testnet"` + ShowVersion bool `short:"V" long:"version" description:"Display version information and exit"` + Wallet bool `long:"wallet" description:"Connect to wallet"` } // normalizeAddress returns addr with the passed default port appended if // there is not already a port specified. -func normalizeAddress(addr string, useTestNet3, useSimNet, useWallet bool) string { +func normalizeAddress(addr string, chain *chaincfg.Params, useWallet bool) (string, error) { _, _, err := net.SplitHostPort(addr) if err != nil { var defaultPort string - switch { - case useTestNet3: + switch chain { + case &chaincfg.TestNet3Params: if useWallet { defaultPort = "18332" } else { defaultPort = "18334" } - case useSimNet: + case &chaincfg.SimNetParams: if useWallet { defaultPort = "18554" } else { defaultPort = "18556" } + case &chaincfg.RegressionNetParams: + if useWallet { + // TODO: add port once regtest is supported in btcwallet + paramErr := fmt.Errorf("cannot use -wallet with -regtest, btcwallet not yet compatible with regtest") + return "", paramErr + } else { + defaultPort = "18334" + } default: if useWallet { defaultPort = "8332" @@ -136,9 +146,9 @@ func normalizeAddress(addr string, useTestNet3, useSimNet, useWallet bool) strin } } - return net.JoinHostPort(addr, defaultPort) + return net.JoinHostPort(addr, defaultPort), nil } - return addr + return addr, nil } // cleanAndExpandPath expands environement variables and leading ~ in the @@ -246,17 +256,27 @@ func loadConfig() (*config, []string, error) { return nil, nil, err } + // default network is mainnet + network := &chaincfg.MainNetParams + // Multiple networks can't be selected simultaneously. numNets := 0 if cfg.TestNet3 { numNets++ + network = &chaincfg.TestNet3Params } if cfg.SimNet { numNets++ + network = &chaincfg.SimNetParams } + if cfg.RegressionTest { + numNets++ + network = &chaincfg.RegressionNetParams + } + if numNets > 1 { - str := "%s: The testnet and simnet params can't be used " + - "together -- choose one of the two" + str := "%s: Multiple network params can't be used " + + "together -- choose one" err := fmt.Errorf(str, "loadConfig") fmt.Fprintln(os.Stderr, err) return nil, nil, err @@ -273,8 +293,10 @@ func loadConfig() (*config, []string, error) { // Add default port to RPC server based on --testnet and --wallet flags // if needed. - cfg.RPCServer = normalizeAddress(cfg.RPCServer, cfg.TestNet3, - cfg.SimNet, cfg.Wallet) + cfg.RPCServer, err = normalizeAddress(cfg.RPCServer, network, cfg.Wallet) + if err != nil { + return nil, nil, err + } return &cfg, remainingArgs, nil } diff --git a/cmd/btcctl/version.go b/cmd/btcctl/version.go index f65cacef7e..f3a3e0b830 100644 --- a/cmd/btcctl/version.go +++ b/cmd/btcctl/version.go @@ -17,8 +17,8 @@ const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqr // versioning 2.0.0 spec (http://semver.org/). const ( appMajor uint = 0 - appMinor uint = 20 - appPatch uint = 1 + appMinor uint = 21 + appPatch uint = 0 // appPreRelease MUST only contain characters from semanticAlphabet // per the semantic versioning spec. diff --git a/config.go b/config.go index 96756f7f97..c44f3a837f 100644 --- a/config.go +++ b/config.go @@ -132,6 +132,7 @@ type config struct { NoOnion bool `long:"noonion" description:"Disable connecting to tor hidden services"` NoPeerBloomFilters bool `long:"nopeerbloomfilters" description:"Disable bloom filtering support"` NoRelayPriority bool `long:"norelaypriority" description:"Do not require free or low-fee transactions to have high priority for relaying"` + NoWinService bool `long:"nowinservice" description:"Do not start as a background service on Windows -- NOTE: This flag only works on the command line, not in the config file"` DisableRPC bool `long:"norpc" description:"Disable built-in RPC server -- NOTE: The RPC server is disabled by default if no rpcuser/rpcpass or rpclimituser/rpclimitpass is specified"` DisableTLS bool `long:"notls" description:"Disable TLS for the RPC server -- NOTE: This is only allowed if the RPC server is bound to localhost"` OnionProxy string `long:"onion" description:"Connect to tor hidden services via SOCKS5 proxy (eg. 127.0.0.1:9050)"` diff --git a/database/ffldb/whitebox_test.go b/database/ffldb/whitebox_test.go index 4e529363dc..161d866d29 100644 --- a/database/ffldb/whitebox_test.go +++ b/database/ffldb/whitebox_test.go @@ -643,9 +643,9 @@ func TestFailureScenarios(t *testing.T) { // context. maxSize := int64(-1) if maxFileSize, ok := tc.maxFileSizes[fileNum]; ok { - maxSize = int64(maxFileSize) + maxSize = maxFileSize } - file := &mockFile{maxSize: int64(maxSize)} + file := &mockFile{maxSize: maxSize} tc.files[fileNum] = &lockableFile{file: file} return file, nil } diff --git a/docs/json_rpc_api.md b/docs/json_rpc_api.md index 4bb0625e89..61884df0a0 100644 --- a/docs/json_rpc_api.md +++ b/docs/json_rpc_api.md @@ -271,13 +271,13 @@ the method name for further details such as parameter and return information. | | | |---|---| |Method|getblock| -|Parameters|1. block hash (string, required) - the hash of the block
2. verbose (boolean, optional, default=true) - specifies the block is returned as a JSON object instead of hex-encoded string
3. verbosetx (boolean, optional, default=false) - specifies that each transaction is returned as a JSON object and only applies if the `verbose` flag is true.**This parameter is a btcd extension**| +|Parameters|1. block hash (string, required) - the hash of the block
2. verbosity (int, optional, default=1) - Specifies whether the block data should be returned as a hex-encoded string (0), as parsed data with a slice of TXIDs (1), or as parsed data with parsed transaction data (2). |Description|Returns information about a block given its hash.| -|Returns (verbose=false)|`"data" (string) hex-encoded bytes of the serialized block`| -|Returns (verbose=true, verbosetx=false)|`{ (json object)`
  `"hash": "blockhash", (string) the hash of the block (same as provided)`
  `"confirmations": n, (numeric) the number of confirmations`
  `"strippedsize", n (numeric) the size of the block without witness data`
  `"size": n, (numeric) the size of the block`
  `"weight": n, (numeric) value of the weight metric`
  `"height": n, (numeric) the height of the block in the block chain`
  `"version": n, (numeric) the block version`
  `"merkleroot": "hash", (string) root hash of the merkle tree`
  `"tx": [ (json array of string) the transaction hashes`
    `"transactionhash", (string) hash of the parent transaction`
    `...`
  `]`
  `"time": n, (numeric) the block time in seconds since 1 Jan 1970 GMT`
  `"nonce": n, (numeric) the block nonce`
  `"bits", n, (numeric) the bits which represent the block difficulty`
  `difficulty: n.nn, (numeric) the proof-of-work difficulty as a multiple of the minimum difficulty`
  `"previousblockhash": "hash", (string) the hash of the previous block`
  `"nextblockhash": "hash", (string) the hash of the next block (only if there is one)`
`}`| -|Returns (verbose=true, verbosetx=true)|`{ (json object)`
  `"hash": "blockhash", (string) the hash of the block (same as provided)`
  `"confirmations": n, (numeric) the number of confirmations`
  `"strippedsize", n (numeric) the size of the block without witness data`
  `"size": n, (numeric) the size of the block`
  `"weight": n, (numeric) value of the weight metric`
  `"height": n, (numeric) the height of the block in the block chain`
  `"version": n, (numeric) the block version`
  `"merkleroot": "hash", (string) root hash of the merkle tree`
  `"rawtx": [ (array of json objects) the transactions as json objects`
    `(see getrawtransaction json object details)`
  `]`
  `"time": n, (numeric) the block time in seconds since 1 Jan 1970 GMT`
  `"nonce": n, (numeric) the block nonce`
  `"bits", n, (numeric) the bits which represent the block difficulty`
  `difficulty: n.nn, (numeric) the proof-of-work difficulty as a multiple of the minimum difficulty`
  `"previousblockhash": "hash", (string) the hash of the previous block`
  `"nextblockhash": "hash", (string) the hash of the next block`
`}`| -|Example Return (verbose=false)|`"010000000000000000000000000000000000000000000000000000000000000000000000`
`3ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49`
`ffff001d1dac2b7c01010000000100000000000000000000000000000000000000000000`
`00000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f`
`4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f`
`6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104`
`678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f`
`4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000"`
**Newlines added for display purposes. The actual return does not contain newlines.**| -|Example Return (verbose=true, verbosetx=false)|`{`
  `"hash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f",`
  `"confirmations": 277113,`
  `"size": 285,`
  `"height": 0,`
  `"version": 1,`
  `"merkleroot": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",`
  `"tx": [`
    `"4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"`
  `],`
  `"time": 1231006505,`
  `"nonce": 2083236893,`
  `"bits": "1d00ffff",`
  `"difficulty": 1,`
  `"previousblockhash": "0000000000000000000000000000000000000000000000000000000000000000",`
  `"nextblockhash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048"`
`}`| +|Returns (verbosity=0)|`"data" (string) hex-encoded bytes of the serialized block`| +|Returns (verbosity=1)|`{ (json object)`
  `"hash": "blockhash", (string) the hash of the block (same as provided)`
  `"confirmations": n, (numeric) the number of confirmations`
  `"strippedsize", n (numeric) the size of the block without witness data`
  `"size": n, (numeric) the size of the block`
  `"weight": n, (numeric) value of the weight metric`
  `"height": n, (numeric) the height of the block in the block chain`
  `"version": n, (numeric) the block version`
  `"merkleroot": "hash", (string) root hash of the merkle tree`
  `"tx": [ (json array of string) the transaction hashes`
    `"transactionhash", (string) hash of the parent transaction`
    `...`
  `]`
  `"time": n, (numeric) the block time in seconds since 1 Jan 1970 GMT`
  `"nonce": n, (numeric) the block nonce`
  `"bits", n, (numeric) the bits which represent the block difficulty`
  `difficulty: n.nn, (numeric) the proof-of-work difficulty as a multiple of the minimum difficulty`
  `"previousblockhash": "hash", (string) the hash of the previous block`
  `"nextblockhash": "hash", (string) the hash of the next block (only if there is one)`
`}`| +|Returns (verbosity=2)|`{ (json object)`
  `"hash": "blockhash", (string) the hash of the block (same as provided)`
  `"confirmations": n, (numeric) the number of confirmations`
  `"strippedsize", n (numeric) the size of the block without witness data`
  `"size": n, (numeric) the size of the block`
  `"weight": n, (numeric) value of the weight metric`
  `"height": n, (numeric) the height of the block in the block chain`
  `"version": n, (numeric) the block version`
  `"merkleroot": "hash", (string) root hash of the merkle tree`
  `"rawtx": [ (array of json objects) the transactions as json objects`
    `(see getrawtransaction json object details)`
  `]`
  `"time": n, (numeric) the block time in seconds since 1 Jan 1970 GMT`
  `"nonce": n, (numeric) the block nonce`
  `"bits", n, (numeric) the bits which represent the block difficulty`
  `difficulty: n.nn, (numeric) the proof-of-work difficulty as a multiple of the minimum difficulty`
  `"previousblockhash": "hash", (string) the hash of the previous block`
  `"nextblockhash": "hash", (string) the hash of the next block`
`}`| +|Example Return (verbosity=0)|`"010000000000000000000000000000000000000000000000000000000000000000000000`
`3ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49`
`ffff001d1dac2b7c01010000000100000000000000000000000000000000000000000000`
`00000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f`
`4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f`
`6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104`
`678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f`
`4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000"`
**Newlines added for display purposes. The actual return does not contain newlines.**| +|Example Return (verbosity=1)|`{`
  `"hash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f",`
  `"confirmations": 277113,`
  `"size": 285,`
  `"height": 0,`
  `"version": 1,`
  `"merkleroot": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b",`
  `"tx": [`
    `"4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"`
  `],`
  `"time": 1231006505,`
  `"nonce": 2083236893,`
  `"bits": "1d00ffff",`
  `"difficulty": 1,`
  `"previousblockhash": "0000000000000000000000000000000000000000000000000000000000000000",`
  `"nextblockhash": "00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048"`
`}`| [Return to Overview](#MethodOverview)
*** diff --git a/go.mod b/go.mod index 766bf8b8dd..582ceefe65 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,8 @@ require ( github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 github.com/btcsuite/winsvc v1.0.0 github.com/davecgh/go-spew v1.1.1 + github.com/decred/dcrd/lru v1.0.0 github.com/jessevdk/go-flags v1.4.0 github.com/jrick/logrotate v1.0.0 - golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 + golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 ) diff --git a/go.sum b/go.sum index 88d8d03454..392d70f1b7 100644 --- a/go.sum +++ b/go.sum @@ -21,10 +21,11 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 h1:R8vQdOQdZ9Y3 github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495 h1:6IyqGr3fnd0tM3YxipK27TUskaOVUjU2nG45yzwcQKY= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/lru v1.0.0 h1:Kbsb1SFDsIlaupWPwsPp+dkxiBY1frcS07PCPgotKz8= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -49,8 +50,8 @@ golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44 h1:9lP3x0pW80sDI6t1UMSLA4 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= -golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/goclean.sh b/goclean.sh index 6d0e0ca104..bb0cd7b600 100755 --- a/goclean.sh +++ b/goclean.sh @@ -1,31 +1,18 @@ #!/bin/bash # The script does automatic checking on a Go package and its sub-packages, including: # 1. gofmt (http://golang.org/cmd/gofmt/) -# 2. golint (https://github.com/golang/lint) # 3. go vet (http://golang.org/cmd/vet) # 4. gosimple (https://github.com/dominikh/go-simple) # 5. unconvert (https://github.com/mdempsky/unconvert) # -# gometalinter (github.com/alecthomas/gometalinter) is used to run each static -# checker. set -ex -# Make sure gometalinter is installed and $GOPATH/bin is in your path. -# $ go get -v github.com/alecthomas/gometalinter" -# $ gometalinter --install" -if [ ! -x "$(type -p gometalinter.v2)" ]; then - exit 1 -fi - -linter_targets=$(go list ./...) +go test -tags="rpctest" ./... # Automatic checks -test -z "$(gometalinter.v2 -j 4 --disable-all \ +golangci-lint run --deadline=10m --disable-all \ --enable=gofmt \ ---enable=golint \ --enable=vet \ --enable=gosimple \ ---enable=unconvert \ ---deadline=10m $linter_targets 2>&1 | grep -v 'ALL_CAPS\|OP_' 2>&1 | tee /dev/stderr)" -GO111MODULE=on go test -tags="rpctest" $linter_targets +--enable=unconvert diff --git a/mempool/mempool.go b/mempool/mempool.go index 7fd93ff489..9e09a31008 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -927,7 +927,7 @@ func (mp *TxPool) validateReplacement(tx *btcutil.Tx, func (mp *TxPool) maybeAcceptTransaction(tx *btcutil.Tx, isNew, rateLimit, rejectDupOrphans bool) ([]*chainhash.Hash, *TxDesc, error) { txHash := tx.Hash() - // If a transaction has iwtness data, and segwit isn't active yet, If + // If a transaction has witness data, and segwit isn't active yet, If // segwit isn't active yet, then we won't accept it into the mempool as // it can't be mined yet. if tx.MsgTx().HasWitness() { @@ -937,8 +937,14 @@ func (mp *TxPool) maybeAcceptTransaction(tx *btcutil.Tx, isNew, rateLimit, rejec } if !segwitActive { + simnetHint := "" + if mp.cfg.ChainParams.Net == wire.SimNet { + bestHeight := mp.cfg.BestHeight() + simnetHint = fmt.Sprintf(" (The threshold for segwit activation is 300 blocks on simnet, "+ + "current best height is %d)", bestHeight) + } str := fmt.Sprintf("transaction %v has witness data, "+ - "but segwit isn't active yet", txHash) + "but segwit isn't active yet%s", txHash, simnetHint) return nil, nil, txRuleError(wire.RejectNonstandard, str) } } diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 84aee10ab2..6d43cfd86e 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -1418,8 +1418,8 @@ func TestAncestorsDescendants(t *testing.T) { // We'll be querying for the ancestors of E. We should expect to see all // of the transactions that it depends on. expectedAncestors := map[chainhash.Hash]struct{}{ - *a.Hash(): struct{}{}, *b.Hash(): struct{}{}, - *c.Hash(): struct{}{}, *d.Hash(): struct{}{}, + *a.Hash(): {}, *b.Hash(): {}, + *c.Hash(): {}, *d.Hash(): {}, } ancestors := ctx.harness.txPool.txAncestors(e, nil) if len(ancestors) != len(expectedAncestors) { @@ -1436,8 +1436,8 @@ func TestAncestorsDescendants(t *testing.T) { // Then, we'll query for the descendants of A. We should expect to see // all of the transactions that depend on it. expectedDescendants := map[chainhash.Hash]struct{}{ - *b.Hash(): struct{}{}, *c.Hash(): struct{}{}, - *d.Hash(): struct{}{}, *e.Hash(): struct{}{}, + *b.Hash(): {}, *c.Hash(): {}, + *d.Hash(): {}, *e.Hash(): {}, } descendants := ctx.harness.txPool.txDescendants(a, nil) if len(descendants) != len(expectedDescendants) { diff --git a/netsync/manager.go b/netsync/manager.go index 32715b8e1e..603fca6ec8 100644 --- a/netsync/manager.go +++ b/netsync/manager.go @@ -79,6 +79,13 @@ type headersMsg struct { peer *peerpkg.Peer } +// notFoundMsg packages a bitcoin notfound message and the peer it came from +// together so the block handler has access to that information. +type notFoundMsg struct { + notFound *wire.MsgNotFound + peer *peerpkg.Peer +} + // donePeerMsg signifies a newly disconnected peer to the block handler. type donePeerMsg struct { peer *peerpkg.Peer @@ -147,6 +154,25 @@ type peerSyncState struct { requestedBlocks map[chainhash.Hash]struct{} } +// limitAdd is a helper function for maps that require a maximum limit by +// evicting a random value if adding the new value would cause it to +// overflow the maximum allowed. +func limitAdd(m map[chainhash.Hash]struct{}, hash chainhash.Hash, limit int) { + if len(m)+1 > limit { + // Remove a random entry from the map. For most compilers, Go's + // range statement iterates starting at a random item although + // that is not 100% guaranteed by the spec. The iteration order + // is not important here because an adversary would have to be + // able to pull off preimage attacks on the hashing function in + // order to target eviction of specific entries anyways. + for txHash := range m { + delete(m, txHash) + break + } + } + m[hash] = struct{}{} +} + // SyncManager is used to communicate block related messages with peers. The // SyncManager is started as by executing Start() in a goroutine. Once started, // it selects peers to sync from and starts the initial block download. Once the @@ -579,8 +605,7 @@ func (sm *SyncManager) handleTxMsg(tmsg *txMsg) { if err != nil { // Do not request this transaction again until a new block // has been processed. - sm.rejectedTxns[*txHash] = struct{}{} - sm.limitMap(sm.rejectedTxns, maxRejectedTxns) + limitAdd(sm.rejectedTxns, *txHash, maxRejectedTxns) // When the error is a rule error, it means the transaction was // simply rejected as opposed to something actually going wrong, @@ -994,6 +1019,32 @@ func (sm *SyncManager) handleHeadersMsg(hmsg *headersMsg) { } } +// handleNotFoundMsg handles notfound messages from all peers. +func (sm *SyncManager) handleNotFoundMsg(nfmsg *notFoundMsg) { + peer := nfmsg.peer + state, exists := sm.peerStates[peer] + if !exists { + log.Warnf("Received notfound message from unknown peer %s", peer) + return + } + for _, inv := range nfmsg.notFound.InvList { + // verify the hash was actually announced by the peer + // before deleting from the global requested maps. + switch inv.Type { + case wire.InvTypeBlock: + if _, exists := state.requestedBlocks[inv.Hash]; exists { + delete(state.requestedBlocks, inv.Hash) + delete(sm.requestedBlocks, inv.Hash) + } + case wire.InvTypeTx: + if _, exists := state.requestedTxns[inv.Hash]; exists { + delete(state.requestedTxns, inv.Hash) + delete(sm.requestedTxns, inv.Hash) + } + } + } +} + // haveInventory returns whether or not the inventory represented by the passed // inventory vector is known. This includes checking all of the various places // inventory can be when it is in different states such as blocks that are part @@ -1202,9 +1253,8 @@ func (sm *SyncManager) handleInvMsg(imsg *invMsg) { // Request the block if there is not already a pending // request. if _, exists := sm.requestedBlocks[iv.Hash]; !exists { - sm.requestedBlocks[iv.Hash] = struct{}{} - sm.limitMap(sm.requestedBlocks, maxRequestedBlocks) - state.requestedBlocks[iv.Hash] = struct{}{} + limitAdd(sm.requestedBlocks, iv.Hash, maxRequestedBlocks) + limitAdd(state.requestedBlocks, iv.Hash, maxRequestedBlocks) if peer.IsWitnessEnabled() { iv.Type = wire.InvTypeWitnessBlock @@ -1220,9 +1270,8 @@ func (sm *SyncManager) handleInvMsg(imsg *invMsg) { // Request the transaction if there is not already a // pending request. if _, exists := sm.requestedTxns[iv.Hash]; !exists { - sm.requestedTxns[iv.Hash] = struct{}{} - sm.limitMap(sm.requestedTxns, maxRequestedTxns) - state.requestedTxns[iv.Hash] = struct{}{} + limitAdd(sm.requestedTxns, iv.Hash, maxRequestedTxns) + limitAdd(state.requestedTxns, iv.Hash, maxRequestedTxns) // If the peer is capable, request the txn // including all witness data. @@ -1245,24 +1294,6 @@ func (sm *SyncManager) handleInvMsg(imsg *invMsg) { } } -// limitMap is a helper function for maps that require a maximum limit by -// evicting a random transaction if adding a new value would cause it to -// overflow the maximum allowed. -func (sm *SyncManager) limitMap(m map[chainhash.Hash]struct{}, limit int) { - if len(m)+1 > limit { - // Remove a random entry from the map. For most compilers, Go's - // range statement iterates starting at a random item although - // that is not 100% guaranteed by the spec. The iteration order - // is not important here because an adversary would have to be - // able to pull off preimage attacks on the hashing function in - // order to target eviction of specific entries anyways. - for txHash := range m { - delete(m, txHash) - return - } - } -} - // blockHandler is the main handler for the sync manager. It must be run as a // goroutine. It processes block and inv messages in a separate goroutine // from the peer handlers so the block (MsgBlock) messages are handled by a @@ -1295,6 +1326,9 @@ out: case *headersMsg: sm.handleHeadersMsg(msg) + case *notFoundMsg: + sm.handleNotFoundMsg(msg) + case *donePeerMsg: sm.handleDonePeerMsg(msg.peer) @@ -1492,6 +1526,18 @@ func (sm *SyncManager) QueueHeaders(headers *wire.MsgHeaders, peer *peerpkg.Peer sm.msgChan <- &headersMsg{headers: headers, peer: peer} } +// QueueNotFound adds the passed notfound message and peer to the block handling +// queue. +func (sm *SyncManager) QueueNotFound(notFound *wire.MsgNotFound, peer *peerpkg.Peer) { + // No channel handling here because peers do not need to block on + // reject messages. + if atomic.LoadInt32(&sm.shutdown) != 0 { + return + } + + sm.msgChan <- ¬FoundMsg{notFound: notFound, peer: peer} +} + // DonePeer informs the blockmanager that a peer has disconnected. func (sm *SyncManager) DonePeer(peer *peerpkg.Peer) { // Ignore if we are shutting down. diff --git a/peer/mruinvmap.go b/peer/mruinvmap.go deleted file mode 100644 index 0bf1b2b34a..0000000000 --- a/peer/mruinvmap.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright (c) 2013-2015 The btcsuite developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package peer - -import ( - "bytes" - "container/list" - "fmt" - "sync" - - "github.com/btcsuite/btcd/wire" -) - -// mruInventoryMap provides a concurrency safe map that is limited to a maximum -// number of items with eviction for the oldest entry when the limit is -// exceeded. -type mruInventoryMap struct { - invMtx sync.Mutex - invMap map[wire.InvVect]*list.Element // nearly O(1) lookups - invList *list.List // O(1) insert, update, delete - limit uint -} - -// String returns the map as a human-readable string. -// -// This function is safe for concurrent access. -func (m *mruInventoryMap) String() string { - m.invMtx.Lock() - defer m.invMtx.Unlock() - - lastEntryNum := len(m.invMap) - 1 - curEntry := 0 - buf := bytes.NewBufferString("[") - for iv := range m.invMap { - buf.WriteString(fmt.Sprintf("%v", iv)) - if curEntry < lastEntryNum { - buf.WriteString(", ") - } - curEntry++ - } - buf.WriteString("]") - - return fmt.Sprintf("<%d>%s", m.limit, buf.String()) -} - -// Exists returns whether or not the passed inventory item is in the map. -// -// This function is safe for concurrent access. -func (m *mruInventoryMap) Exists(iv *wire.InvVect) bool { - m.invMtx.Lock() - _, exists := m.invMap[*iv] - m.invMtx.Unlock() - - return exists -} - -// Add adds the passed inventory to the map and handles eviction of the oldest -// item if adding the new item would exceed the max limit. Adding an existing -// item makes it the most recently used item. -// -// This function is safe for concurrent access. -func (m *mruInventoryMap) Add(iv *wire.InvVect) { - m.invMtx.Lock() - defer m.invMtx.Unlock() - - // When the limit is zero, nothing can be added to the map, so just - // return. - if m.limit == 0 { - return - } - - // When the entry already exists move it to the front of the list - // thereby marking it most recently used. - if node, exists := m.invMap[*iv]; exists { - m.invList.MoveToFront(node) - return - } - - // Evict the least recently used entry (back of the list) if the the new - // entry would exceed the size limit for the map. Also reuse the list - // node so a new one doesn't have to be allocated. - if uint(len(m.invMap))+1 > m.limit { - node := m.invList.Back() - lru := node.Value.(*wire.InvVect) - - // Evict least recently used item. - delete(m.invMap, *lru) - - // Reuse the list node of the item that was just evicted for the - // new item. - node.Value = iv - m.invList.MoveToFront(node) - m.invMap[*iv] = node - return - } - - // The limit hasn't been reached yet, so just add the new item. - node := m.invList.PushFront(iv) - m.invMap[*iv] = node -} - -// Delete deletes the passed inventory item from the map (if it exists). -// -// This function is safe for concurrent access. -func (m *mruInventoryMap) Delete(iv *wire.InvVect) { - m.invMtx.Lock() - if node, exists := m.invMap[*iv]; exists { - m.invList.Remove(node) - delete(m.invMap, *iv) - } - m.invMtx.Unlock() -} - -// newMruInventoryMap returns a new inventory map that is limited to the number -// of entries specified by limit. When the number of entries exceeds the limit, -// the oldest (least recently used) entry will be removed to make room for the -// new entry. -func newMruInventoryMap(limit uint) *mruInventoryMap { - m := mruInventoryMap{ - invMap: make(map[wire.InvVect]*list.Element), - invList: list.New(), - limit: limit, - } - return &m -} diff --git a/peer/mruinvmap_test.go b/peer/mruinvmap_test.go deleted file mode 100644 index e55746010b..0000000000 --- a/peer/mruinvmap_test.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) 2013-2016 The btcsuite developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package peer - -import ( - "crypto/rand" - "fmt" - "testing" - - "github.com/btcsuite/btcd/chaincfg/chainhash" - "github.com/btcsuite/btcd/wire" -) - -// TestMruInventoryMap ensures the MruInventoryMap behaves as expected including -// limiting, eviction of least-recently used entries, specific entry removal, -// and existence tests. -func TestMruInventoryMap(t *testing.T) { - // Create a bunch of fake inventory vectors to use in testing the mru - // inventory code. - numInvVects := 10 - invVects := make([]*wire.InvVect, 0, numInvVects) - for i := 0; i < numInvVects; i++ { - hash := &chainhash.Hash{byte(i)} - iv := wire.NewInvVect(wire.InvTypeBlock, hash) - invVects = append(invVects, iv) - } - - tests := []struct { - name string - limit int - }{ - {name: "limit 0", limit: 0}, - {name: "limit 1", limit: 1}, - {name: "limit 5", limit: 5}, - {name: "limit 7", limit: 7}, - {name: "limit one less than available", limit: numInvVects - 1}, - {name: "limit all available", limit: numInvVects}, - } - -testLoop: - for i, test := range tests { - // Create a new mru inventory map limited by the specified test - // limit and add all of the test inventory vectors. This will - // cause evicition since there are more test inventory vectors - // than the limits. - mruInvMap := newMruInventoryMap(uint(test.limit)) - for j := 0; j < numInvVects; j++ { - mruInvMap.Add(invVects[j]) - } - - // Ensure the limited number of most recent entries in the - // inventory vector list exist. - for j := numInvVects - test.limit; j < numInvVects; j++ { - if !mruInvMap.Exists(invVects[j]) { - t.Errorf("Exists #%d (%s) entry %s does not "+ - "exist", i, test.name, *invVects[j]) - continue testLoop - } - } - - // Ensure the entries before the limited number of most recent - // entries in the inventory vector list do not exist. - for j := 0; j < numInvVects-test.limit; j++ { - if mruInvMap.Exists(invVects[j]) { - t.Errorf("Exists #%d (%s) entry %s exists", i, - test.name, *invVects[j]) - continue testLoop - } - } - - // Readd the entry that should currently be the least-recently - // used entry so it becomes the most-recently used entry, then - // force an eviction by adding an entry that doesn't exist and - // ensure the evicted entry is the new least-recently used - // entry. - // - // This check needs at least 2 entries. - if test.limit > 1 { - origLruIndex := numInvVects - test.limit - mruInvMap.Add(invVects[origLruIndex]) - - iv := wire.NewInvVect(wire.InvTypeBlock, - &chainhash.Hash{0x00, 0x01}) - mruInvMap.Add(iv) - - // Ensure the original lru entry still exists since it - // was updated and should've have become the mru entry. - if !mruInvMap.Exists(invVects[origLruIndex]) { - t.Errorf("MRU #%d (%s) entry %s does not exist", - i, test.name, *invVects[origLruIndex]) - continue testLoop - } - - // Ensure the entry that should've become the new lru - // entry was evicted. - newLruIndex := origLruIndex + 1 - if mruInvMap.Exists(invVects[newLruIndex]) { - t.Errorf("MRU #%d (%s) entry %s exists", i, - test.name, *invVects[newLruIndex]) - continue testLoop - } - } - - // Delete all of the entries in the inventory vector list, - // including those that don't exist in the map, and ensure they - // no longer exist. - for j := 0; j < numInvVects; j++ { - mruInvMap.Delete(invVects[j]) - if mruInvMap.Exists(invVects[j]) { - t.Errorf("Delete #%d (%s) entry %s exists", i, - test.name, *invVects[j]) - continue testLoop - } - } - } -} - -// TestMruInventoryMapStringer tests the stringized output for the -// MruInventoryMap type. -func TestMruInventoryMapStringer(t *testing.T) { - // Create a couple of fake inventory vectors to use in testing the mru - // inventory stringer code. - hash1 := &chainhash.Hash{0x01} - hash2 := &chainhash.Hash{0x02} - iv1 := wire.NewInvVect(wire.InvTypeBlock, hash1) - iv2 := wire.NewInvVect(wire.InvTypeBlock, hash2) - - // Create new mru inventory map and add the inventory vectors. - mruInvMap := newMruInventoryMap(uint(2)) - mruInvMap.Add(iv1) - mruInvMap.Add(iv2) - - // Ensure the stringer gives the expected result. Since map iteration - // is not ordered, either entry could be first, so account for both - // cases. - wantStr1 := fmt.Sprintf("<%d>[%s, %s]", 2, *iv1, *iv2) - wantStr2 := fmt.Sprintf("<%d>[%s, %s]", 2, *iv2, *iv1) - gotStr := mruInvMap.String() - if gotStr != wantStr1 && gotStr != wantStr2 { - t.Fatalf("unexpected string representation - got %q, want %q "+ - "or %q", gotStr, wantStr1, wantStr2) - } -} - -// BenchmarkMruInventoryList performs basic benchmarks on the most recently -// used inventory handling. -func BenchmarkMruInventoryList(b *testing.B) { - // Create a bunch of fake inventory vectors to use in benchmarking - // the mru inventory code. - b.StopTimer() - numInvVects := 100000 - invVects := make([]*wire.InvVect, 0, numInvVects) - for i := 0; i < numInvVects; i++ { - hashBytes := make([]byte, chainhash.HashSize) - rand.Read(hashBytes) - hash, _ := chainhash.NewHash(hashBytes) - iv := wire.NewInvVect(wire.InvTypeBlock, hash) - invVects = append(invVects, iv) - } - b.StartTimer() - - // Benchmark the add plus evicition code. - limit := 20000 - mruInvMap := newMruInventoryMap(uint(limit)) - for i := 0; i < b.N; i++ { - mruInvMap.Add(invVects[i%numInvVects]) - } -} diff --git a/peer/mrunoncemap.go b/peer/mrunoncemap.go deleted file mode 100644 index 5f7c798429..0000000000 --- a/peer/mrunoncemap.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (c) 2015 The btcsuite developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package peer - -import ( - "bytes" - "container/list" - "fmt" - "sync" -) - -// mruNonceMap provides a concurrency safe map that is limited to a maximum -// number of items with eviction for the oldest entry when the limit is -// exceeded. -type mruNonceMap struct { - mtx sync.Mutex - nonceMap map[uint64]*list.Element // nearly O(1) lookups - nonceList *list.List // O(1) insert, update, delete - limit uint -} - -// String returns the map as a human-readable string. -// -// This function is safe for concurrent access. -func (m *mruNonceMap) String() string { - m.mtx.Lock() - defer m.mtx.Unlock() - - lastEntryNum := len(m.nonceMap) - 1 - curEntry := 0 - buf := bytes.NewBufferString("[") - for nonce := range m.nonceMap { - buf.WriteString(fmt.Sprintf("%d", nonce)) - if curEntry < lastEntryNum { - buf.WriteString(", ") - } - curEntry++ - } - buf.WriteString("]") - - return fmt.Sprintf("<%d>%s", m.limit, buf.String()) -} - -// Exists returns whether or not the passed nonce is in the map. -// -// This function is safe for concurrent access. -func (m *mruNonceMap) Exists(nonce uint64) bool { - m.mtx.Lock() - _, exists := m.nonceMap[nonce] - m.mtx.Unlock() - - return exists -} - -// Add adds the passed nonce to the map and handles eviction of the oldest item -// if adding the new item would exceed the max limit. Adding an existing item -// makes it the most recently used item. -// -// This function is safe for concurrent access. -func (m *mruNonceMap) Add(nonce uint64) { - m.mtx.Lock() - defer m.mtx.Unlock() - - // When the limit is zero, nothing can be added to the map, so just - // return. - if m.limit == 0 { - return - } - - // When the entry already exists move it to the front of the list - // thereby marking it most recently used. - if node, exists := m.nonceMap[nonce]; exists { - m.nonceList.MoveToFront(node) - return - } - - // Evict the least recently used entry (back of the list) if the the new - // entry would exceed the size limit for the map. Also reuse the list - // node so a new one doesn't have to be allocated. - if uint(len(m.nonceMap))+1 > m.limit { - node := m.nonceList.Back() - lru := node.Value.(uint64) - - // Evict least recently used item. - delete(m.nonceMap, lru) - - // Reuse the list node of the item that was just evicted for the - // new item. - node.Value = nonce - m.nonceList.MoveToFront(node) - m.nonceMap[nonce] = node - return - } - - // The limit hasn't been reached yet, so just add the new item. - node := m.nonceList.PushFront(nonce) - m.nonceMap[nonce] = node -} - -// Delete deletes the passed nonce from the map (if it exists). -// -// This function is safe for concurrent access. -func (m *mruNonceMap) Delete(nonce uint64) { - m.mtx.Lock() - if node, exists := m.nonceMap[nonce]; exists { - m.nonceList.Remove(node) - delete(m.nonceMap, nonce) - } - m.mtx.Unlock() -} - -// newMruNonceMap returns a new nonce map that is limited to the number of -// entries specified by limit. When the number of entries exceeds the limit, -// the oldest (least recently used) entry will be removed to make room for the -// new entry. -func newMruNonceMap(limit uint) *mruNonceMap { - m := mruNonceMap{ - nonceMap: make(map[uint64]*list.Element), - nonceList: list.New(), - limit: limit, - } - return &m -} diff --git a/peer/mrunoncemap_test.go b/peer/mrunoncemap_test.go deleted file mode 100644 index bd7fb1082e..0000000000 --- a/peer/mrunoncemap_test.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright (c) 2015 The btcsuite developers -// Use of this source code is governed by an ISC -// license that can be found in the LICENSE file. - -package peer - -import ( - "fmt" - "testing" -) - -// TestMruNonceMap ensures the mruNonceMap behaves as expected including -// limiting, eviction of least-recently used entries, specific entry removal, -// and existence tests. -func TestMruNonceMap(t *testing.T) { - // Create a bunch of fake nonces to use in testing the mru nonce code. - numNonces := 10 - nonces := make([]uint64, 0, numNonces) - for i := 0; i < numNonces; i++ { - nonces = append(nonces, uint64(i)) - } - - tests := []struct { - name string - limit int - }{ - {name: "limit 0", limit: 0}, - {name: "limit 1", limit: 1}, - {name: "limit 5", limit: 5}, - {name: "limit 7", limit: 7}, - {name: "limit one less than available", limit: numNonces - 1}, - {name: "limit all available", limit: numNonces}, - } - -testLoop: - for i, test := range tests { - // Create a new mru nonce map limited by the specified test - // limit and add all of the test nonces. This will cause - // evicition since there are more test nonces than the limits. - mruNonceMap := newMruNonceMap(uint(test.limit)) - for j := 0; j < numNonces; j++ { - mruNonceMap.Add(nonces[j]) - } - - // Ensure the limited number of most recent entries in the list - // exist. - for j := numNonces - test.limit; j < numNonces; j++ { - if !mruNonceMap.Exists(nonces[j]) { - t.Errorf("Exists #%d (%s) entry %d does not "+ - "exist", i, test.name, nonces[j]) - continue testLoop - } - } - - // Ensure the entries before the limited number of most recent - // entries in the list do not exist. - for j := 0; j < numNonces-test.limit; j++ { - if mruNonceMap.Exists(nonces[j]) { - t.Errorf("Exists #%d (%s) entry %d exists", i, - test.name, nonces[j]) - continue testLoop - } - } - - // Readd the entry that should currently be the least-recently - // used entry so it becomes the most-recently used entry, then - // force an eviction by adding an entry that doesn't exist and - // ensure the evicted entry is the new least-recently used - // entry. - // - // This check needs at least 2 entries. - if test.limit > 1 { - origLruIndex := numNonces - test.limit - mruNonceMap.Add(nonces[origLruIndex]) - - mruNonceMap.Add(uint64(numNonces) + 1) - - // Ensure the original lru entry still exists since it - // was updated and should've have become the mru entry. - if !mruNonceMap.Exists(nonces[origLruIndex]) { - t.Errorf("MRU #%d (%s) entry %d does not exist", - i, test.name, nonces[origLruIndex]) - continue testLoop - } - - // Ensure the entry that should've become the new lru - // entry was evicted. - newLruIndex := origLruIndex + 1 - if mruNonceMap.Exists(nonces[newLruIndex]) { - t.Errorf("MRU #%d (%s) entry %d exists", i, - test.name, nonces[newLruIndex]) - continue testLoop - } - } - - // Delete all of the entries in the list, including those that - // don't exist in the map, and ensure they no longer exist. - for j := 0; j < numNonces; j++ { - mruNonceMap.Delete(nonces[j]) - if mruNonceMap.Exists(nonces[j]) { - t.Errorf("Delete #%d (%s) entry %d exists", i, - test.name, nonces[j]) - continue testLoop - } - } - } -} - -// TestMruNonceMapStringer tests the stringized output for the mruNonceMap type. -func TestMruNonceMapStringer(t *testing.T) { - // Create a couple of fake nonces to use in testing the mru nonce - // stringer code. - nonce1 := uint64(10) - nonce2 := uint64(20) - - // Create new mru nonce map and add the nonces. - mruNonceMap := newMruNonceMap(uint(2)) - mruNonceMap.Add(nonce1) - mruNonceMap.Add(nonce2) - - // Ensure the stringer gives the expected result. Since map iteration - // is not ordered, either entry could be first, so account for both - // cases. - wantStr1 := fmt.Sprintf("<%d>[%d, %d]", 2, nonce1, nonce2) - wantStr2 := fmt.Sprintf("<%d>[%d, %d]", 2, nonce2, nonce1) - gotStr := mruNonceMap.String() - if gotStr != wantStr1 && gotStr != wantStr2 { - t.Fatalf("unexpected string representation - got %q, want %q "+ - "or %q", gotStr, wantStr1, wantStr2) - } -} - -// BenchmarkMruNonceList performs basic benchmarks on the most recently used -// nonce handling. -func BenchmarkMruNonceList(b *testing.B) { - // Create a bunch of fake nonces to use in benchmarking the mru nonce - // code. - b.StopTimer() - numNonces := 100000 - nonces := make([]uint64, 0, numNonces) - for i := 0; i < numNonces; i++ { - nonces = append(nonces, uint64(i)) - } - b.StartTimer() - - // Benchmark the add plus evicition code. - limit := 20000 - mruNonceMap := newMruNonceMap(uint(limit)) - for i := 0; i < b.N; i++ { - mruNonceMap.Add(nonces[i%numNonces]) - } -} diff --git a/peer/peer.go b/peer/peer.go index 200acb32af..82010f3b84 100644 --- a/peer/peer.go +++ b/peer/peer.go @@ -24,6 +24,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/btcsuite/go-socks/socks" "github.com/davecgh/go-spew/spew" + "github.com/decred/dcrd/lru" ) const ( @@ -82,7 +83,7 @@ var ( // sentNonces houses the unique nonces that are generated when pushing // version messages that are used to detect self connections. - sentNonces = newMruNonceMap(50) + sentNonces = lru.NewCache(50) // allowSelfConns is only used to allow the tests to bypass the self // connection detecting and disconnect logic since they intentionally @@ -450,7 +451,7 @@ type Peer struct { wireEncoding wire.MessageEncoding - knownInventory *mruInventoryMap + knownInventory lru.Cache prevGetBlocksMtx sync.Mutex prevGetBlocksBegin *chainhash.Hash prevGetBlocksStop *chainhash.Hash @@ -1626,7 +1627,7 @@ out: // Don't send inventory that became known after // the initial check. - if p.knownInventory.Exists(iv) { + if p.knownInventory.Contains(iv) { continue } @@ -1832,7 +1833,7 @@ func (p *Peer) QueueMessageWithEncoding(msg wire.Message, doneChan chan<- struct func (p *Peer) QueueInventory(invVect *wire.InvVect) { // Don't add the inventory to the send queue if the peer is already // known to have it. - if p.knownInventory.Exists(invVect) { + if p.knownInventory.Contains(invVect) { return } @@ -1891,7 +1892,7 @@ func (p *Peer) readRemoteVersionMsg() error { } // Detect self connections. - if !allowSelfConns && sentNonces.Exists(msg.Nonce) { + if !allowSelfConns && sentNonces.Contains(msg.Nonce) { return errors.New("disconnecting peer connected to self") } @@ -2224,7 +2225,7 @@ func newPeerBase(origCfg *Config, inbound bool) *Peer { p := Peer{ inbound: inbound, wireEncoding: wire.BaseEncoding, - knownInventory: newMruInventoryMap(maxKnownInventory), + knownInventory: lru.NewCache(maxKnownInventory), stallControl: make(chan stallControlMsg, 1), // nonblocking sync outputQueue: make(chan outMsg, outputBufferSize), sendQueue: make(chan outMsg, 1), // nonblocking sync diff --git a/release/README.md b/release/README.md index a5c9681947..ea01c7d0bb 100644 --- a/release/README.md +++ b/release/README.md @@ -6,11 +6,85 @@ binaries are now reproducible, allowing developers to build the binary on distinct machines, and end up with a byte-for-byte identical binary. However, this wasn't _fully_ solved in `go1.13`, as the build system still includes the directory the binary is built into the binary itself. As a result, our scripts -utilize a work around needed until `go1.13.2`. +utilize a work around needed until `go1.13.2`. +Every release should note which Go version was used to build the release, so +that version should be used for verifying the release. ## Building a New Release -### macOS/Linux/Windows (WSL) +### Tagging and pushing a new tag (for maintainers) + +Before running release scripts, a few things need to happen in order to finally +create a release and make sure there are no mistakes in the release process. + +First, make sure that before the tagged commit there are modifications to the +[CHANGES](../CHANGES) file committed. +The CHANGES file should be a changelog that roughly mirrors the release notes. +Generally, the PRs that have been merged since the last release have been +listed in the CHANGES file and categorized. +For example, these changes have had the following format in the past: +``` +Changes in X.YY.Z (Month Day Year): + - Protocol and Network-related changes: + - PR Title One (#PRNUM) + - PR Title Two (#PRNUMTWO) + ... + - RPC changes: + - Crypto changes: + ... + + - Contributors (alphabetical order): + - Contributor A + - Contributor B + - Contributor C + ... +``` + +If the previous tag is, for example, `vA.B.C`, then you can get the list of +contributors (from `vA.B.C` until the current `HEAD`) using the following command: +```bash +git log vA.B.C..HEAD --pretty="%an" | sort | uniq +``` +After committing changes to the CHANGES file, the tagged release commit +should be created. + +The tagged commit should be a commit that bumps version numbers in `version.go` +and `cmd/btcctl/version.go`. +For example (taken from [f3ec130](https://github.com/btcsuite/btcd/commit/f3ec13030e4e828869954472cbc51ac36bee5c1d)): +```diff +diff --git a/cmd/btcctl/version.go b/cmd/btcctl/version.go +index 2195175c71..f65cacef7e 100644 +--- a/cmd/btcctl/version.go ++++ b/cmd/btcctl/version.go +@@ -18,7 +18,7 @@ const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqr + const ( + appMajor uint = 0 + appMinor uint = 20 +- appPatch uint = 0 ++ appPatch uint = 1 + + // appPreRelease MUST only contain characters from semanticAlphabet + // per the semantic versioning spec. +diff --git a/version.go b/version.go +index 92fd60fdd4..fba55b5a37 100644 +--- a/version.go ++++ b/version.go +@@ -18,7 +18,7 @@ const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqr + const ( + appMajor uint = 0 + appMinor uint = 20 +- appPatch uint = 0 ++ appPatch uint = 1 + + // appPreRelease MUST only contain characters from semanticAlphabet + // per the semantic versioning spec. +``` + +Next, this commit should be signed by the maintainer using `git commit -S`. +The commit should be tagged and signed with `git tag -s`, and should be +pushed using `git push origin TAG`. + +### Building a release on macOS/Linux/Windows (WSL) No prior set up is needed on Linux or macOS is required in order to build the release binaries. However, on Windows, the only way to build the release @@ -19,13 +93,52 @@ the release binaries following these steps: 1. `git clone https://github.com/btcsuite/btcd.git` 2. `cd btcd` -3. `./build/release/release.sh # is the name of the next - release/tag` +3. `./release/release.sh # is the name of the next release/tag` This will then create a directory of the form `btcd-` containing archives of the release binaries for each supported operating system and architecture, and a manifest file containing the hash of each archive. +### Pushing a release (for maintainers) + +Now that the directory `btcd-` is created, the manifest file needs to be +signed by a maintainer and the release files need to be published to GitHub. + +Sign the `manifest-.txt` file like so: +```sh +gpg --sign --detach-sig manifest-.txt +``` +This will create a file named `manifest-.txt.sig`, which will must +be included in the release files later. + +#### Note before publishing +Before publishing, go through the reproducible build process that is outlined +in this document with the files created from `release/release.sh`. This includes +verifying commit and tag signatures using `git verify-commit` and git `verify-tag` +respectively. + +Now that we've double-checked everything and have all of the necessary files, +it's time to publish release files on GitHub. +Follow [this documentation](https://docs.github.com/en/github/administering-a-repository/managing-releases-in-a-repository) +to create a release using the GitHub UI, and make sure to write release notes +which roughly follow the format of [previous release notes](https://github.com/btcsuite/btcd/releases/tag/v0.20.1-beta). +This is different from the [CHANGES](../CHANGES) file, which should be before the +tagged commit in the git history. +Much of the information in the release notes will be the same as the CHANGES +file. +It's important to include the Go version used to produce the release files in +the release notes, so users know the correct version of Go to use to reproduce +and verify the build. +When following the GitHub documentation, include every file in the `btcd-` +directory. + +At this point, a signed commit and tag on that commit should be pushed to the main +branch. The directory created from running `release/release.sh` should be included +as release files in the GitHub release UI, and the `manifest-.txt` file +signature, called `manifest-.txt.sig`, should also be included. +A release notes document should be created and written in the GitHub release UI. +Once all of this is done, feel free to click `Publish Release`! + ## Verifying a Release With `go1.13`, it's now possible for third parties to verify release binaries. @@ -64,7 +177,7 @@ and `go` (matching the same version used in the release): release with `git checkout `. 7. Proceed to verify the tag with `git verify-tag ` and compile the binaries from source for the intended operating system and architecture with - `BTCDBUILDSYS=OS-ARCH ./build/release/release.sh `. + `BTCDBUILDSYS=OS-ARCH ./release/release.sh `. 8. Extract the archive found in the `btcd-` directory created by the release script and recompute the `SHA256` hash of the release binaries (btcd and btcctl) with `shasum -a 256 `. These should match __exactly__ diff --git a/release/notes.sample b/release/notes.sample deleted file mode 100644 index 86a79a61f5..0000000000 --- a/release/notes.sample +++ /dev/null @@ -1,6 +0,0 @@ -- Each release note is preceded by a dash -- Each release note must not exceed 74 characters per line -- Release notes that require a longer explanation than will fit on a - single line should be wrapped with the text indented as in this line -- No periods at the end of each release note -- Other minor cleanup and bug fixes diff --git a/release/prep_release.sh b/release/prep_release.sh deleted file mode 100644 index c64824a644..0000000000 --- a/release/prep_release.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2013 Conformal Systems LLC -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -# -# -# Prepares for a release: -# - Bumps version according to specified level (major, minor, or patch) -# - Updates all version files and package control files with new version -# - Performs some basic validation on specified release notes -# - Updates project changes file with release notes -# - -PROJECT=btcd -PROJECT_UC=$(echo $PROJECT | tr '[:lower:]' '[:upper:]') -SCRIPT=$(basename $0) -VERFILE=../version.go -VERFILES="$VERFILE ../cmd/btcctl/version.go" -PROJ_CHANGES=../CHANGES - -# verify params -if [ $# -lt 2 ]; then - echo "usage: $SCRIPT {major | minor | patch} release-notes-file" - exit 1 -fi - -CUR_DIR=$(pwd) -cd "$(dirname $0)" - -# verify version files exist -for verfile in $VERFILES; do - if [ ! -f "$verfile" ]; then - echo "$SCRIPT: error: $verfile does not exist" 1>&2 - exit 1 - fi -done - -# verify changes file exists -if [ ! -f "$PROJ_CHANGES" ]; then - echo "$SCRIPT: error: $PROJ_CHANGES does not exist" 1>&2 - exit 1 -fi - -RTYPE="$1" -RELEASE_NOTES="$2" -if [ $(echo $RELEASE_NOTES | cut -c1) != "/" ]; then - RELEASE_NOTES="$CUR_DIR/$RELEASE_NOTES" -fi - -# verify valid release type -if [ "$RTYPE" != "major" -a "$RTYPE" != "minor" -a "$RTYPE" != "patch" ]; then - echo "$SCRIPT: error: release type must be major, minor, or patch" - exit 1 -fi - -# verify release notes -if [ ! -e "$RELEASE_NOTES" ]; then - echo "$SCRIPT: error: specified release notes file does not exist" - exit 1 -fi - -if [ ! -s "$RELEASE_NOTES" ]; then - echo "$SCRIPT: error: specified release notes file is empty" - exit 1 -fi - -# verify release notes format -while IFS='' read line; do - if [ -z "$line" ]; then - echo "$SCRIPT: error: release notes must not have blank lines" - exit 1 - fi - if [ ${#line} -gt 74 ]; then - echo -n "$SCRIPT: error: release notes must not contain lines " - echo "with more than 74 characters" - exit 1 - fi - if expr "$line" : ".*\.$" >/dev/null 2>&1 ; then - echo -n "$SCRIPT: error: release notes must not contain lines " - echo "that end in a period" - exit 1 - fi - if ! expr "$line" : "\-" >/dev/null 2>&1; then - if ! expr "$line" : " " >/dev/null 2>&1; then - echo -n "$SCRIPT: error: release notes must not contain lines " - echo "that do not begin with a dash and are not indented" - exit 1 - fi - fi -done <"$RELEASE_NOTES" - -# verify git is available -if ! type git >/dev/null 2>&1; then - echo -n "$SCRIPT: error: Unable to find 'git' in the system path." - exit 1 -fi - -# verify the git repository is on the master branch -BRANCH=$(git branch | grep '\*' | cut -c3-) -if [ "$BRANCH" != "master" ]; then - echo "$SCRIPT: error: git repository must be on the master branch." - exit 1 -fi - -# verify there are no uncommitted modifications prior to release modifications -NUM_MODIFIED=$(git diff 2>/dev/null | wc -l | sed 's/^[ \t]*//') -NUM_STAGED=$(git diff --cached 2>/dev/null | wc -l | sed 's/^[ \t]*//') -if [ "$NUM_MODIFIED" != "0" -o "$NUM_STAGED" != "0" ]; then - echo -n "$SCRIPT: error: the working directory contains uncommitted " - echo "modifications" - exit 1 -fi - -# get version -PAT_PREFIX="(^[[:space:]]+app" -PAT_SUFFIX='[[:space:]]+uint[[:space:]]+=[[:space:]]+)[0-9]+$' -MAJOR=$(egrep "${PAT_PREFIX}Major${PAT_SUFFIX}" $VERFILE | awk '{print $4}') -MINOR=$(egrep "${PAT_PREFIX}Minor${PAT_SUFFIX}" $VERFILE | awk '{print $4}') -PATCH=$(egrep "${PAT_PREFIX}Patch${PAT_SUFFIX}" $VERFILE | awk '{print $4}') -if [ -z "$MAJOR" -o -z "$MINOR" -o -z "$PATCH" ]; then - echo "$SCRIPT: error: unable to get version from $VERFILE" 1>&2 - exit 1 -fi - -# bump version according to level -if [ "$RTYPE" = "major" ]; then - MAJOR=$(expr $MAJOR + 1) - MINOR=0 - PATCH=0 -elif [ "$RTYPE" = "minor" ]; then - MINOR=$(expr $MINOR + 1) - PATCH=0 -elif [ "$RTYPE" = "patch" ]; then - PATCH=$(expr $PATCH + 1) -fi -PROJ_VER="$MAJOR.$MINOR.$PATCH" - -# update project changes with release notes -DATE=$(date "+%a %b %d %Y") -awk -v D="$DATE" -v VER="$PROJ_VER" ' -/=======/ && first_line==0 { - first_line=1 - print $0 - next -} -/=======/ && first_line==1 { - print $0 - print "" - print "Changes in "VER" ("D")" - exit -} -{ print $0 } -' <"$PROJ_CHANGES" >"${PROJ_CHANGES}.tmp" -cat "$RELEASE_NOTES" | sed 's/^/ /' >>"${PROJ_CHANGES}.tmp" -awk ' -/=======/ && first_line==0 { - first_line=1 - next -} -/=======/ && first_line==1 { - second_line=1 - next -} -second_line==1 { print $0 } -' <"$PROJ_CHANGES" >>"${PROJ_CHANGES}.tmp" - -# update version filef with new version -for verfile in $VERFILES; do - sed -E " - s/${PAT_PREFIX}Major${PAT_SUFFIX}/\1${MAJOR}/; - s/${PAT_PREFIX}Minor${PAT_SUFFIX}/\1${MINOR}/; - s/${PAT_PREFIX}Patch${PAT_SUFFIX}/\1${PATCH}/; - " <"$verfile" >"${verfile}.tmp" -done - - -# Apply changes -mv "${PROJ_CHANGES}.tmp" "$PROJ_CHANGES" -for verfile in $VERFILES; do - mv "${verfile}.tmp" "$verfile" -done - -echo "All files have been prepared for release." -echo "Use the following commands to review the changes for accuracy:" -echo " git status" -echo " git diff" -echo "" -echo "If everything is accurate, use the following commands to commit, tag," -echo "and push the changes" -echo " git commit -am \"Prepare for release ${PROJ_VER}.\"" -echo -n " git tag -a \"${PROJECT_UC}_${MAJOR}_${MINOR}_${PATCH}\" -m " -echo "\"Release ${PROJ_VER}\"" -echo " git push" -echo " git push --tags" diff --git a/release/release.sh b/release/release.sh index 7b0885dac6..cc65fb6cf1 100755 --- a/release/release.sh +++ b/release/release.sh @@ -1,11 +1,12 @@ #!/bin/bash +# Copyright (c) 2016 Company 0, LLC. +# Copyright (c) 2016-2020 The btcsuite developers +# Use of this source code is governed by an ISC +# license that can be found in the LICENSE file. + # Simple bash script to build basic btcd tools for all the platforms we support # with the golang cross-compiler. -# -# Copyright (c) 2016 Company 0, LLC. -# Use of this source code is governed by the ISC -# license. set -e diff --git a/rpcclient/chain.go b/rpcclient/chain.go index 707978ca65..9f0c6c684d 100644 --- a/rpcclient/chain.go +++ b/rpcclient/chain.go @@ -52,14 +52,61 @@ func (c *Client) GetBestBlockHash() (*chainhash.Hash, error) { return c.GetBestBlockHashAsync().Receive() } +// legacyGetBlockRequest constructs and sends a legacy getblock request which +// contains two separate bools to denote verbosity, in contract to a single int +// parameter. +func (c *Client) legacyGetBlockRequest(hash string, verbose, + verboseTx bool) ([]byte, error) { + + hashJSON, err := json.Marshal(hash) + if err != nil { + return nil, err + } + verboseJSON, err := json.Marshal(btcjson.Bool(verbose)) + if err != nil { + return nil, err + } + verboseTxJSON, err := json.Marshal(btcjson.Bool(verboseTx)) + if err != nil { + return nil, err + } + return c.RawRequest("getblock", []json.RawMessage{ + hashJSON, verboseJSON, verboseTxJSON, + }) +} + +// waitForGetBlockRes waits for the response of a getblock request. If the +// response indicates an invalid parameter was provided, a legacy style of the +// request is resent and its response is returned instead. +func (c *Client) waitForGetBlockRes(respChan chan *response, hash string, + verbose, verboseTx bool) ([]byte, error) { + + res, err := receiveFuture(respChan) + + // If we receive an invalid parameter error, then we may be + // communicating with a btcd node which only understands the legacy + // request, so we'll try that. + if err, ok := err.(*btcjson.RPCError); ok && + err.Code == btcjson.ErrRPCInvalidParams.Code { + return c.legacyGetBlockRequest(hash, verbose, verboseTx) + } + + // Otherwise, we can return the response as is. + return res, err +} + // FutureGetBlockResult is a future promise to deliver the result of a // GetBlockAsync RPC invocation (or an applicable error). -type FutureGetBlockResult chan *response +type FutureGetBlockResult struct { + client *Client + hash string + Response chan *response +} // Receive waits for the response promised by the future and returns the raw // block requested from the server given its hash. func (r FutureGetBlockResult) Receive() (*wire.MsgBlock, error) { - res, err := receiveFuture(r) + res, err := r.client.waitForGetBlockRes(r.Response, r.hash, false, false) if err != nil { return nil, err } @@ -97,8 +144,12 @@ func (c *Client) GetBlockAsync(blockHash *chainhash.Hash) FutureGetBlockResult { hash = blockHash.String() } - cmd := btcjson.NewGetBlockCmd(hash, nil) - return c.sendCmd(cmd) + cmd := btcjson.NewGetBlockCmd(hash, btcjson.Int(0)) + return FutureGetBlockResult{ + client: c, + hash: hash, + Response: c.sendCmd(cmd), + } } // GetBlock returns a raw block from the server given its hash. @@ -111,12 +162,16 @@ func (c *Client) GetBlock(blockHash *chainhash.Hash) (*wire.MsgBlock, error) { // FutureGetBlockVerboseResult is a future promise to deliver the result of a // GetBlockVerboseAsync RPC invocation (or an applicable error). -type FutureGetBlockVerboseResult chan *response +type FutureGetBlockVerboseResult struct { + client *Client + hash string + Response chan *response +} // Receive waits for the response promised by the future and returns the data // structure from the server with information about the requested block. func (r FutureGetBlockVerboseResult) Receive() (*btcjson.GetBlockVerboseResult, error) { - res, err := receiveFuture(r) + res, err := r.client.waitForGetBlockRes(r.Response, r.hash, true, false) if err != nil { return nil, err } @@ -143,7 +198,11 @@ func (c *Client) GetBlockVerboseAsync(blockHash *chainhash.Hash) FutureGetBlockV // From the bitcoin-cli getblock documentation: // "If verbosity is 1, returns an Object with information about block ." cmd := btcjson.NewGetBlockCmd(hash, btcjson.Int(1)) - return c.sendCmd(cmd) + return FutureGetBlockVerboseResult{ + client: c, + hash: hash, + Response: c.sendCmd(cmd), + } } // GetBlockVerbose returns a data structure from the server with information @@ -155,10 +214,18 @@ func (c *Client) GetBlockVerbose(blockHash *chainhash.Hash) (*btcjson.GetBlockVe return c.GetBlockVerboseAsync(blockHash).Receive() } -type FutureGetBlockVerboseTxResult chan *response +// FutureGetBlockVerboseTxResult is a future promise to deliver the result of a +// GetBlockVerboseTxResult RPC invocation (or an applicable error). +type FutureGetBlockVerboseTxResult struct { + client *Client + hash string + Response chan *response +} +// Receive waits for the response promised by the future and returns a verbose +// version of the block including detailed information about its transactions. func (r FutureGetBlockVerboseTxResult) Receive() (*btcjson.GetBlockVerboseTxResult, error) { - res, err := receiveFuture(r) + res, err := r.client.waitForGetBlockRes(r.Response, r.hash, true, true) if err != nil { return nil, err } @@ -182,11 +249,17 @@ func (c *Client) GetBlockVerboseTxAsync(blockHash *chainhash.Hash) FutureGetBloc if blockHash != nil { hash = blockHash.String() } + // From the bitcoin-cli getblock documentation: - // "If verbosity is 2, returns an Object with information about block and information about each transaction." + // + // If verbosity is 2, returns an Object with information about block + // and information about each transaction. cmd := btcjson.NewGetBlockCmd(hash, btcjson.Int(2)) - - return c.sendCmd(cmd) + return FutureGetBlockVerboseTxResult{ + client: c, + hash: hash, + Response: c.sendCmd(cmd), + } } // GetBlockVerboseTx returns a data structure from the server with information @@ -234,6 +307,79 @@ func (c *Client) GetBlockCount() (int64, error) { return c.GetBlockCountAsync().Receive() } +// FutureGetChainTxStatsResult is a future promise to deliver the result of a +// GetChainTxStatsAsync RPC invocation (or an applicable error). +type FutureGetChainTxStatsResult chan *response + +// Receive waits for the response promised by the future and returns transaction statistics +func (r FutureGetChainTxStatsResult) Receive() (*btcjson.GetChainTxStatsResult, error) { + res, err := receiveFuture(r) + if err != nil { + return nil, err + } + + var chainTxStats btcjson.GetChainTxStatsResult + err = json.Unmarshal(res, &chainTxStats) + if err != nil { + return nil, err + } + + return &chainTxStats, nil +} + +// GetChainTxStatsAsync returns an instance of a type that can be used to get +// the result of the RPC at some future time by invoking the Receive function on +// the returned instance. +// +// See GetChainTxStats for the blocking version and more details. +func (c *Client) GetChainTxStatsAsync() FutureGetChainTxStatsResult { + cmd := btcjson.NewGetChainTxStatsCmd(nil, nil) + return c.sendCmd(cmd) +} + +// GetChainTxStatsNBlocksAsync returns an instance of a type that can be used to get +// the result of the RPC at some future time by invoking the Receive function on +// the returned instance. +// +// See GetChainTxStatsNBlocks for the blocking version and more details. +func (c *Client) GetChainTxStatsNBlocksAsync(nBlocks int32) FutureGetChainTxStatsResult { + cmd := btcjson.NewGetChainTxStatsCmd(&nBlocks, nil) + return c.sendCmd(cmd) +} + +// GetChainTxStatsNBlocksBlockHashAsync returns an instance of a type that can be used to get +// the result of the RPC at some future time by invoking the Receive function on +// the returned instance. +// +// See GetChainTxStatsNBlocksBlockHash for the blocking version and more details. +func (c *Client) GetChainTxStatsNBlocksBlockHashAsync(nBlocks int32, blockHash chainhash.Hash) FutureGetChainTxStatsResult { + hash := blockHash.String() + cmd := btcjson.NewGetChainTxStatsCmd(&nBlocks, &hash) + return c.sendCmd(cmd) +} + +// GetChainTxStats returns statistics about the total number and rate of transactions in the chain. +// +// Size of the window is one month and it ends at chain tip. +func (c *Client) GetChainTxStats() (*btcjson.GetChainTxStatsResult, error) { + return c.GetChainTxStatsAsync().Receive() +} + +// GetChainTxStatsNBlocks returns statistics about the total number and rate of transactions in the chain. +// +// The argument specifies size of the window in number of blocks. The window ends at chain tip. +func (c *Client) GetChainTxStatsNBlocks(nBlocks int32) (*btcjson.GetChainTxStatsResult, error) { + return c.GetChainTxStatsNBlocksAsync(nBlocks).Receive() +} + +// GetChainTxStatsNBlocksBlockHash returns statistics about the total number and rate of transactions in the chain. +// +// First argument specifies size of the window in number of blocks. +// Second argument is the hash of the block that ends the window. +func (c *Client) GetChainTxStatsNBlocksBlockHash(nBlocks int32, blockHash chainhash.Hash) (*btcjson.GetChainTxStatsResult, error) { + return c.GetChainTxStatsNBlocksBlockHashAsync(nBlocks, blockHash).Receive() +} + // FutureGetDifficultyResult is a future promise to deliver the result of a // GetDifficultyAsync RPC invocation (or an applicable error). type FutureGetDifficultyResult chan *response diff --git a/rpcclient/cookiefile.go b/rpcclient/cookiefile.go new file mode 100644 index 0000000000..c3f7068b30 --- /dev/null +++ b/rpcclient/cookiefile.go @@ -0,0 +1,38 @@ +// Copyright (c) 2017 The Namecoin developers +// Copyright (c) 2019 The btcsuite developers +// Use of this source code is governed by an ISC +// license that can be found in the LICENSE file. + +package rpcclient + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +func readCookieFile(path string) (username, password string, err error) { + f, err := os.Open(path) + if err != nil { + return + } + defer f.Close() + + scanner := bufio.NewScanner(f) + scanner.Scan() + err = scanner.Err() + if err != nil { + return + } + s := scanner.Text() + + parts := strings.SplitN(s, ":", 2) + if len(parts) != 2 { + err = fmt.Errorf("malformed cookie file") + return + } + + username, password = parts[0], parts[1] + return +} diff --git a/rpcclient/infrastructure.go b/rpcclient/infrastructure.go index a2079886d3..8609e7c5ad 100644 --- a/rpcclient/infrastructure.go +++ b/rpcclient/infrastructure.go @@ -19,6 +19,7 @@ import ( "net" "net/http" "net/url" + "os" "strings" "sync" "sync/atomic" @@ -851,7 +852,12 @@ func (c *Client) sendPost(jReq *jsonRequest) { httpReq.Header.Set("Content-Type", "application/json") // Configure basic access authorization. - httpReq.SetBasicAuth(c.config.User, c.config.Pass) + user, pass, err := c.config.getAuth() + if err != nil { + jReq.responseChan <- &response{result: nil, err: err} + return + } + httpReq.SetBasicAuth(user, pass) log.Tracef("Sending command [%s] with id %d", jReq.method, jReq.id) c.sendPostRequest(httpReq, jReq) @@ -1096,6 +1102,17 @@ type ConnConfig struct { // Pass is the passphrase to use to authenticate to the RPC server. Pass string + // CookiePath is the path to a cookie file containing the username and + // passphrase to use to authenticate to the RPC server. It is used + // instead of User and Pass if non-empty. + CookiePath string + + cookieLastCheckTime time.Time + cookieLastModTime time.Time + cookieLastUser string + cookieLastPass string + cookieLastErr error + // Params is the string representing the network that the server // is running. If there is no parameter set in the config, then // mainnet will be used by default. @@ -1149,6 +1166,43 @@ type ConnConfig struct { EnableBCInfoHacks bool } +// getAuth returns the username and passphrase that will actually be used for +// this connection. This will be the result of checking the cookie if a cookie +// path is configured; if not, it will be the user-configured username and +// passphrase. +func (config *ConnConfig) getAuth() (username, passphrase string, err error) { + // Try username+passphrase auth first. + if config.Pass != "" { + return config.User, config.Pass, nil + } + + // If no username or passphrase is set, try cookie auth. + return config.retrieveCookie() +} + +// retrieveCookie returns the cookie username and passphrase. +func (config *ConnConfig) retrieveCookie() (username, passphrase string, err error) { + if !config.cookieLastCheckTime.IsZero() && time.Now().Before(config.cookieLastCheckTime.Add(30*time.Second)) { + return config.cookieLastUser, config.cookieLastPass, config.cookieLastErr + } + + config.cookieLastCheckTime = time.Now() + + st, err := os.Stat(config.CookiePath) + if err != nil { + config.cookieLastErr = err + return config.cookieLastUser, config.cookieLastPass, config.cookieLastErr + } + + modTime := st.ModTime() + if !modTime.Equal(config.cookieLastModTime) { + config.cookieLastModTime = modTime + config.cookieLastUser, config.cookieLastPass, config.cookieLastErr = readCookieFile(config.CookiePath) + } + + return config.cookieLastUser, config.cookieLastPass, config.cookieLastErr +} + // newHTTPClient returns a new http client that is configured according to the // proxy and TLS settings in the associated connection configuration. func newHTTPClient(config *ConnConfig) (*http.Client, error) { @@ -1218,7 +1272,11 @@ func dial(config *ConnConfig) (*websocket.Conn, error) { // The RPC server requires basic authorization, so create a custom // request header with the Authorization header set. - login := config.User + ":" + config.Pass + user, pass, err := config.getAuth() + if err != nil { + return nil, err + } + login := user + ":" + pass auth := "Basic " + base64.StdEncoding.EncodeToString([]byte(login)) requestHeader := make(http.Header) requestHeader.Add("Authorization", auth) diff --git a/rpcclient/rawtransactions.go b/rpcclient/rawtransactions.go index 23754d961d..4e8d4e4d9c 100644 --- a/rpcclient/rawtransactions.go +++ b/rpcclient/rawtransactions.go @@ -205,6 +205,47 @@ func (c *Client) DecodeRawTransaction(serializedTx []byte) (*btcjson.TxRawResult return c.DecodeRawTransactionAsync(serializedTx).Receive() } +// FutureFundRawTransactionResult is a future promise to deliver the result +// of a FutureFundRawTransactionAsync RPC invocation (or an applicable error). +type FutureFundRawTransactionResult chan *response + +// Receive waits for the response promised by the future and returns information +// about a funding attempt +func (r FutureFundRawTransactionResult) Receive() (*btcjson.FundRawTransactionResult, error) { + res, err := receiveFuture(r) + if err != nil { + return nil, err + } + + var marshalled btcjson.FundRawTransactionResult + if err := json.Unmarshal(res, &marshalled); err != nil { + return nil, err + } + + return &marshalled, nil +} + +// FundRawTransactionAsync returns an instance of a type that can be used to +// get the result of the RPC at some future time by invoking the Receive +// function on the returned instance. +// +// See FundRawTransaction for the blocking version and more details. +func (c *Client) FundRawTransactionAsync(tx *wire.MsgTx, opts btcjson.FundRawTransactionOpts, isWitness *bool) FutureFundRawTransactionResult { + var txBuf bytes.Buffer + if err := tx.Serialize(&txBuf); err != nil { + return newFutureError(err) + } + + cmd := btcjson.NewFundRawTransactionCmd(txBuf.Bytes(), opts, isWitness) + return c.sendCmd(cmd) +} + +// FundRawTransaction returns the result of trying to fund the given transaction with +// funds from the node wallet +func (c *Client) FundRawTransaction(tx *wire.MsgTx, opts btcjson.FundRawTransactionOpts, isWitness *bool) (*btcjson.FundRawTransactionResult, error) { + return c.FundRawTransactionAsync(tx, opts, isWitness).Receive() +} + // FutureCreateRawTransactionResult is a future promise to deliver the result // of a CreateRawTransactionAsync RPC invocation (or an applicable error). type FutureCreateRawTransactionResult chan *response @@ -261,7 +302,8 @@ func (c *Client) CreateRawTransactionAsync(inputs []btcjson.TransactionInput, } // CreateRawTransaction returns a new transaction spending the provided inputs -// and sending to the provided addresses. +// and sending to the provided addresses. If the inputs are either nil or an +// empty slice, it is interpreted as an empty slice. func (c *Client) CreateRawTransaction(inputs []btcjson.TransactionInput, amounts map[btcutil.Address]btcutil.Amount, lockTime *int64) (*wire.MsgTx, error) { diff --git a/rpcclient/wallet.go b/rpcclient/wallet.go index d43be26181..37bf9471e0 100644 --- a/rpcclient/wallet.go +++ b/rpcclient/wallet.go @@ -20,7 +20,8 @@ import ( // ***************************** // FutureGetTransactionResult is a future promise to deliver the result -// of a GetTransactionAsync RPC invocation (or an applicable error). +// of a GetTransactionAsync or GetTransactionWatchOnlyAsync RPC invocation +// (or an applicable error). type FutureGetTransactionResult chan *response // Receive waits for the response promised by the future and returns detailed @@ -63,6 +64,28 @@ func (c *Client) GetTransaction(txHash *chainhash.Hash) (*btcjson.GetTransaction return c.GetTransactionAsync(txHash).Receive() } +// GetTransactionWatchOnlyAsync returns an instance of a type that can be used +// to get the result of the RPC at some future time by invoking the Receive function on +// the returned instance. +// +// See GetTransactionWatchOnly for the blocking version and more details. +func (c *Client) GetTransactionWatchOnlyAsync(txHash *chainhash.Hash, watchOnly bool) FutureGetTransactionResult { + hash := "" + if txHash != nil { + hash = txHash.String() + } + + cmd := btcjson.NewGetTransactionCmd(hash, &watchOnly) + return c.sendCmd(cmd) +} + +// GetTransactionWatchOnly returns detailed information about a wallet +// transaction, and allow including watch-only addresses in balance +// calculation and details. +func (c *Client) GetTransactionWatchOnly(txHash *chainhash.Hash, watchOnly bool) (*btcjson.GetTransactionResult, error) { + return c.GetTransactionWatchOnlyAsync(txHash, watchOnly).Receive() +} + // FutureListTransactionsResult is a future promise to deliver the result of a // ListTransactionsAsync, ListTransactionsCountAsync, or // ListTransactionsCountFromAsync RPC invocation (or an applicable error). @@ -1547,6 +1570,43 @@ func (c *Client) GetBalanceMinConf(account string, minConfirms int) (btcutil.Amo return c.GetBalanceMinConfAsync(account, minConfirms).Receive() } +// FutureGetBalancesResult is a future promise to deliver the result of a +// GetBalancesAsync RPC invocation (or an applicable error). +type FutureGetBalancesResult chan *response + +// Receive waits for the response promised by the future and returns the +// available balances from the server. +func (r FutureGetBalancesResult) Receive() (*btcjson.GetBalancesResult, error) { + res, err := receiveFuture(r) + if err != nil { + return nil, err + } + + // Unmarshal result as a floating point number. + var balances btcjson.GetBalancesResult + err = json.Unmarshal(res, &balances) + if err != nil { + return nil, err + } + + return &balances, nil +} + +// GetBalancesAsync returns an instance of a type that can be used to get the +// result of the RPC at some future time by invoking the Receive function on the +// returned instance. +// +// See GetBalances for the blocking version and more details. +func (c *Client) GetBalancesAsync() FutureGetBalancesResult { + cmd := btcjson.NewGetBalancesCmd() + return c.sendCmd(cmd) +} + +// GetBalances returns the available balances from the server. +func (c *Client) GetBalances() (*btcjson.GetBalancesResult, error) { + return c.GetBalancesAsync().Receive() +} + // FutureGetReceivedByAccountResult is a future promise to deliver the result of // a GetReceivedByAccountAsync or GetReceivedByAccountMinConfAsync RPC // invocation (or an applicable error). diff --git a/rpcserver.go b/rpcserver.go index 2da7e9f57e..89817db856 100644 --- a/rpcserver.go +++ b/rpcserver.go @@ -1081,14 +1081,12 @@ func handleGetBlock(s *rpcServer, cmd interface{}, closeChan <-chan struct{}) (i Message: "Block not found", } } - - // When the verbose flag isn't set, simply return the serialized block - // as a hex-encoded string. + // If verbosity is 0, return the serialized block as a hex encoded string. if c.Verbosity != nil && *c.Verbosity == 0 { return hex.EncodeToString(blkBytes), nil } - // The verbose flag is set, so generate the JSON object and return it. + // Otherwise, generate the JSON object and return it. // Deserialize the block. blk, err := btcutil.NewBlockFromBytes(blkBytes) diff --git a/rpcserverhelp.go b/rpcserverhelp.go index cee2407566..cc3b33f3ce 100644 --- a/rpcserverhelp.go +++ b/rpcserverhelp.go @@ -243,8 +243,8 @@ var helpDescsEnUS = map[string]string{ "getblockverboseresult-version": "The block version", "getblockverboseresult-versionHex": "The block version in hexadecimal", "getblockverboseresult-merkleroot": "Root hash of the merkle tree", - "getblockverboseresult-tx": "The transaction hashes (only when verbosetx=false)", - "getblockverboseresult-rawtx": "The transactions as JSON objects (only when verbosetx=true)", + "getblockverboseresult-tx": "The transaction hashes (only when verbosity=1)", + "getblockverboseresult-rawtx": "The transactions as JSON objects (only when verbosity=2)", "getblockverboseresult-time": "The block time in seconds since 1 Jan 1970 GMT", "getblockverboseresult-nonce": "The block nonce", "getblockverboseresult-bits": "The bits which represent the block difficulty", @@ -818,7 +818,7 @@ func (c *helpCacher) rpcUsage(includeWebsockets bool) (string, error) { } } - sort.Sort(sort.StringSlice(usageTexts)) + sort.Strings(usageTexts) c.usage = strings.Join(usageTexts, "\n") return c.usage, nil } diff --git a/server.go b/server.go index b9bb18d0f5..c9f23fa638 100644 --- a/server.go +++ b/server.go @@ -364,14 +364,14 @@ func (sp *serverPeer) pushAddrMsg(addresses []*wire.NetAddress) { // threshold, a warning is logged including the reason provided. Further, if // the score is above the ban threshold, the peer will be banned and // disconnected. -func (sp *serverPeer) addBanScore(persistent, transient uint32, reason string) { +func (sp *serverPeer) addBanScore(persistent, transient uint32, reason string) bool { // No warning is logged and no score is calculated if banning is disabled. if cfg.DisableBanning { - return + return false } if sp.isWhitelisted { peerLog.Debugf("Misbehaving whitelisted peer %s: %s", sp, reason) - return + return false } warnThreshold := cfg.BanThreshold >> 1 @@ -383,7 +383,7 @@ func (sp *serverPeer) addBanScore(persistent, transient uint32, reason string) { peerLog.Warnf("Misbehaving peer %s: %s -- ban score is %d, "+ "it was not increased this time", sp, reason, score) } - return + return false } score := sp.banScore.Increase(persistent, transient) if score > warnThreshold { @@ -394,8 +394,10 @@ func (sp *serverPeer) addBanScore(persistent, transient uint32, reason string) { sp) sp.server.BanPeer(sp) sp.Disconnect() + return true } } + return false } // hasServices returns whether or not the provided advertised service flags have @@ -498,7 +500,9 @@ func (sp *serverPeer) OnMemPool(_ *peer.Peer, msg *wire.MsgMemPool) { // The ban score accumulates and passes the ban threshold if a burst of // mempool messages comes from a peer. The score decays each minute to // half of its value. - sp.addBanScore(0, 33, "mempool") + if sp.addBanScore(0, 33, "mempool") { + return + } // Generate inventory message with the available transactions in the // transaction memory pool. Limit it to the max allowed inventory @@ -638,7 +642,9 @@ func (sp *serverPeer) OnGetData(_ *peer.Peer, msg *wire.MsgGetData) { // bursts of small requests are not penalized as that would potentially ban // peers performing IBD. // This incremental score decays each minute to half of its value. - sp.addBanScore(0, uint32(length)*99/wire.MaxInvPerMsg, "getdata") + if sp.addBanScore(0, uint32(length)*99/wire.MaxInvPerMsg, "getdata") { + return + } // We wait on this wait channel periodically to prevent queuing // far more data than we can send in a reasonable time, wasting memory. @@ -1304,6 +1310,44 @@ func (sp *serverPeer) OnWrite(_ *peer.Peer, bytesWritten int, msg wire.Message, sp.server.AddBytesSent(uint64(bytesWritten)) } +// OnNotFound is invoked when a peer sends a notfound message. +func (sp *serverPeer) OnNotFound(p *peer.Peer, msg *wire.MsgNotFound) { + if !sp.Connected() { + return + } + + var numBlocks, numTxns uint32 + for _, inv := range msg.InvList { + switch inv.Type { + case wire.InvTypeBlock: + numBlocks++ + case wire.InvTypeTx: + numTxns++ + default: + peerLog.Debugf("Invalid inv type '%d' in notfound message from %s", + inv.Type, sp) + sp.Disconnect() + return + } + } + if numBlocks > 0 { + blockStr := pickNoun(uint64(numBlocks), "block", "blocks") + reason := fmt.Sprintf("%d %v not found", numBlocks, blockStr) + if sp.addBanScore(20*numBlocks, 0, reason) { + return + } + } + if numTxns > 0 { + txStr := pickNoun(uint64(numTxns), "transaction", "transactions") + reason := fmt.Sprintf("%d %v not found", numBlocks, txStr) + if sp.addBanScore(0, 10*numTxns, reason) { + return + } + } + + sp.server.syncManager.QueueNotFound(msg, p) +} + // randomUint16Number returns a random uint16 in a specified input range. Note // that the range is in zeroth ordering; if you pass it 1800, you will get // values from 0 to 1800. @@ -1998,6 +2042,7 @@ func newPeerConfig(sp *serverPeer) *peer.Config { OnAddr: sp.OnAddr, OnRead: sp.OnRead, OnWrite: sp.OnWrite, + OnNotFound: sp.OnNotFound, // Note: The reference client currently bans peers that send alerts // not signed with its key. We could verify against their key, but @@ -2270,9 +2315,7 @@ out: // When an InvVect has been added to a block, we can // now remove it, if it was present. case broadcastInventoryDel: - if _, ok := pendingInvs[*msg]; ok { - delete(pendingInvs, *msg) - } + delete(pendingInvs, *msg) } case <-timer.C: diff --git a/service_windows.go b/service_windows.go index 8101ae6514..378c9204f8 100644 --- a/service_windows.go +++ b/service_windows.go @@ -275,6 +275,22 @@ func performServiceCommand(command string) error { // returned to the caller so the application can determine whether to exit (when // running as a service) or launch in normal interactive mode. func serviceMain() (bool, error) { + // Don't run as a service if the user explicitly requested it. This is + // needed to run btcd on Windows in CI environments like Travis. + // We can't use the config struct to access the value because that's not + // parsed yet. But we add the flag to the struct anyway so the parser + // won't complain about it later. + noService := false + for _, arg := range os.Args { + if arg == "--nowinservice" { + noService = true + break + } + } + if noService { + return false, nil + } + // Don't run as a service if we're running interactively (or that can't // be determined due to an error). isInteractive, err := svc.IsAnInteractiveSession() diff --git a/txscript/opcode_test.go b/txscript/opcode_test.go index 6e3205a209..1487dde590 100644 --- a/txscript/opcode_test.go +++ b/txscript/opcode_test.go @@ -124,7 +124,7 @@ func TestOpcodeDisasm(t *testing.T) { // OP_UNKNOWN#. case opcodeVal >= 0xba && opcodeVal <= 0xf9 || opcodeVal == 0xfc: - expectedStr = "OP_UNKNOWN" + strconv.Itoa(int(opcodeVal)) + expectedStr = "OP_UNKNOWN" + strconv.Itoa(opcodeVal) } pop := parsedOpcode{opcode: &opcodeArray[opcodeVal], data: data} @@ -190,7 +190,7 @@ func TestOpcodeDisasm(t *testing.T) { // OP_UNKNOWN#. case opcodeVal >= 0xba && opcodeVal <= 0xf9 || opcodeVal == 0xfc: - expectedStr = "OP_UNKNOWN" + strconv.Itoa(int(opcodeVal)) + expectedStr = "OP_UNKNOWN" + strconv.Itoa(opcodeVal) } pop := parsedOpcode{opcode: &opcodeArray[opcodeVal], data: data} diff --git a/txscript/pkscript_test.go b/txscript/pkscript_test.go index 841842c268..49e2db8afa 100644 --- a/txscript/pkscript_test.go +++ b/txscript/pkscript_test.go @@ -337,9 +337,9 @@ func TestComputePkScript(t *testing.T) { name: "P2WSH witness", sigScript: nil, witness: [][]byte{ - []byte{}, + {}, // Witness script. - []byte{ + { 0x21, 0x03, 0x82, 0x62, 0xa6, 0xc6, 0xce, 0xc9, 0x3c, 0x2d, 0x3e, 0xcd, 0x6c, 0x60, 0x72, 0xef, 0xea, 0x86, @@ -367,9 +367,9 @@ func TestComputePkScript(t *testing.T) { witness: [][]byte{ // Signature is not needed to re-derive the // pkScript. - []byte{}, + {}, // Compressed pubkey. - []byte{ + { 0x03, 0x82, 0x62, 0xa6, 0xc6, 0xce, 0xc9, 0x3c, 0x2d, 0x3e, 0xcd, 0x6c, 0x60, 0x72, 0xef, 0xea, 0x86, 0xd0, @@ -398,9 +398,9 @@ func TestComputePkScript(t *testing.T) { witness: [][]byte{ // Signature is not needed to re-derive the // pkScript. - []byte{}, + {}, // Malformed compressed pubkey. - []byte{ + { 0x03, 0x82, 0x62, 0xa6, 0xc6, 0xce, 0xc9, 0x3c, 0x2d, 0x3e, 0xcd, 0x6c, 0x60, 0x72, 0xef, 0xea, 0x86, 0xd0, diff --git a/version.go b/version.go index fba55b5a37..ac294de232 100644 --- a/version.go +++ b/version.go @@ -17,8 +17,8 @@ const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqr // versioning 2.0.0 spec (http://semver.org/). const ( appMajor uint = 0 - appMinor uint = 20 - appPatch uint = 1 + appMinor uint = 21 + appPatch uint = 0 // appPreRelease MUST only contain characters from semanticAlphabet // per the semantic versioning spec. diff --git a/wire/common_test.go b/wire/common_test.go index fa963d4b5d..46e3fa6613 100644 --- a/wire/common_test.go +++ b/wire/common_test.go @@ -118,15 +118,15 @@ func TestElementWire(t *testing.T) { }, }, { - ServiceFlag(SFNodeNetwork), + SFNodeNetwork, []byte{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, }, { - InvType(InvTypeTx), + InvTypeTx, []byte{0x01, 0x00, 0x00, 0x00}, }, { - BitcoinNet(MainNet), + MainNet, []byte{0xf9, 0xbe, 0xb4, 0xd9}, }, // Type not supported by the "fast" path and requires reflection. @@ -211,9 +211,9 @@ func TestElementWireErrors(t *testing.T) { }), 0, io.ErrShortWrite, io.EOF, }, - {ServiceFlag(SFNodeNetwork), 0, io.ErrShortWrite, io.EOF}, - {InvType(InvTypeTx), 0, io.ErrShortWrite, io.EOF}, - {BitcoinNet(MainNet), 0, io.ErrShortWrite, io.EOF}, + {SFNodeNetwork, 0, io.ErrShortWrite, io.EOF}, + {InvTypeTx, 0, io.ErrShortWrite, io.EOF}, + {MainNet, 0, io.ErrShortWrite, io.EOF}, } t.Logf("Running %d tests", len(tests))