Merge remote-tracking branch 'upstream/v0.8.2-beta-branch'

build/version.go

@@ -23,7 +23,7 @@ const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqr
 const (
 	appMajor uint = 0
 	appMinor uint = 8
-	appPatch uint = 0
+	appPatch uint = 2
 
 	// appPreRelease MUST only contain characters from semanticAlphabet
 	// per the semantic versioning spec.

chainntnfs/bitcoindnotify/bitcoind.go

@@ -262,6 +262,14 @@ out:
 						return
 					}
 
+					chainntnfs.Log.Infof("Historical "+
+						"spend dispatch finished"+
+						"for request %v (start=%v "+
+						"end=%v) with details: %v",
+						msg.SpendRequest,
+						msg.StartHeight, msg.EndHeight,
+						spendDetails)
+
 					// If the historical dispatch finished
 					// without error, we will invoke
 					// UpdateSpendDetails even if none were

chainntnfs/interface.go

@@ -241,6 +241,12 @@ type SpendDetail struct {
 	SpendingHeight    int32
 }
 
+// String returns a string representation of SpendDetail.
+func (s *SpendDetail) String() string {
+	return fmt.Sprintf("%v[%d] spending %v at height=%v", s.SpenderTxHash,
+		s.SpenderInputIndex, s.SpentOutPoint, s.SpendingHeight)
+}
+
 // SpendEvent encapsulates a spentness notification. Its only field 'Spend' will
 // be sent upon once the target output passed into RegisterSpendNtfn has been
 // spent on the blockchain.

chainntnfs/txnotifier.go

@@ -325,9 +325,10 @@ func NewSpendRequest(op *wire.OutPoint, pkScript []byte) (SpendRequest, error) {
 // String returns the string representation of the SpendRequest.
 func (r SpendRequest) String() string {
 	if r.OutPoint != ZeroOutPoint {
-		return fmt.Sprintf("outpoint=%v", r.OutPoint)
+		return fmt.Sprintf("outpoint=%v, script=%v", r.OutPoint,
+			r.PkScript)
 	}
-	return fmt.Sprintf("script=%v", r.PkScript)
+	return fmt.Sprintf("outpoint=<zero>, script=%v", r.PkScript)
 }
 
 // SpendHintKey returns the key that will be used to index the spend request's
@@ -1080,8 +1081,8 @@ func (n *TxNotifier) RegisterSpend(outpoint *wire.OutPoint, pkScript []byte,
 	// notifications don't also attempt a historical dispatch.
 	spendSet.rescanStatus = rescanPending
 
-	Log.Debugf("Dispatching historical spend rescan for %v",
-		ntfn.SpendRequest)
+	Log.Infof("Dispatching historical spend rescan for %v, start=%d, "+
+		"end=%d", ntfn.SpendRequest, startHeight, n.currentHeight)
 
 	return &SpendRegistration{
 		Event: ntfn.Event,
@@ -1241,6 +1242,8 @@ func (n *TxNotifier) updateSpendDetails(spendRequest SpendRequest,
 				n.currentHeight, spendRequest, err)
 		}
 
+		Log.Debugf("Updated spend hint to height=%v for unconfirmed "+
+			"spend request %v", n.currentHeight, spendRequest)
 		return nil
 	}
 
@@ -1266,6 +1269,9 @@ func (n *TxNotifier) updateSpendDetails(spendRequest SpendRequest,
 			details.SpendingHeight, spendRequest, err)
 	}
 
+	Log.Debugf("Updated spend hint to height=%v for confirmed spend "+
+		"request %v", details.SpendingHeight, spendRequest)
+
 	spendSet.details = details
 	for _, ntfn := range spendSet.ntfns {
 		err := n.dispatchSpendDetails(ntfn, spendSet.details)
@@ -1284,11 +1290,15 @@ func (n *TxNotifier) dispatchSpendDetails(ntfn *SpendNtfn, details *SpendDetail)
 	// If there are no spend details to dispatch or if the notification has
 	// already been dispatched, then we can skip dispatching to this client.
 	if details == nil || ntfn.dispatched {
+		Log.Debugf("Skipping dispatch of spend details(%v) for "+
+			"request %v, dispatched=%v", details, ntfn.SpendRequest,
+			ntfn.dispatched)
 		return nil
 	}
 
-	Log.Infof("Dispatching confirmed spend notification for %v at height=%d",
-		ntfn.SpendRequest, n.currentHeight)
+	Log.Infof("Dispatching confirmed spend notification for %v at "+
+		"current height=%d: %v", ntfn.SpendRequest, n.currentHeight,
+		details)
 
 	select {
 	case ntfn.Event.Spend <- details:
@@ -1341,6 +1351,8 @@ func (n *TxNotifier) ConnectTip(blockHash *chainhash.Hash, blockHeight uint32,
 
 	// First, we'll iterate over all the transactions found in this block to
 	// determine if it includes any relevant transactions to the TxNotifier.
+	Log.Debugf("Filtering %d txns for %d spend requests at height %d",
+		len(txns), len(n.spendNotifications), blockHeight)
 	for _, tx := range txns {
 		n.filterTx(
 			tx, blockHash, blockHeight, n.handleConfDetailsAtTip,
@@ -1382,6 +1394,8 @@ func (n *TxNotifier) ConnectTip(blockHash *chainhash.Hash, blockHeight uint32,
 				}
 			}
 
+			Log.Debugf("Deleting mature spend request %v at "+
+				"height=%d", spendRequest, blockHeight)
 			delete(n.spendNotifications, spendRequest)
 		}
 		delete(n.spendsByHeight, matureBlockHeight)
@@ -1574,6 +1588,9 @@ func (n *TxNotifier) handleSpendDetailsAtTip(spendRequest SpendRequest,
 		n.spendsByHeight[spendHeight] = opSet
 	}
 	opSet[spendRequest] = struct{}{}
+
+	Log.Debugf("Spend request %v spent at tip=%d", spendRequest,
+		spendHeight)
 }
 
 // NotifyHeight dispatches confirmation and spend notifications to the clients

chainregistry.go

@@ -593,6 +593,12 @@ var (
 				"soa.lnd.nodes.directory",
 			},
 		},
+
+		monacoinTestnetGenesis: {
+			{
+				"testlnd.nodes.directory",
+			},
+		},
 	}
 )
 

chanbackup/recover.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/btcsuite/btcd/btcec"
 	"github.com/davecgh/go-spew/spew"
+	"github.com/monasuite/lnd/channeldb"
 	"github.com/monasuite/lnd/keychain"
 )
 
@@ -47,6 +48,14 @@ func Recover(backups []Single, restorer ChannelRestorer,
 			backup.FundingOutpoint)
 
 		err := restorer.RestoreChansFromSingles(backup)
+
+		// If a channel is already present in the channel DB, we can
+		// just continue. No reason to fail a whole set of multi backups
+		// for example. This allows resume of a restore in case another
+		// error happens.
+		if err == channeldb.ErrChanAlreadyExists {
+			continue
+		}
 		if err != nil {
 			return err
 		}

chanbackup/single.go

@@ -66,6 +66,9 @@ type Single struct {
 	// ShortChannelID encodes the exact location in the chain in which the
 	// channel was initially confirmed. This includes: the block height,
 	// transaction index, and the output within the target transaction.
+	// Channels that were not confirmed at the time of backup creation will
+	// have the funding TX broadcast height set as their block height in
+	// the ShortChannelID.
 	ShortChannelID lnwire.ShortChannelID
 
 	// RemoteNodePub is the identity public key of the remote node this
@@ -126,11 +129,21 @@ func NewSingle(channel *channeldb.OpenChannel,
 	// key.
 	_, shaChainPoint := btcec.PrivKeyFromBytes(btcec.S256(), b.Bytes())
 
+	// If a channel is unconfirmed, the block height of the ShortChannelID
+	// is zero. This will lead to problems when trying to restore that
+	// channel as the spend notifier would get a height hint of zero.
+	// To work around that problem, we add the channel broadcast height
+	// to the channel ID so we can use that as height hint on restore.
+	chanID := channel.ShortChanID()
+	if chanID.BlockHeight == 0 {
+		chanID.BlockHeight = channel.FundingBroadcastHeight
+	}
+
 	single := Single{
 		IsInitiator:     channel.IsInitiator,
 		ChainHash:       channel.ChainHash,
 		FundingOutpoint: channel.FundingOutpoint,
-		ShortChannelID:  channel.ShortChannelID,
+		ShortChannelID:  chanID,
 		RemoteNodePub:   channel.IdentityPub,
 		Addresses:       nodeAddrs,
 		Capacity:        channel.Capacity,

chanbackup/single_test.go

@@ -410,4 +410,44 @@ func TestSinglePackStaticChanBackups(t *testing.T) {
 	}
 }
 
+// TestSingleUnconfirmedChannel tests that unconfirmed channels get serialized
+// correctly by encoding the funding broadcast height as block height of the
+// short channel ID.
+func TestSingleUnconfirmedChannel(t *testing.T) {
+	t.Parallel()
+
+	var fundingBroadcastHeight = uint32(1234)
+
+	// Let's create an open channel shell that contains all the information
+	// we need to create a static channel backup but simulate an
+	// unconfirmed channel by setting the block height to 0.
+	channel, err := genRandomOpenChannelShell()
+	if err != nil {
+		t.Fatalf("unable to gen open channel: %v", err)
+	}
+	channel.ShortChannelID.BlockHeight = 0
+	channel.FundingBroadcastHeight = fundingBroadcastHeight
+
+	singleChanBackup := NewSingle(channel, []net.Addr{addr1, addr2})
+	keyRing := &mockKeyRing{}
+
+	// Pack it and then unpack it again to make sure everything is written
+	// correctly, then check that the block height of the unpacked
+	// is the funding broadcast height we set before.
+	var b bytes.Buffer
+	if err := singleChanBackup.PackToWriter(&b, keyRing); err != nil {
+		t.Fatalf("unable to pack single: %v", err)
+	}
+	var unpackedSingle Single
+	err = unpackedSingle.UnpackFromReader(&b, keyRing)
+	if err != nil {
+		t.Fatalf("unable to unpack single: %v", err)
+	}
+	if unpackedSingle.ShortChannelID.BlockHeight != fundingBroadcastHeight {
+		t.Fatalf("invalid block height. got %d expected %d.",
+			unpackedSingle.ShortChannelID.BlockHeight,
+			fundingBroadcastHeight)
+	}
+}
+
 // TODO(roasbsef): fuzz parsing

chanrestore.go

@@ -2,9 +2,11 @@ package lnd
 
 import (
 	"fmt"
+	"math"
 	"net"
 
 	"github.com/btcsuite/btcd/btcec"
+	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/monasuite/lnd/chanbackup"
 	"github.com/monasuite/lnd/channeldb"
@@ -14,6 +16,18 @@ import (
 	"github.com/monasuite/lnd/shachain"
 )
 
+const (
+	// mainnetSCBLaunchBlock is the approximate block height of the bitcoin
+	// mainnet chain of the date when SCBs first were released in lnd
+	// (v0.6.0-beta). The block date is 4/15/2019, 10:54 PM UTC.
+	mainnetSCBLaunchBlock = 571800
+
+	// testnetSCBLaunchBlock is the approximate block height of the bitcoin
+	// testnet3 chain of the date when SCBs first were released in lnd
+	// (v0.6.0-beta). The block date is 4/16/2019, 08:04 AM UTC.
+	testnetSCBLaunchBlock = 1489300
+)
+
 // chanDBRestorer is an implementation of the chanbackup.ChannelRestorer
 // interface that is able to properly map a Single backup, into a
 // channeldb.ChannelShell which is required to fully restore a channel. We also
@@ -126,15 +140,72 @@ func (c *chanDBRestorer) openChannelShell(backup chanbackup.Single) (
 // NOTE: Part of the chanbackup.ChannelRestorer interface.
 func (c *chanDBRestorer) RestoreChansFromSingles(backups ...chanbackup.Single) error {
 	channelShells := make([]*channeldb.ChannelShell, 0, len(backups))
+	firstChanHeight := uint32(math.MaxUint32)
 	for _, backup := range backups {
 		chanShell, err := c.openChannelShell(backup)
 		if err != nil {
 			return err
 		}
 
+		// Find the block height of the earliest channel in this backup.
+		chanHeight := chanShell.Chan.ShortChanID().BlockHeight
+		if chanHeight != 0 && chanHeight < firstChanHeight {
+			firstChanHeight = chanHeight
+		}
+
 		channelShells = append(channelShells, chanShell)
 	}
 
+	// In case there were only unconfirmed channels, we will have to scan
+	// the chain beginning from the launch date of SCBs.
+	if firstChanHeight == math.MaxUint32 {
+		chainHash := channelShells[0].Chan.ChainHash
+		switch {
+		case chainHash.IsEqual(chaincfg.MainNetParams.GenesisHash):
+			firstChanHeight = mainnetSCBLaunchBlock
+
+		case chainHash.IsEqual(chaincfg.TestNet3Params.GenesisHash):
+			firstChanHeight = testnetSCBLaunchBlock
+
+		default:
+			// Worst case: We have no height hint and start at
+			// block 1. Should only happen for SCBs in regtest,
+			// simnet and litecoin.
+			firstChanHeight = 1
+		}
+	}
+
+	// If there were channels in the backup that were not confirmed at the
+	// time of the backup creation, they won't have a block height in the
+	// ShortChanID which would lead to an error in the chain watcher.
+	// We want to at least set the funding broadcast height that the chain
+	// watcher can use instead. We have two possible fallback values for
+	// the broadcast height that we are going to try here.
+	for _, chanShell := range channelShells {
+		channel := chanShell.Chan
+
+		switch {
+		// Fallback case 1: It is extremely unlikely at this point that
+		// a channel we are trying to restore has a coinbase funding TX.
+		// Therefore we can be quite certain that if the TxIndex is
+		// zero, it was an unconfirmed channel where we used the
+		// BlockHeight to encode the funding TX broadcast height. To not
+		// end up with an invalid short channel ID that looks valid, we
+		// restore the "original" unconfirmed one here.
+		case channel.ShortChannelID.TxIndex == 0:
+			broadcastHeight := channel.ShortChannelID.BlockHeight
+			channel.FundingBroadcastHeight = broadcastHeight
+			channel.ShortChannelID.BlockHeight = 0
+
+		// Fallback case 2: This is an unconfirmed channel from an old
+		// backup file where we didn't have any workaround in place.
+		// Best we can do here is set the funding broadcast height to a
+		// reasonable value that we determined earlier.
+		case channel.ShortChanID().BlockHeight == 0:
+			channel.FundingBroadcastHeight = firstChanHeight
+		}
+	}
+
 	ltndLog.Infof("Inserting %v SCB channel shells into DB",
 		len(channelShells))