feat: send sbtc requests context to signers and store sighashs to db (#…

…954)

* WIP

* Add an outpoint type

* WIP

* WIP

* Rename the enum

* Change the validation output

* More refactoring

* fake docs

* Add documentation

* This is unnecessary

* Fix up more comments

* Add todos for more work for validation

* add sighashes tables

* fix enum name

* add bitcoin_withdrawals_outputs table and new queries

* fix tests. add new batch queries

* Remove locking and just use an ID for
keeping track of the signer in the new WanNetwork

* Add some more comments

* Fix up after changes

* Add code to transform the TerminationHandle
into a BroadcastStream.

* Add a new function to the MessageTransfer trait,
implement it for the types that implement the trait

* Add a new function to the Context trait that
has a default implementation that combines
all streams that a signer may be interested in

* Use the new streams in our event loops

* Add in an enum variant

* Fix up after using the new types

* updates after using the new stream

* Temporarily ignore some tests

* Minor changes in the TxSigner

* minor test update

* Fix the WSTS bug

* Ignore tests that are ignored in later updates

* unnecessary

* Ignore another test

* fix the test

* This one can be fixed now

* No more aborting

* this test is already fixed

* Add a new request decider event loop

* Add a new signer event type

* Make sure to use the right events

* Small refactor

* Move the sleep to before the chain tip lookup

* move it back

* revert

* cargo fmt

* Add in the new event loop and remove
unnecessary field from the tx-signer

* add integration tests. fix prevout_type type in the query

* add block_hash to bitcoin_withdrawals_outputs primary key

* address review comments

* Comment update

* Remove unnecessary cruft

* Add a will sign field in case we want to
make the query super simple

* add prevout_type back

* Add some more comments, cargo fmt

* Remove the withdrawal check in the
prevalidation function

* Make sure the people cannot have fees
assessments that are greater than the deposit amount

* wip

* Allow the max-fee to be configurable

* Simplify the block observer

* fix typo

* save txs sighashes on db

* recreate wstscoordinato at each sign round

* rename BitcoinBlockSbtcRequests, reuse signers_key

* Remove the construction version

* Add integration tests and test fixtures

* add fees to the BitcoinBlockSbtcRequests

* fix validation tests

* update schemas to remove version

* update function name

* test the amount fee thingy

* bump the timeout

* fix test that was randomly failing

* add wait timer after sending requests context

* re-create coordinator state machine at each signing round

* rename SbtcRequestsContextMessage

* fix linter warning

* fix function naming

* Add another test

* Another test

* Finish adding tests

* That was silly

* add test

* fix typo

* get_bitcoin_tx_sighash -> will_sign_bitcoin_tx_sighas

* update test

* address review comments

* change queries signature to accept slices

* address review comments

* Feat/add is unique implementation for request packages (#888)

* add is_unique logic

* fix lint warning

* add missing import

* fix test. address comments. make sighash the pk

* fix test

* move new tables into new migration file

* wait for signers acks after sending bitcoin pre request

* Revert "wait for signers acks after sending bitcoin pre request"

This reverts commit 468153f.

* Fix up the new tables

* drop the database at the end

* add todo for BitcoinPreSignRequest proto

---------

Co-authored-by: djordon <[email protected]>

sbtc/src/testing/regtest.rs

@@ -1,5 +1,5 @@
 //! Integration testing helper functions
-//!  
+//!
 
 use bitcoin::absolute::LockTime;
 use bitcoin::key::TapTweak;

signer/src/bitcoin/utxo.rs

@@ -30,6 +30,8 @@ use secp256k1::Keypair;
 use secp256k1::Message;
 use secp256k1::XOnlyPublicKey;
 use secp256k1::SECP256K1;
+use serde::Deserialize;
+use serde::Serialize;
 
 use crate::bitcoin::packaging::compute_optimal_packages;
 use crate::bitcoin::packaging::Weighted;
@@ -39,6 +41,7 @@ use crate::keys::SignerScriptPubKey as _;
 use crate::storage::model;
 use crate::storage::model::BitcoinTx;
 use crate::storage::model::BitcoinTxId;
+use crate::storage::model::QualifiedRequestId;
 use crate::storage::model::ScriptPubKey;
 use crate::storage::model::SignerVotes;
 use crate::storage::model::StacksBlockHash;
@@ -77,7 +80,7 @@ const SATS_PER_VBYTE_INCREMENT: f64 = 0.001;
 const OP_RETURN_VERSION: u8 = 0;
 
 /// Describes the fees for a transaction.
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
 pub struct Fees {
     /// The total fee paid in sats for the transaction.
     pub total: u64,
@@ -452,6 +455,15 @@ impl WithdrawalRequest {
             block_hash: request.block_hash,
         }
     }
+
+    /// Return the identifier for the withdrawal request.
+    pub fn qualified_id(&self) -> QualifiedRequestId {
+        QualifiedRequestId {
+            request_id: self.request_id,
+            txid: self.txid,
+            block_hash: self.block_hash,
+        }
+    }
 }
 
 /// A reference to either a deposit or withdraw request
@@ -871,7 +883,7 @@ impl<'a> UnsignedTransaction<'a> {
     /// ```text
     ///  0       2    3     5        21
     ///  |-------|----|-----|--------|
-    ///    magic   op   N_d   bitmap   
+    ///    magic   op   N_d   bitmap
     /// ```
     ///
     /// In the above layout, magic is the UTF-8 encoded string "ST", op is

signer/src/bitcoin/validation.rs

@@ -7,12 +7,15 @@ use bitcoin::Amount;
 use bitcoin::OutPoint;
 use bitcoin::ScriptBuf;
 use bitcoin::XOnlyPublicKey;
+use serde::Deserialize;
+use serde::Serialize;
 
 use crate::bitcoin::utxo::FeeAssessment;
 use crate::bitcoin::utxo::SignerBtcState;
 use crate::context::Context;
 use crate::error::Error;
 use crate::keys::PublicKey;
+use crate::message::OutPointMessage;
 use crate::storage::model::BitcoinBlockHash;
 use crate::storage::model::BitcoinTxId;
 use crate::storage::model::BitcoinTxSigHash;
@@ -54,15 +57,29 @@ pub struct BitcoinTxContext {
 
 /// This type is a container for all deposits and withdrawals that are part
 /// of a transaction package.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct TxRequestIds {
     /// The deposit requests associated with the inputs in the transaction.
-    pub deposits: Vec<OutPoint>,
+    pub deposits: Vec<OutPointMessage>,
     /// The withdrawal requests associated with the outputs in the current
     /// transaction.
     pub withdrawals: Vec<QualifiedRequestId>,
 }
 
+impl From<&Requests<'_>> for TxRequestIds {
+    fn from(requests: &Requests) -> Self {
+        let mut deposits = Vec::new();
+        let mut withdrawals = Vec::new();
+        for request in requests.iter() {
+            match request {
+                RequestRef::Deposit(deposit) => deposits.push(deposit.outpoint.into()),
+                RequestRef::Withdrawal(withdrawal) => withdrawals.push(withdrawal.qualified_id()),
+            }
+        }
+        TxRequestIds { deposits, withdrawals }
+    }
+}
+
 /// Check that this does not contain duplicate deposits or withdrawals.
 pub fn is_unique(packages: &[TxRequestIds]) -> bool {
     let mut deposits_set = HashSet::new();
@@ -174,7 +191,6 @@ impl BitcoinTxContext {
             withdrawals,
             signer_state,
         };
-
         let mut signer_state = signer_state;
         let tx = reports.create_transaction()?;
         let sighashes = tx.construct_digests()?;
@@ -942,8 +958,8 @@ mod tests {
     #[test_case(
         vec![TxRequestIds {
             deposits: vec![
-                OutPoint::new(Txid::from_byte_array([1; 32]), 0),
-                OutPoint::new(Txid::from_byte_array([1; 32]), 1)
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 0},
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 1}
             ],
             withdrawals: vec![
                 QualifiedRequestId {
@@ -960,8 +976,8 @@ mod tests {
     #[test_case(
         vec![TxRequestIds {
             deposits: vec![
-                OutPoint::new(Txid::from_byte_array([1; 32]), 0),
-                OutPoint::new(Txid::from_byte_array([1; 32]), 0)
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 0},
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 0}
             ],
             withdrawals: vec![
                 QualifiedRequestId {
@@ -978,8 +994,8 @@ mod tests {
     #[test_case(
         vec![TxRequestIds {
             deposits: vec![
-                OutPoint::new(Txid::from_byte_array([1; 32]), 0),
-                OutPoint::new(Txid::from_byte_array([1; 32]), 1)
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 0},
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 1}
             ],
             withdrawals: vec![
                 QualifiedRequestId {
@@ -996,8 +1012,8 @@ mod tests {
     #[test_case(
         vec![TxRequestIds {
             deposits: vec![
-                OutPoint::new(Txid::from_byte_array([1; 32]), 0),
-                OutPoint::new(Txid::from_byte_array([1; 32]), 1)
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 0},
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 1}
             ],
             withdrawals: vec![
                 QualifiedRequestId {
@@ -1013,7 +1029,7 @@ mod tests {
         ]},
         TxRequestIds {
             deposits: vec![
-                OutPoint::new(Txid::from_byte_array([1; 32]), 1)
+                OutPointMessage{txid: Txid::from_byte_array([1; 32]), vout: 1}
             ],
             withdrawals: vec![]
         }], false; "duplicate-requests-in-different-txs")]

signer/src/message.rs

@@ -1,13 +1,17 @@
 //! Signer message definition for network communication
 
+use bitcoin::OutPoint;
 use secp256k1::ecdsa::RecoverableSignature;
 use sha2::Digest;
 
+use crate::bitcoin::utxo::Fees;
+use crate::bitcoin::validation::TxRequestIds;
 use crate::keys::PublicKey;
 use crate::keys::SignerScriptPubKey as _;
 use crate::signature::RecoverableEcdsaSignature as _;
 use crate::stacks::contracts::StacksTx;
 use crate::storage::model::BitcoinBlockHash;
+use crate::storage::model::QualifiedRequestId;
 use crate::storage::model::StacksTxId;
 
 /// Messages exchanged between signers
@@ -38,6 +42,8 @@ pub enum Payload {
     WstsMessage(WstsMessage),
     /// Information about a new sweep transaction
     SweepTransactionInfo(SweepTransactionInfo),
+    /// Information about a new Bitcoin block sign request
+    BitcoinPreSignRequest(BitcoinPreSignRequest),
 }
 
 impl std::fmt::Display for Payload {
@@ -72,6 +78,7 @@ impl std::fmt::Display for Payload {
                 write!(f, ")")
             }
             Self::SweepTransactionInfo(_) => write!(f, "SweepTransactionInfo(..)"),
+            Self::BitcoinPreSignRequest(_) => write!(f, "BitcoinPreSignRequest(..)"),
         }
     }
 }
@@ -134,6 +141,12 @@ impl From<SweepTransactionInfo> for Payload {
     }
 }
 
+impl From<BitcoinPreSignRequest> for Payload {
+    fn from(value: BitcoinPreSignRequest) -> Self {
+        Self::BitcoinPreSignRequest(value)
+    }
+}
+
 /// Represents information about a new sweep transaction.
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 pub struct SweepTransactionInfo {
@@ -322,6 +335,43 @@ pub struct BitcoinTransactionSignAck {
     pub txid: bitcoin::Txid,
 }
 
+/// The message version of an the [`OutPoint`].
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
+pub struct OutPointMessage {
+    /// The referenced transaction's txid.
+    pub txid: bitcoin::Txid,
+    /// The index of the referenced output in its transaction's vout.
+    pub vout: u32,
+}
+
+impl From<OutPoint> for OutPointMessage {
+    fn from(outpoint: OutPoint) -> Self {
+        OutPointMessage {
+            txid: outpoint.txid,
+            vout: outpoint.vout,
+        }
+    }
+}
+
+impl From<OutPointMessage> for OutPoint {
+    fn from(val: OutPointMessage) -> Self {
+        OutPoint { txid: val.txid, vout: val.vout }
+    }
+}
+
+/// The transaction context needed by the signers to reconstruct the transaction.
+#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
+pub struct BitcoinPreSignRequest {
+    /// The set of sBTC requests with additional relevant
+    /// information for the transaction.
+    pub requests: Vec<TxRequestIds>,
+    /// The current market fee rate in sat/vByte.
+    pub fee_rate: f64,
+    /// The total fee amount and the fee rate for the last transaction that
+    /// used this UTXO as an input.
+    pub last_fees: Option<Fees>,
+}
+
 /// A wsts message.
 #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
 pub struct WstsMessage {
@@ -351,6 +401,7 @@ impl wsts::net::Signable for Payload {
             Self::StacksTransactionSignRequest(msg) => msg.hash(hasher),
             Self::StacksTransactionSignature(msg) => msg.hash(hasher),
             Self::SweepTransactionInfo(msg) => msg.hash(hasher),
+            Self::BitcoinPreSignRequest(msg) => msg.hash(hasher),
         }
     }
 }
@@ -452,6 +503,42 @@ impl wsts::net::Signable for WstsMessage {
     }
 }
 
+impl wsts::net::Signable for OutPointMessage {
+    fn hash(&self, hasher: &mut sha2::Sha256) {
+        hasher.update(self.txid);
+        hasher.update(self.vout.to_be_bytes());
+    }
+}
+
+impl wsts::net::Signable for QualifiedRequestId {
+    fn hash(&self, hasher: &mut sha2::Sha256) {
+        hasher.update(self.request_id.to_be_bytes());
+        hasher.update(self.txid.as_bytes());
+        hasher.update(self.block_hash.as_bytes());
+    }
+}
+
+impl wsts::net::Signable for TxRequestIds {
+    fn hash(&self, hasher: &mut sha2::Sha256) {
+        for deposit in &self.deposits {
+            deposit.hash(hasher);
+        }
+        for withdrawal in &self.withdrawals {
+            withdrawal.hash(hasher);
+        }
+    }
+}
+
+impl wsts::net::Signable for BitcoinPreSignRequest {
+    fn hash(&self, hasher: &mut sha2::Sha256) {
+        hasher.update("SIGNER_NEW_BITCOIN_TX_CONTEXT");
+        for request in &self.requests {
+            request.hash(hasher);
+        }
+        hasher.update(self.fee_rate.to_be_bytes());
+    }
+}
+
 /// Convenient type aliases
 type StacksBlockHash = [u8; 32];
 

signer/src/network/libp2p/network.rs

@@ -197,7 +197,7 @@ mod tests {
             handle1.abort();
             handle2.abort();
             panic!(
-                r#"Test timed out, we waited for 30 seconds but this usually takes around 5 seconds. 
+                r#"Test timed out, we waited for 30 seconds but this usually takes around 5 seconds.
             This is generally due to connectivity issues between the two swarms."#
             );
         }

signer/src/proto/convert.rs

@@ -1294,6 +1294,7 @@ impl From<SignerMessage> for proto::SignerMessage {
             Payload::SweepTransactionInfo(inner) => {
                 proto::signer_message::Payload::SweepTransactionInfo(inner.into())
             }
+            Payload::BitcoinPreSignRequest(_) => todo!(),
         };
         proto::SignerMessage {
             bitcoin_chain_tip: Some(value.bitcoin_chain_tip.into()),

signer/src/request_decider.rs

@@ -166,6 +166,7 @@ where
             }
             Payload::StacksTransactionSignRequest(_)
             | Payload::BitcoinTransactionSignRequest(_)
+            | Payload::BitcoinPreSignRequest(_)
             | Payload::WstsMessage(_)
             | Payload::SweepTransactionInfo(_)
             | Payload::StacksTransactionSignature(_)

signer/src/storage/model.rs

@@ -780,7 +780,7 @@ pub enum TxPrevoutType {
 ///
 /// A request-id and a Stacks Block ID is enough to uniquely identify the
 /// request, but we add in the transaction ID for completeness.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub struct QualifiedRequestId {
     /// The ID that was generated in the clarity contract call for the
     /// withdrawal request.
@@ -955,7 +955,7 @@ impl From<&BitcoinBlock> for BitcoinBlockRef {
 }
 
 /// The Stacks block ID. This is different from the block header hash.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub struct StacksBlockHash(StacksBlockId);
 
 impl Deref for StacksBlockHash {