Merge develop to main branch (release v2.0.0)

.env.example

@@ -30,12 +30,15 @@ JWT_SECRET=<your_secret>
 # JWT token denylist
 # JWT_DENYLIST=
 
-# Bitcoin JSON-RPC URL and credentials
-BITCOIN_JSON_RPC_URL=<http://bitcoin:8332>
-BITCOIN_JSON_RPC_USERNAME=<rpc_username>
-BITCOIN_JSON_RPC_PASSWORD=<rpc_password>
-
+# Bitcoin data provider, support mempool and electrs
+# use mempool.space as default, electrs as fallback
+# change to electrs if you want to use electrs as default and mempool.space as fallback
+BITCOIN_DATA_PROVIDER=mempool
+# Bitcoin Mempool.space API URL
+# optinal when BITCOIN_DATA_PROVIDER=electrs
+BITCOIN_MEMPOOL_SPACE_API_URL=https://mempool.space
 # Electrs API URL
+# optinal when BITCOIN_DATA_PROVIDER=mempool
 BITCOIN_ELECTRS_API_URL=<http://electrs:3002>
 
 # SPV Service URL
@@ -52,18 +55,24 @@ PAYMASTER_CELL_CAPACITY=31600000000
 PAYMASTER_CELL_PRESET_COUNT=500
 # Paymaster cell refill threshold, refill paymaster cell when the balance is less than this threshold.
 PAYMASTER_CELL_REFILL_THRESHOLD=0.3
+# Check the paymaster BTC UTXO when processing rgb++ ckb transaction
+PAYMASTER_RECEIVE_UTXO_CHECK=false
 # Paymaster bitcoin address, used to receive BTC from users
 PAYMASTER_RECEIVE_BTC_ADDRESS=<paymaster_btc_address>
 # Paymaster receives BTC UTXO size in sats
 PAYMASTER_BTC_CONTAINER_FEE_SATS=7000
 
-# BTCTimeLock cell unlock batch size
-UNLOCKER_CELL_BATCH_SIZE=100
 # BTCTimeLock cell unlock cron job schedule, default is every 5 minutes
 UNLOCKER_CRON_SCHEDULE='*/5 * * * *'
+# BTCTimeLock cell unlock batch size
+UNLOCKER_CELL_BATCH_SIZE=100
 # BTCTimeLock cell unlocker monitor slug, used for monitoring unlocker status on sentry
 UNLOCKER_MONITOR_SLUG=btctimelock-cells-unlocker
 
 # RGB++ CKB transaction Queue cron job delay in milliseconds
 # the /rgbpp/v1/transaction/ckb-tx endpoint is called, the transaction will be added to the queue
-TRANSACTION_QUEUE_JOB_DELAY=12000
+TRANSACTION_QUEUE_JOB_DELAY=120000
+# RGB++ CKB transaction Queue cron job attempts
+TRANSACTION_QUEUE_JOB_ATTEMPTS=6
+# Pay fee for transaction with pool reject by min fee rate, false by default
+TRANSACTION_PAY_FOR_MIN_FEE_RATE_REJECT=false

.github/workflows/test.yaml

@@ -4,7 +4,7 @@ name: Unit Tests
 on:
   workflow_dispatch:
   push:
-    branches: 
+    branches:
       - main
       - develop
   pull_request:
@@ -15,6 +15,9 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        BITCOIN_DATA_PROVIDER: [mempool, electrs]
 
     services:
       redis:
@@ -48,9 +51,8 @@ jobs:
       - name: Run Unit Tests
         env:
           JWT_SECRET: ${{ secrets.JWT_SECRET }}
-          BITCOIN_JSON_RPC_URL: ${{ secrets.BITCOIN_JSON_RPC_URL }}
-          BITCOIN_JSON_RPC_USERNAME: ${{ secrets.BITCOIN_JSON_RPC_USERNAME }}
-          BITCOIN_JSON_RPC_PASSWORD: ${{ secrets.BITCOIN_JSON_RPC_PASSWORD }}
+          BITCOIN_DATA_PROVIDER: ${{ matrix.BITCOIN_DATA_PROVIDER }}
+          BITCOIN_MEMPOOL_SPACE_API_URL: ${{ secrets.BITCOIN_MEMPOOL_SPACE_API_URL }}
           BITCOIN_ELECTRS_API_URL: ${{ secrets.BITCOIN_ELECTRS_API_URL }}
           BITCOIN_SPV_SERVICE_URL: ${{ secrets.BITCOIN_SPV_SERVICE_URL }}
           PAYMASTER_RECEIVE_BTC_ADDRESS: ${{ secrets.PAYMASTER_RECEIVE_BTC_ADDRESS }}

README.md

@@ -3,6 +3,7 @@
 A service for Retrieving BTC/RGB++ information/assets and processing transactions with these assets
 
 ### Features
+
 - Retrieving Blockchain Information such as Bitcoin chain info, blocks, headers, transactions, addresses and RGB++ assets
 - Transaction Handling by posting transactions to the /bitcoin/v1/transaction or /rgbpp/v1/transaction/ckb-tx endpoint
 - RGB++ CKB transaction Queue simplifies the RGB++ assets workflows by some cron jobs
@@ -11,13 +12,14 @@ A service for Retrieving BTC/RGB++ information/assets and processing transaction
 
 #### Requirements
 
-- [bitcoind](https://github.com/bitcoin/bitcoin): Running a Bitcoin full node
-- [mempool/electrs](https://github.com/mempool/electrs): Electrum Rust Server (Electrs) indexes Bitcoin chain data
+- [mempool.space](https://mempool.space/docs) or [mempool/electrs](https://github.com/mempool/electrs): provides data about the Bitcoin network.
+  - We can use either of them as data provider
+  - Or use both, designating one as the primary provider and the other as the fallback
 - [ckb-cell/ckb-bitcoin-spv-service](https://github.com/ckb-cell/ckb-bitcoin-spv-service): CKB Bitcoin SPV Service
 
 #### Configuration
 
-Copy the `.env.example` file to `.env`: 
+Copy the `.env.example` file to `.env`:
 
 ```bash
 cp .env.example .env
@@ -58,12 +60,15 @@ JWT_SECRET=<your_secret>
 # JWT token denylist
 # JWT_DENYLIST=
 
-# Bitcoin JSON-RPC URL and credentials
-BITCOIN_JSON_RPC_URL=<http://bitcoin:8332>
-BITCOIN_JSON_RPC_USERNAME=<rpc_username>
-BITCOIN_JSON_RPC_PASSWORD=<rpc_password>
-
+# Bitcoin data provider, support mempool and electrs
+# use mempool.space as default, electrs as fallback
+# change to electrs if you want to use electrs as default and mempool.space as fallback
+BITCOIN_DATA_PROVIDER=mempool
+# Bitcoin Mempool.space API URL
+# optinal when BITCOIN_DATA_PROVIDER=electrs
+BITCOIN_MEMPOOL_SPACE_API_URL=https://mempool.space
 # Electrs API URL
+# optinal when BITCOIN_DATA_PROVIDER=mempool
 BITCOIN_ELECTRS_API_URL=<http://electrs:3002>
 
 # SPV Service URL
@@ -80,21 +85,27 @@ PAYMASTER_CELL_CAPACITY=31600000000
 PAYMASTER_CELL_PRESET_COUNT=500
 # Paymaster cell refill threshold, refill paymaster cell when the balance is less than this threshold.
 PAYMASTER_CELL_REFILL_THRESHOLD=0.3
+# Check the paymaster BTC UTXO when processing rgb++ ckb transaction
+PAYMASTER_RECEIVE_UTXO_CHECK=false
 # Paymaster bitcoin address, used to receive BTC from users
 PAYMASTER_RECEIVE_BTC_ADDRESS=<paymaster_btc_address>
 # Paymaster receives BTC UTXO size in sats
 PAYMASTER_BTC_CONTAINER_FEE_SATS=7000
 
-# BTCTimeLock cell unlock batch size
-UNLOCKER_CELL_BATCH_SIZE=100
 # BTCTimeLock cell unlock cron job schedule, default is every 5 minutes
 UNLOCKER_CRON_SCHEDULE='*/5 * * * *'
+# BTCTimeLock cell unlock batch size
+UNLOCKER_CELL_BATCH_SIZE=100
 # BTCTimeLock cell unlocker monitor slug, used for monitoring unlocker status on sentry
 UNLOCKER_MONITOR_SLUG=btctimelock-cells-unlocker
 
 # RGB++ CKB transaction Queue cron job delay in milliseconds
 # the /rgbpp/v1/transaction/ckb-tx endpoint is called, the transaction will be added to the queue
-TRANSACTION_QUEUE_JOB_DELAY=12000
+TRANSACTION_QUEUE_JOB_DELAY=120000
+# RGB++ CKB transaction Queue cron job attempts
+TRANSACTION_QUEUE_JOB_ATTEMPTS=6
+# Pay fee for transaction with pool reject by min fee rate, false by default
+TRANSACTION_PAY_FOR_MIN_FEE_RATE_REJECT=false
 ```
 
 More configuration options can be found in the `src/env.ts` file.

docs/upgrading-to-v2.md

@@ -0,0 +1,49 @@
+## Upgrading to V2
+
+We have released v2.0.0 for support `mempool.space` API as a Bitcoin data provider, and it 
+provides a set of [IBitcoinDataProvider](https://github.com/ckb-cell/btc-assets-api/blob/8fb495576c957e9006ef648d6c24312a3f10e34f/src/services/bitcoin/interface.ts#L3) interfaces. Note that it is still compatible with the `electrs` used previously. 
+
+There are two ways to upgrade: 
+
+### Upgrading from v1.x.x and use electrs (**compatible, by default**)
+Suppose you do not want to use the mempool.space API as the main data provider, **you do not need to make any changes**. 
+
+But we recommend you remove the following env vars for safety:
+
+```env
+BITCOIN_JSON_RPC_URL=<http://bitcoin:8332>
+BITCOIN_JSON_RPC_USERNAME=<rpc_username>
+BITCOIN_JSON_RPC_PASSWORD=<rpc_password>
+```
+
+and add the following env vars to make sure to use electrs as the primary data provider, and add mempool.space API as a fallback:
+
+```env
+BITCOIN_DATA_PROVIDER=electrs # recommend, electrs by default
+BITCOIN_MEMPOOL_SPACE_API_URL=https://mempool.space # optional, mempool.space as the fallback
+```
+
+### Upgrading from v1.x.x and using mempool.space API (**new feature**)
+The new feature in v2.0.0, we can use mempool.space API as the primary data provider, and use electrs as a fallback.
+
+Add the following env vars:
+
+```env
+BITCOIN_DATA_PROVIDER=mempool 
+BITCOIN_MEMPOOL_SPACE_API_URL=https://mempool.space
+```
+
+If you want to use the previous electrs as a fallback, keep the original `BITCOIN_ELECTRS_API_URL` env var. Otherwise, remove this var to avoid using electrs.
+
+```env
+BITCOIN_ELECTRS_API_URL=<http://electrs:3002> # optional, electrs as fallback
+```
+
+#### Recommended Fees API
+If use mempool.space API as the primary data provider, then we can use `/bitcoin/v1/fees/recommended` to get the bitcoin fees. and we will calculate fees when mempool.space recommend fees API unavailable (see https://github.com/ckb-cell/btc-assets-api/pull/114).
+
+**use electrs as the primary data provider and dosen't set `BITCOIN_MEMPOOL_SPACE_API_URL` as a fallback, then recommended fees API will be unavailable**
+
+
+
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "btc-assets-api",
-  "version": "1.5.0",
+  "version": "2.0.0",
   "title": "Bitcoin/RGB++ Assets API",
   "description": "",
   "main": "index.js",
@@ -25,6 +25,7 @@
     ]
   },
   "dependencies": {
+    "@cell-studio/mempool.js": "^2.4.0",
     "@ckb-lumos/base": "^0.22.2 ",
     "@ckb-lumos/ckb-indexer": "^0.22.2",
     "@ckb-lumos/codec": "^0.22.2",
@@ -40,9 +41,9 @@
     "@fastify/swagger-ui": "^3.0.0",
     "@immobiliarelabs/fastify-sentry": "^8.0.1",
     "@nervosnetwork/ckb-sdk-utils": "^0.109.1",
-    "@rgbpp-sdk/btc": "0.0.0-snap-20240423144119",
-    "@rgbpp-sdk/ckb": "0.0.0-snap-20240423144119",
-    "@rgbpp-sdk/service": "0.0.0-snap-20240423144119",
+    "@rgbpp-sdk/btc": "0.0.0-snap-20240430102443",
+    "@rgbpp-sdk/ckb": "0.0.0-snap-20240430102443",
+    "@rgbpp-sdk/service": "0.0.0-snap-20240430102443",
     "@sentry/node": "^7.102.1",
     "@sentry/profiling-node": "^7.102.1",
     "awilix": "^10.0.1",