diff --git a/Cargo.lock b/Cargo.lock index 2523c0160..8e4d3cab9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -413,6 +413,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bdk_coin_select" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd50028fb9c48ce062859565fc59974cd1ee614f8db1c4bfc8c094ef1ffe6ff6" + [[package]] name = "bech32" version = "0.9.1" @@ -2209,6 +2215,7 @@ dependencies = [ "anyhow", "async-trait", "bdk", + "bdk_coin_select", "bip39", "bitcoin", "dlc", diff --git a/crates/ln-dlc-node/Cargo.toml b/crates/ln-dlc-node/Cargo.toml index 2442351c1..c6b632313 100644 --- a/crates/ln-dlc-node/Cargo.toml +++ b/crates/ln-dlc-node/Cargo.toml @@ -10,6 +10,7 @@ description = "A common interface for using Lightning and DLC channels side-by-s anyhow = { version = "1", features = ["backtrace"] } async-trait = "0.1.71" bdk = { version = "0.28.0", default-features = false, features = ["key-value-db", "use-esplora-blocking", "std"] } +bdk_coin_select = "0.2.0" bip39 = { version = "2", features = ["rand_core"] } bitcoin = "0.29.2" dlc = { version = "0.4.0" } diff --git a/crates/ln-dlc-node/src/ldk_node_wallet.rs b/crates/ln-dlc-node/src/ldk_node_wallet.rs index 5e59fc5f8..0de4a331d 100644 --- a/crates/ln-dlc-node/src/ldk_node_wallet.rs +++ b/crates/ln-dlc-node/src/ldk_node_wallet.rs @@ -15,22 +15,28 @@ use bdk::FeeRate; use bdk::SignOptions; use bdk::SyncOptions; use bdk::TransactionDetails; +use bdk_coin_select::metrics::LowestFee; +use bdk_coin_select::Candidate; +use bdk_coin_select::ChangePolicy; +use bdk_coin_select::CoinSelector; +use bdk_coin_select::DrainWeights; +use bdk_coin_select::Target; use bitcoin::consensus::encode::serialize_hex; use bitcoin::psbt::PartiallySignedTransaction; use bitcoin::Address; use bitcoin::Amount; use bitcoin::BlockHash; -use bitcoin::Network; use bitcoin::OutPoint; use bitcoin::Script; use bitcoin::Transaction; +use bitcoin::TxIn; use bitcoin::Txid; +use bitcoin::VarInt; use dlc_manager::Utxo; use lightning::chain::chaininterface::BroadcasterInterface; use lightning::chain::chaininterface::ConfirmationTarget; use parking_lot::Mutex; use parking_lot::MutexGuard; -use rust_bitcoin_coin_selection::select_coins; use std::sync::Arc; use std::time::Instant; use tokio::sync::RwLock; @@ -50,6 +56,8 @@ where inner: Mutex>, settings: RwLock, fee_rate_estimator: Arc, + // Only cleared upon restart. This means that if a locked outpoint ends up unspent, it will + // remain locked until the binary is restarted. locked_outpoints: Mutex>, node_storage: Arc, } @@ -170,6 +178,10 @@ where .address) } + pub(crate) fn get_new_address(&self) -> Result
{ + Ok(self.bdk_lock().get_address(AddressIndex::New)?.address) + } + pub fn is_mine(&self, script: &Script) -> Result { Ok(self.bdk_lock().is_mine(script)?) } @@ -183,58 +195,121 @@ where Ok(utxos) } - pub fn get_utxos_for_amount( + pub fn get_utxos_for_dlc_funding_transaction( &self, amount: u64, - lock_utxos: bool, - network: Network, + fee_rate: Option, + should_lock_utxos: bool, ) -> Result> { - let utxos = self.get_utxos()?; - // get temporarily reserved utxo from in-memory storage + let network = { + let bdk = self.bdk_lock(); + bdk.network() + }; + + let fee_rate = fee_rate.map(|fee_rate| fee_rate as f32).unwrap_or_else(|| { + self.get_fee_rate(ConfirmationTarget::Normal) + .as_sat_per_vb() + }); + + // Get temporarily reserved UTXOs from in-memory storage. let mut reserved_outpoints = self.locked_outpoints.lock(); - // filter reserved utxos from all known utxos to not accidentally double spend and those who - // have actually been spent already + let utxos = self.get_utxos()?; + + // Filter out reserved and spent UTXOs to prevent double-spending attempts. let utxos = utxos .iter() .filter(|utxo| !reserved_outpoints.contains(&utxo.outpoint)) .filter(|utxo| !utxo.is_spent) .collect::>(); - let mut utxos = utxos - .into_iter() - .map(|x| UtxoWrap { - utxo: Utxo { - tx_out: x.txout.clone(), - outpoint: x.outpoint, - address: Address::from_script(&x.txout.script_pubkey, network) - .expect("to be a valid address"), - redeem_script: Default::default(), - reserved: false, - }, + let candidates = utxos + .iter() + .map(|utxo| { + let tx_in = TxIn { + previous_output: utxo.outpoint, + ..Default::default() + }; + + // Inspired by `rust-bitcoin:0.30.2`. + let segwit_weight = { + let legacy_weight = { + let script_sig_size = tx_in.script_sig.len(); + (36 + VarInt(script_sig_size as u64).len() + script_sig_size + 4) * 4 + }; + + legacy_weight + tx_in.witness.serialized_len() + }; + + // The 10101 wallet always generates SegWit addresses. + // + // TODO: Rework this once we use Taproot. + let is_witness_program = true; + + Candidate::new(utxo.txout.value, segwit_weight as u32, is_witness_program) }) .collect::>(); - // select enough utxos for our needs - let selected_local_utxos = select_coins(amount, 20, &mut utxos); - match selected_local_utxos { - None => Ok(vec![]), - Some(selected_local_utxos) => { - // update our temporarily reserved utxos with the selected once. - // note: this storage is only cleared up on a restart, meaning, if the protocol - // fails later on, the utxos will remain reserved - if lock_utxos { - for utxo in selected_local_utxos.clone() { - reserved_outpoints.push(utxo.utxo.outpoint); - } - } + // This is a standard base weight (without inputs or change outputs) for on-chain DLCs. We + // assume that this value is still correct for DLC channels. + let funding_tx_base_weight = 212; + + let target = Target { + feerate: bdk_coin_select::FeeRate::from_sat_per_vb(fee_rate), + min_fee: 0, + value: amount, + }; + + let mut coin_selector = CoinSelector::new(&candidates, funding_tx_base_weight); - Ok(selected_local_utxos - .into_iter() - .map(|utxo| utxo.utxo) - .collect()) + let dust_limit = 0; + let long_term_feerate = bdk_coin_select::FeeRate::default_min_relay_fee(); + + let change_policy = ChangePolicy::min_value_and_waste( + DrainWeights::default(), + dust_limit, + target.feerate, + long_term_feerate, + ); + + let metric = LowestFee { + target, + long_term_feerate, + change_policy, + }; + + coin_selector + .run_bnb(metric, 100_000) + .context("Failed to select coins")?; + + debug_assert!(coin_selector.is_target_met(target)); + + let indices = coin_selector.selected_indices(); + + let mut selected_utxos: Vec = Vec::with_capacity(indices.len()); + for index in indices { + let utxo = &utxos[*index]; + + let address = Address::from_script(&utxo.txout.script_pubkey, network) + .expect("to be a valid address"); + + let utxo = Utxo { + tx_out: utxo.txout.clone(), + outpoint: utxo.outpoint, + address, + redeem_script: Script::new(), + reserved: false, + }; + + if should_lock_utxos { + // Add selected UTXOs to reserve to prevent future double-spend attempts. + reserved_outpoints.push(utxo.outpoint); } + + selected_utxos.push(utxo); } + + Ok(selected_utxos) } /// Build the PSBT for sending funds to a given script and signs it diff --git a/crates/ln-dlc-node/src/ln_dlc_wallet.rs b/crates/ln-dlc-node/src/ln_dlc_wallet.rs index 586aeeb98..bf007c3f8 100644 --- a/crates/ln-dlc-node/src/ln_dlc_wallet.rs +++ b/crates/ln-dlc-node/src/ln_dlc_wallet.rs @@ -122,6 +122,10 @@ impl LnDlcWallet { self.address_cache.read().clone() } + pub fn new_address(&self) -> Result
{ + self.ldk_wallet().get_new_address() + } + pub fn is_mine(&self, script: &Script) -> Result { self.ldk_wallet().is_mine(script) } @@ -270,23 +274,20 @@ impl dlc_manager::Wallet for LnDlcWallet Ok(sk) } + // This is only used to create the funding transaction of a DLC or a DLC channel. fn get_utxos_for_amount( &self, amount: u64, - _: Option, + fee_rate: Option, lock_utxos: bool, ) -> Result, Error> { let utxos = self .ldk_wallet() - .get_utxos_for_amount(amount, lock_utxos, self.network) + .get_utxos_for_dlc_funding_transaction(amount, fee_rate, lock_utxos) .map_err(|error| { - Error::InvalidState(format!("Could not find utxos for mount {error:?}")) + Error::InvalidState(format!("Could not find utxos for amount: {error:?}")) })?; - if utxos.is_empty() { - return Err(Error::InvalidState( - "Not enough UTXOs for amount".to_string(), - )); - } + Ok(utxos) } diff --git a/crates/ln-dlc-node/src/node/wallet.rs b/crates/ln-dlc-node/src/node/wallet.rs index 259f4fafb..c060c13ed 100644 --- a/crates/ln-dlc-node/src/node/wallet.rs +++ b/crates/ln-dlc-node/src/node/wallet.rs @@ -59,6 +59,12 @@ impl Node { self.wallet.unused_address() } + pub fn get_new_address(&self) -> Result
{ + self.wallet + .new_address() + .context("Failed to get new address") + } + pub fn get_blockchain_height(&self) -> Result { self.wallet .get_blockchain_height() diff --git a/crates/tests-e2e/src/bitcoind.rs b/crates/tests-e2e/src/bitcoind.rs index 1bb649a8c..f3a533281 100644 --- a/crates/tests-e2e/src/bitcoind.rs +++ b/crates/tests-e2e/src/bitcoind.rs @@ -5,6 +5,7 @@ use bitcoin::Amount; use reqwest::Client; use reqwest::Response; use serde::Deserialize; +use serde_json::json; use std::time::Duration; /// A wrapper over the bitcoind HTTP API @@ -27,12 +28,7 @@ impl Bitcoind { /// Instructs `bitcoind` to generate to address. pub async fn mine(&self, n: u16) -> Result<()> { - #[derive(Deserialize, Debug)] - struct BitcoindResponse { - result: String, - } - - let response: BitcoindResponse = self + let response: GetNewAddressResponse = self .client .post(&self.host) .body(r#"{"jsonrpc": "1.0", "method": "getnewaddress", "params": []}"#.to_string()) @@ -75,6 +71,106 @@ impl Bitcoind { Ok(response) } + pub async fn send_multiple_utxos_to_address( + &self, + address_fn: F, + utxo_amount: Amount, + n_utxos: u64, + ) -> Result<()> + where + F: Fn() -> Address, + { + let total_amount = utxo_amount * n_utxos; + + let response: ListUnspentResponse = self + .client + .post(&self.host) + .body(r#"{"jsonrpc": "1.0", "method": "listunspent", "params": []}"#) + .send() + .await? + .json() + .await?; + + let utxo = response + .result + .iter() + // We try to find one UTXO that can cover the whole transaction. We could cover the + // amount with multiple UTXOs too, but this is simpler and will probably succeed. + .find(|utxo| utxo.spendable && utxo.amount >= total_amount) + .expect("to find UTXO to cover multi-payment"); + + let mut outputs = serde_json::value::Map::new(); + + for _ in 0..n_utxos { + let address = address_fn(); + outputs.insert(address.to_string(), json!(utxo_amount.to_btc())); + } + + let create_raw_tx_request = json!( + { + "jsonrpc": "1.0", + "method": "createrawtransaction", + "params": + [ + [ {"txid": utxo.txid, "vout": utxo.vout} ], + outputs + ] + } + ); + + let create_raw_tx_response: CreateRawTransactionResponse = self + .client + .post(&self.host) + .json(&create_raw_tx_request) + .send() + .await? + .json() + .await?; + + let sign_raw_tx_with_wallet_request = json!( + { + "jsonrpc": "1.0", + "method": "signrawtransactionwithwallet", + "params": [ create_raw_tx_response.result ] + } + ); + + let sign_raw_tx_with_wallet_response: SignRawTransactionWithWalletResponse = self + .client + .post(&self.host) + .json(&sign_raw_tx_with_wallet_request) + .send() + .await? + .json() + .await?; + + let send_raw_tx_request = json!( + { + "jsonrpc": "1.0", + "method": "sendrawtransaction", + "params": [ sign_raw_tx_with_wallet_response.result.hex, 0 ] + } + ); + + let send_raw_tx_response: SendRawTransactionResponse = self + .client + .post(&self.host) + .json(&send_raw_tx_request) + .send() + .await? + .json() + .await?; + + tracing::info!( + txid = %send_raw_tx_response.result, + %utxo_amount, + %n_utxos, + "Published multi-utxo transaction" + ); + + Ok(()) + } + pub async fn post(&self, endpoint: &str, body: Option) -> Result { let mut builder = self.client.post(endpoint.to_string()); if let Some(body) = body { @@ -88,3 +184,42 @@ impl Bitcoind { Ok(response) } } + +#[derive(Deserialize, Debug)] +struct GetNewAddressResponse { + result: String, +} + +#[derive(Deserialize, Debug)] +struct ListUnspentResponse { + result: Vec, +} + +#[derive(Deserialize, Debug)] +struct Utxo { + txid: String, + vout: usize, + #[serde(with = "bitcoin::util::amount::serde::as_btc")] + amount: Amount, + spendable: bool, +} + +#[derive(Deserialize, Debug)] +struct CreateRawTransactionResponse { + result: String, +} + +#[derive(Deserialize, Debug)] +struct SignRawTransactionWithWalletResponse { + result: SignRawTransactionWithWalletResponseBody, +} + +#[derive(Deserialize, Debug)] +struct SignRawTransactionWithWalletResponseBody { + hex: String, +} + +#[derive(Deserialize, Debug)] +struct SendRawTransactionResponse { + result: String, +} diff --git a/crates/tests-e2e/src/setup.rs b/crates/tests-e2e/src/setup.rs index 0e3ae46e6..28338ab4d 100644 --- a/crates/tests-e2e/src/setup.rs +++ b/crates/tests-e2e/src/setup.rs @@ -22,8 +22,7 @@ pub struct TestSetup { } impl TestSetup { - /// Start test with a running app and a funded wallet. - pub async fn new_after_funding(fund_amount: Option) -> Self { + pub async fn new() -> Self { init_tracing(); let client = init_reqwest(); @@ -35,17 +34,6 @@ impl TestSetup { assert!(coordinator.is_running().await); - // Ensure that the coordinator has a free UTXO available. - let address = coordinator.get_new_address().await.unwrap(); - - bitcoind - .send_to_address(&address, Amount::ONE_BTC) - .await - .unwrap(); - - bitcoind.mine(1).await.unwrap(); - coordinator.sync_node().await.unwrap(); - // App setup let app = run_app(None).await; @@ -62,37 +50,65 @@ impl TestSetup { "App should start with empty off-chain wallet" ); - let fund_amount = fund_amount.unwrap_or(Amount::ONE_BTC); + Self { + app, + coordinator, + bitcoind, + } + } + + pub async fn fund_coordinator(&self, amount: Amount) { + // Ensure that the coordinator has a free UTXO available. + let address = self.coordinator.get_new_address().await.unwrap(); + + self.bitcoind + .send_to_address(&address, amount) + .await + .unwrap(); + self.bitcoind.mine(1).await.unwrap(); + + self.sync_coordinator().await; + + // TODO: Get coordinator balance to verify this claim. + tracing::info!("Successfully funded coordinator"); + } + + pub async fn fund_app(&self, fund_amount: Amount) { let address = api::get_unused_address(); let address = &address.0.parse().unwrap(); - bitcoind + self.bitcoind .send_to_address(address, fund_amount) .await .unwrap(); - bitcoind.mine(1).await.unwrap(); + self.bitcoind.mine(1).await.unwrap(); wait_until!({ refresh_wallet_info(); - app.rx.wallet_info().unwrap().balances.on_chain == fund_amount.to_sat() + self.app.rx.wallet_info().unwrap().balances.on_chain >= fund_amount.to_sat() }); - let on_chain_balance = app.rx.wallet_info().unwrap().balances.on_chain; + let on_chain_balance = self.app.rx.wallet_info().unwrap().balances.on_chain; tracing::info!(%fund_amount, %on_chain_balance, "Successfully funded app"); + } - Self { - app, - coordinator, - bitcoind, - } + /// Start test with a running app and a funded wallet. + pub async fn new_after_funding() -> Self { + let setup = Self::new().await; + + setup.fund_coordinator(Amount::ONE_BTC).await; + + setup.fund_app(Amount::ONE_BTC).await; + + setup } /// Start test with a running app with a funded wallet and an open position. pub async fn new_with_open_position() -> Self { - let setup = Self::new_after_funding(None).await; + let setup = Self::new_after_funding().await; let rx = &setup.app.rx; tracing::info!("Opening a position"); @@ -119,10 +135,16 @@ impl TestSetup { sync_dlc_channels(); refresh_wallet_info(); - setup.coordinator.sync_node().await.unwrap(); + setup.sync_coordinator().await; setup } + + async fn sync_coordinator(&self) { + if let Err(e) = self.coordinator.sync_node().await { + tracing::error!("Got error from coordinator sync: {e:#}"); + }; + } } pub fn dummy_order() -> NewOrder { diff --git a/crates/tests-e2e/tests/basic.rs b/crates/tests-e2e/tests/basic.rs index f3d8f281e..53afb9d43 100644 --- a/crates/tests-e2e/tests/basic.rs +++ b/crates/tests-e2e/tests/basic.rs @@ -4,7 +4,7 @@ use tests_e2e::setup::TestSetup; #[tokio::test(flavor = "multi_thread")] #[ignore = "need to be run with 'just e2e' command"] async fn app_can_be_funded_with_bitcoind() -> Result<()> { - TestSetup::new_after_funding(None).await; + TestSetup::new_after_funding().await; Ok(()) } diff --git a/crates/tests-e2e/tests/open_position.rs b/crates/tests-e2e/tests/open_position.rs index c5a53eb0a..4b555d483 100644 --- a/crates/tests-e2e/tests/open_position.rs +++ b/crates/tests-e2e/tests/open_position.rs @@ -9,24 +9,20 @@ use tests_e2e::setup::TestSetup; use tests_e2e::wait_until; use tokio::task::spawn_blocking; -fn dummy_order() -> NewOrder { - NewOrder { +#[tokio::test(flavor = "multi_thread")] +#[ignore = "need to be run with 'just e2e' command"] +async fn can_open_position() { + let test = TestSetup::new_after_funding().await; + let app = &test.app; + + let order = NewOrder { leverage: 2.0, contract_symbol: ContractSymbol::BtcUsd, direction: api::Direction::Long, quantity: 1.0, order_type: Box::new(OrderType::Market), stable: false, - } -} - -#[tokio::test(flavor = "multi_thread")] -#[ignore = "need to be run with 'just e2e' command"] -async fn can_open_position() { - let test = TestSetup::new_after_funding(None).await; - let app = &test.app; - - let order = dummy_order(); + }; spawn_blocking({ let order = order.clone(); move || api::submit_order(order).unwrap() diff --git a/crates/tests-e2e/tests/open_position_small_utxos.rs b/crates/tests-e2e/tests/open_position_small_utxos.rs new file mode 100644 index 000000000..86e23e58d --- /dev/null +++ b/crates/tests-e2e/tests/open_position_small_utxos.rs @@ -0,0 +1,94 @@ +use bitcoin::Amount; +use native::api; +use native::api::calculate_margin; +use native::api::ContractSymbol; +use native::trade::order::api::NewOrder; +use native::trade::order::api::OrderType; +use native::trade::position::PositionState; +use rust_decimal::prelude::ToPrimitive; +use std::str::FromStr; +use tests_e2e::app::refresh_wallet_info; +use tests_e2e::setup::TestSetup; +use tests_e2e::wait_until; +use tokio::task::spawn_blocking; + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "need to be run with 'just e2e' command"] +async fn can_open_position_with_multiple_small_utxos() { + // Arrange + + let setup = TestSetup::new().await; + + setup.fund_coordinator(Amount::ONE_BTC).await; + + let app = &setup.app; + + // Fund app with multiple small UTXOs that can cover the required margin. + + let order = NewOrder { + leverage: 2.0, + contract_symbol: ContractSymbol::BtcUsd, + direction: api::Direction::Long, + quantity: 100.0, + order_type: Box::new(OrderType::Market), + stable: false, + }; + + // We take the ask price because the app is going long. + let ask_price = app + .rx + .prices() + .unwrap() + .get(&ContractSymbol::BtcUsd) + .unwrap() + .ask + .unwrap() + .to_f32() + .unwrap(); + + let margin_app = calculate_margin(ask_price, order.quantity, order.leverage).0; + + // We want to use small UTXOs. + let utxo_size = 1_000; + + let n_utxos = margin_app / utxo_size; + + // Double the number of UTXOs to cover costs beyond the margin i.e. fees. + let n_utxos = 2 * n_utxos; + + let address_fn = || bitcoin::Address::from_str(&api::get_new_address().unwrap()).unwrap(); + + setup + .bitcoind + .send_multiple_utxos_to_address(address_fn, Amount::from_sat(utxo_size), n_utxos) + .await + .unwrap(); + + let fund_amount = n_utxos * utxo_size; + + setup.bitcoind.mine(1).await.unwrap(); + + wait_until!({ + refresh_wallet_info(); + app.rx.wallet_info().unwrap().balances.on_chain >= fund_amount + }); + + // Act + + spawn_blocking({ + let order = order.clone(); + move || api::submit_order(order).unwrap() + }) + .await + .unwrap(); + + // Assert + + wait_until!(matches!( + app.rx.position(), + Some(native::trade::position::Position { + position_state: PositionState::Open, + .. + }) + )); +} diff --git a/crates/tests-e2e/tests/reject_offer.rs b/crates/tests-e2e/tests/reject_offer.rs index 3bcc8c371..8f1e0890c 100644 --- a/crates/tests-e2e/tests/reject_offer.rs +++ b/crates/tests-e2e/tests/reject_offer.rs @@ -14,7 +14,10 @@ use tokio::task::spawn_blocking; #[tokio::test(flavor = "multi_thread")] #[ignore = "need to be run with 'just e2e' command"] async fn reject_offer() { - let test = TestSetup::new_after_funding(Some(Amount::from_sat(250_000))).await; + let test = TestSetup::new().await; + test.fund_coordinator(Amount::ONE_BTC).await; + test.fund_app(Amount::from_sat(250_000)).await; + let app = &test.app; let invalid_order = NewOrder { diff --git a/mobile/native/src/api.rs b/mobile/native/src/api.rs index 2748094bf..af6738289 100644 --- a/mobile/native/src/api.rs +++ b/mobile/native/src/api.rs @@ -405,6 +405,10 @@ pub fn get_unused_address() -> SyncReturn { SyncReturn(ln_dlc::get_unused_address()) } +pub fn get_new_address() -> Result { + ln_dlc::get_new_address() +} + #[tokio::main(flavor = "current_thread")] pub async fn close_channel() -> Result<()> { ln_dlc::close_channel(false).await diff --git a/mobile/native/src/ln_dlc/mod.rs b/mobile/native/src/ln_dlc/mod.rs index 749c367dc..8ad652571 100644 --- a/mobile/native/src/ln_dlc/mod.rs +++ b/mobile/native/src/ln_dlc/mod.rs @@ -697,6 +697,12 @@ pub fn get_unused_address() -> String { state::get_node().inner.get_unused_address().to_string() } +pub fn get_new_address() -> Result { + let address = state::get_node().inner.get_new_address()?; + + Ok(address.to_string()) +} + pub async fn close_channel(is_force_close: bool) -> Result<()> { tracing::info!(force = is_force_close, "Offering to close a channel"); let node = state::try_get_node().context("failed to get ln dlc node")?;