From 8760067f05e3bbf9e2a54f855c6b90e8790b92e7 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 22 Dec 2023 18:17:39 +0100 Subject: [PATCH 001/125] Add progress watch channel to TransferContext --- exe-unit/components/transfer/src/lib.rs | 20 +++++++++++++++++-- exe-unit/components/transfer/src/progress.rs | 10 ++++++++++ exe-unit/components/transfer/src/transfer.rs | 17 ++++++++++++++-- .../components/transfer/tests/test_deploy.rs | 2 ++ .../transfer/tests/test_transfer_abort.rs | 1 + .../transfer/tests/test_transfer_resume.rs | 1 + .../transfer/tests/test_transfer_service.rs | 3 +++ exe-unit/src/lib.rs | 6 +++++- 8 files changed, 55 insertions(+), 5 deletions(-) diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index 34e055680a..ffc45e133c 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -15,6 +15,7 @@ mod traverse; use std::cell::RefCell; use std::pin::Pin; use std::rc::Rc; +use std::sync::Arc; use std::time::Duration; use bytes::Bytes; @@ -38,6 +39,8 @@ pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; use crate::hash::with_hash_stream; +use crate::progress::progress_report_channel; +use crate::transfer::Progress; use ya_client_model::activity::TransferArgs; /// Transfers data from `stream` to a `TransferSink` @@ -73,7 +76,7 @@ where log::debug!("Transferring from offset: {}", ctx.state.offset()); let stream = with_hash_stream(src.source(&src_url.url, ctx), src_url, dst_url, ctx)?; - let sink = dst.destination(&dst_url.url, ctx); + let sink = progress_report_channel(dst.destination(&dst_url.url, ctx), ctx); transfer(stream, sink).await?; Ok::<_, Error>(()) @@ -296,6 +299,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, + pub report: Arc>>>, } impl TransferContext { @@ -304,7 +308,19 @@ impl TransferContext { let state = TransferState::default(); state.set_offset(offset); - Self { args, state } + Self { + args, + state, + report: Arc::new(std::sync::Mutex::new(None)), + } + } + + pub fn register_reporter(&self, report: tokio::sync::watch::Sender) { + *self.report.lock().unwrap() = Some(report); + } + + pub fn take_reporter(&self) -> Option> { + self.report.lock().unwrap().take() } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 71edaa7419..5c0ba866f5 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -1,4 +1,5 @@ use crate::error::Error; +use crate::transfer::Progress; use crate::{abortable_sink, abortable_stream, TransferSink, TransferStream}; use crate::{TransferContext, TransferData}; use futures::{SinkExt, StreamExt, TryFutureExt}; @@ -48,6 +49,15 @@ where type Sink = TransferSink; +pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { + let report = ctx.take_reporter(); + wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { + if let Some(report) = &report { + report.send(Progress { progress, size }).ok(); + } + }) +} + /// Wraps a sink to report progress. /// The `report` function is called with the current offset and the total size. /// The total size is 0 if the size is unknown. (For example, when the source is a directory.) diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index bc24672a44..4f33688a02 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -35,12 +35,22 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug, Message)] +#[derive(Clone, Debug)] +pub struct Progress { + pub progress: u64, + pub size: Option, +} + +#[derive(Debug, Message)] #[rtype(result = "Result<()>")] pub struct TransferResource { pub from: String, pub to: String, pub args: TransferArgs, + + /// Channel for watching for transfer progress. `None` means that there + /// will be no progress updates. + pub progress: Option>, } #[derive(Message)] @@ -53,10 +63,13 @@ impl AddVolumes { } } -#[derive(Clone, Debug, Message)] +#[derive(Debug, Message)] #[rtype(result = "Result>")] pub struct DeployImage { pub task_package: Option, + /// Channel for watching for deploy progress. `None` means that there + /// will be no progress updates. + pub progress: Option>, } #[derive(Clone, Debug, Message)] diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 53dab763a3..96ed3ed77e 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -67,6 +67,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re let result = addr .send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await?; log::info!("Deployment stopped"); @@ -76,6 +77,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re log::info!("Re-deploying the same image"); addr.send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index 69e864f2d7..63f167d3a2 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -38,6 +38,7 @@ async fn interrupted_transfer( from: src.to_owned(), to: dest.to_owned(), args: TransferArgs::default(), + progress: None, }) .await?; diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 2986cc4471..9b561d2ef2 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -113,6 +113,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, + progress: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index a96e6f7a00..e92d5809b3 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -31,6 +31,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, + progress: None, }) .await??; @@ -98,6 +99,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment with hash verification"); addr.send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await??; log::warn!("Deployment complete"); @@ -106,6 +108,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment from cache"); addr.send(DeployImage { task_package: task_package.clone(), + progress: None, }) .await??; log::warn!("Deployment from cache complete"); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index d92d97eebc..56a70f27ac 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -308,13 +308,17 @@ impl RuntimeRef { from: from.clone(), to: to.clone(), args: args.clone(), + progress: None, }; transfer_service.send(msg).await??; } ExeScriptCommand::Deploy { net, hosts } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. let task_package = transfer_service - .send(DeployImage { task_package: None }) + .send(DeployImage { + task_package: None, + progress: None, + }) .await??; runtime .send(UpdateDeployment { From 732872817558ac11c4f6b81cc39c345bc2b251bc Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 14:16:14 +0100 Subject: [PATCH 002/125] Register progress reporter for transfers and deploy --- exe-unit/components/transfer/src/lib.rs | 4 ++-- exe-unit/components/transfer/src/transfer.rs | 10 +++++++--- exe-unit/components/transfer/tests/test_deploy.rs | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index ffc45e133c..b30befac1c 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -315,8 +315,8 @@ impl TransferContext { } } - pub fn register_reporter(&self, report: tokio::sync::watch::Sender) { - *self.report.lock().unwrap() = Some(report); + pub fn register_reporter(&self, report: Option>) { + *self.report.lock().unwrap() = report; } pub fn take_reporter(&self) -> Option> { diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 4f33688a02..4c27964b94 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -35,7 +35,7 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct Progress { pub progress: u64, pub size: Option, @@ -173,6 +173,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, + progress: Option>, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) @@ -194,6 +195,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, + progress: Option>, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); @@ -206,6 +208,7 @@ impl TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.deploy_retry.clone()); + ctx.register_reporter(progress); // Using partially downloaded image from previous executions could speed up deploy // process, but it comes with the cost: If image under URL changed, Requestor will get @@ -283,10 +286,10 @@ impl Handler for TransferService { log::info!("Deploying from {:?} to {:?}", src_url.url, path); #[cfg(not(feature = "sgx"))] - return self.deploy_no_sgx(src_url, src_name, path); + return self.deploy_no_sgx(src_url, src_name, path, deploy.progress); #[cfg(feature = "sgx")] - return self.deploy_sgx(src_url, src_name, path); + return self.deploy_sgx(src_url, src_name, path, deploy.progress); } } @@ -301,6 +304,7 @@ impl Handler for TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); + ctx.register_reporter(msg.progress); let (abort, reg) = Abort::new_pair(); diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 96ed3ed77e..25e3d145c2 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -17,7 +17,7 @@ use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, Transf #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("deploy-restart")?; let temp_dir = dir.path(); From 6c6f3f1556b987b4589ef40bd5e447dd7eece3c1 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 14:53:03 +0100 Subject: [PATCH 003/125] Fix http tranfer provider to always ask server for file size --- exe-unit/components/transfer/src/http.rs | 11 +-- .../components/transfer/tests/test_deploy.rs | 88 +++++++++++++++++-- 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/exe-unit/components/transfer/src/http.rs b/exe-unit/components/transfer/src/http.rs index 97bb1b2026..03e1acfd7e 100644 --- a/exe-unit/components/transfer/src/http.rs +++ b/exe-unit/components/transfer/src/http.rs @@ -119,10 +119,6 @@ impl TransferProvider for HttpTransferProvider { url: &Url, ctx: &TransferContext, ) -> LocalBoxFuture<'a, Result<(), Error>> { - if ctx.state.offset() == 0 { - return futures::future::ok(()).boxed_local(); - } - let url = url.clone(); let state = ctx.state.clone(); @@ -137,8 +133,13 @@ impl TransferProvider for HttpTransferProvider { .get(header::CONTENT_LENGTH) .and_then(|v| v.to_str().ok().and_then(|s| u64::from_str(s).ok())); + match &size { + None => log::info!("File size unknown. Http source server didn't respond with CONTENT_LENGTH header."), + Some(size) => log::info!("Http source size reported by server: {size} B"), + }; + state.set_size(size); - if !ranges { + if state.offset() != 0 && !ranges { log::warn!("Transfer resuming is not supported by the server"); state.set_offset(0); } diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 25e3d145c2..9fe2ad70db 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -9,7 +9,9 @@ use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; +use ya_transfer::transfer::{ + AbortTransfers, DeployImage, Progress, TransferService, TransferServiceContext, +}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -31,10 +33,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re std::fs::create_dir_all(dir)?; } - let chunk_size = 4096_usize; - let chunk_count = 1024 * 10; - - let hash = generate_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 1024 * 10); log::debug!("Starting HTTP servers"); let path = temp_dir.to_path_buf(); @@ -83,3 +82,82 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re Ok(()) } + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("deploy-restart")?; + let temp_dir = dir.path(); + + log::debug!("Creating directories in: {}", temp_dir.display()); + let work_dir = temp_dir.join("work_dir"); + let cache_dir = temp_dir.join("cache_dir"); + let sub_dir = temp_dir.join("sub_dir"); + + for dir in vec![work_dir.clone(), cache_dir.clone(), sub_dir.clone()] { + std::fs::create_dir_all(dir)?; + } + + let chunk_size = 4096_usize; + let chunk_count = 1024 * 1; + let file_size = (chunk_size * chunk_count) as u64; + let hash = generate_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + + log::debug!("Starting HTTP servers"); + let path = temp_dir.to_path_buf(); + start_http(ctx, path) + .await + .expect("unable to start http servers"); + + let task_package = Some(format!( + "hash://sha3:{}:http://127.0.0.1:8001/rnd", + hex::encode(hash) + )); + + log::debug!("Starting TransferService"); + let exe_ctx = TransferServiceContext { + work_dir: work_dir.clone(), + cache_dir, + ..TransferServiceContext::default() + }; + let addr = TransferService::new(exe_ctx).start(); + + log::info!("[>>] Deployment with hash verification"); + let (tx, mut rx) = tokio::sync::watch::channel::(Progress::default()); + + tokio::task::spawn_local(async move { + let _result = addr + .send(DeployImage { + task_package: task_package.clone(), + progress: Some(tx), + }) + .await??; + log::info!("Deployment stopped"); + anyhow::Ok(()) + }); + + let mut last_progress = 0u64; + loop { + if rx.changed().await.is_err() { + break; + } + + let progress = rx.borrow_and_update(); + + assert_eq!(progress.size.unwrap(), file_size); + assert!(progress.progress >= last_progress); + + last_progress = progress.progress; + + log::info!( + "Progress: {}/{}", + progress.progress, + progress.size.unwrap_or(0) + ); + } + + Ok(()) +} From ffc31ffe9629e192278560cda51bf8adcd423f51 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 15:10:41 +0100 Subject: [PATCH 004/125] Prettify progress loop in test --- exe-unit/components/transfer/tests/test_deploy.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 9fe2ad70db..323bb318a6 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -87,7 +87,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("deploy-restart")?; let temp_dir = dir.path(); @@ -140,11 +140,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< }); let mut last_progress = 0u64; - loop { - if rx.changed().await.is_err() { - break; - } - + while let Ok(_) = rx.changed().await { let progress = rx.borrow_and_update(); assert_eq!(progress.size.unwrap(), file_size); From 1428ffe17b5ebf679863fb84576a6e8a7fe97c9a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 2 Jan 2024 16:38:26 +0100 Subject: [PATCH 005/125] Progress struct is serializable --- exe-unit/components/transfer/src/transfer.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 4c27964b94..615dd17bf1 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -5,6 +5,7 @@ use std::rc::Rc; use actix::prelude::*; use futures::future::Abortable; +use serde::{Deserialize, Serialize}; use url::Url; use crate::cache::{Cache, CachePath}; @@ -35,7 +36,7 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct Progress { pub progress: u64, pub size: Option, From 319c7bf3b6d9806fd9e567242c56779819e0513a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 3 Jan 2024 17:54:22 +0100 Subject: [PATCH 006/125] Move CommandProgress to ya-client-model --- Cargo.lock | 478 ++++++++++-------- Cargo.toml | 8 +- agent/provider/Cargo.toml | 4 +- core/activity/Cargo.toml | 2 +- core/gsb-api/Cargo.toml | 2 +- core/identity/Cargo.toml | 2 +- core/market/Cargo.toml | 2 +- core/model/Cargo.toml | 2 +- core/net/Cargo.toml | 2 +- core/payment-driver/base/Cargo.toml | 2 +- core/payment-driver/erc20/Cargo.toml | 2 +- core/payment-driver/erc20next/Cargo.toml | 2 +- core/payment/Cargo.toml | 4 +- core/persistence/Cargo.toml | 2 +- core/serv-api/web/Cargo.toml | 2 +- core/sgx/Cargo.toml | 2 +- core/version/Cargo.toml | 2 +- core/vpn/Cargo.toml | 4 +- exe-unit/Cargo.toml | 2 +- exe-unit/components/transfer/Cargo.toml | 2 +- exe-unit/components/transfer/src/lib.rs | 16 +- exe-unit/components/transfer/src/progress.rs | 6 +- exe-unit/components/transfer/src/transfer.rs | 14 +- .../components/transfer/tests/test_deploy.rs | 4 +- exe-unit/src/state.rs | 1 + golem_cli/Cargo.toml | 2 +- .../test-framework/framework-basic/Cargo.toml | 2 +- utils/agreement-utils/Cargo.toml | 2 +- utils/manifest-utils/Cargo.toml | 2 +- 29 files changed, 303 insertions(+), 274 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b84e7e8269..fd96f033fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ "actix_derive", "bitflags 2.4.1", "bytes 1.5.0", - "crossbeam-channel 0.5.9", + "crossbeam-channel 0.5.10", "futures-core", "futures-sink", "futures-task", @@ -84,16 +84,16 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.4.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92ef85799cba03f76e4f7c10f533e66d87c9a7e7055f3391f09000ad8351bc9" +checksum = "129d4c88e98860e1758c5de288d1632b07970a16d59bdf7b8d66053d582bb71f" dependencies = [ "actix-codec", "actix-rt", "actix-service", "actix-tls", "actix-utils", - "ahash 0.8.6", + "ahash 0.8.7", "base64 0.21.5", "bitflags 2.4.1", "brotli", @@ -154,14 +154,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] name = "actix-router" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", "http", @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.4.0" +version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4a5b5e29603ca8c94a77c65cf874718ceb60292c5a5c3e5f4ace041af462b9" +checksum = "e43428f3bf11dee6d166b00ec2df4e3aa8cc1606aaa0b7433c146852e2f4e03b" dependencies = [ "actix-codec", "actix-http", @@ -284,7 +284,7 @@ dependencies = [ "actix-tls", "actix-utils", "actix-web-codegen", - "ahash 0.8.6", + "ahash 0.8.7", "bytes 1.5.0", "bytestring", "cfg-if 1.0.0", @@ -336,7 +336,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if 1.0.0", "getrandom 0.2.11", @@ -515,9 +515,9 @@ checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" [[package]] name = "anyhow" -version = "1.0.76" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59d2a3357dde987206219e78ecfbbb6e8dad06cbb65292758d3270e6254f7355" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "appdirs" @@ -597,13 +597,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.75" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf6721fb0140e4f897002dd086c06f6c27775df19cfe1fccb21181a48fd2c98" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -652,9 +652,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "awc" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fa3c705a9c7917ac0f41c0757a0a747b43bbc29b0b364b081bd7c5fc67fb223" +checksum = "b625cad34428b3b82d0bd548b26a1cd0a3d70b6109e9b4e3355d8f1802a8b1c6" dependencies = [ "actix-codec", "actix-http", @@ -941,7 +941,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "syn_derive", ] @@ -968,9 +968,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +checksum = "c48f0051a4b4c5e0b6d365cd04af53aeaa209e3cc15ec2cdb69e73cc87fbd0dc" dependencies = [ "memchr", "regex-automata", @@ -1333,12 +1333,12 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c3242926edf34aec4ac3a77108ad4854bffaa2e4ddc1824124ce59231302d5" +checksum = "82a9b73a36529d9c47029b9fb3a6f0ea3cc916a261195352ba19e770fc1748b2" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.17", + "crossbeam-utils 0.8.18", ] [[package]] @@ -1380,12 +1380,12 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9bcf5bdbfdd6030fb4a1c497b5d5fc5921aa2f60d359a17e249c0e6df3de153" +checksum = "adc6598521bb5a83d491e8c1fe51db7296019d2ca3cb93cc6c2a20369a4d78a2" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.17", + "crossbeam-utils 0.8.18", ] [[package]] @@ -1401,9 +1401,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d96137f14f244c37f989d9fff8f95e6c18b918e71f36638f8c49112e4c78f" +checksum = "c3a430a770ebd84726f584a90ee7f020d28db52c6d02138900f22341f866d39c" dependencies = [ "cfg-if 1.0.0", ] @@ -1517,12 +1517,12 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e95fbd621905b854affdc67943b043a0fbb6ed7385fd5a25650d19a8a6cfdf" +checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" dependencies = [ "nix 0.27.1", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1612,7 +1612,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -1914,7 +1914,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -1927,7 +1927,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -1986,7 +1986,7 @@ dependencies = [ "erc20_payment_lib_common", "erc20_rpc_pool", "fastrand", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "hex", "humantime 2.1.0", @@ -2025,7 +2025,7 @@ dependencies = [ "chrono", "dotenv", "fastrand", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "hex", "humantime 2.1.0", @@ -2070,7 +2070,7 @@ dependencies = [ "env_logger 0.10.1", "erc20_payment_lib_common", "fastrand", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "hex", "humantime 2.1.0", @@ -2508,9 +2508,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -2523,9 +2523,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -2533,15 +2533,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -2561,32 +2561,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -2596,9 +2596,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures 0.1.31", "futures-channel", @@ -2672,7 +2672,7 @@ dependencies = [ "digest 0.8.1", "dotenv", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "log", "rand 0.8.5", "serde", @@ -2711,7 +2711,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -2752,7 +2752,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.5.0", ] [[package]] @@ -2769,7 +2769,7 @@ dependencies = [ "directories", "dotenv", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "libc", "log", @@ -2857,7 +2857,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.7", "allocator-api2", ] @@ -3125,9 +3125,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3350,13 +3350,13 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi 0.3.3", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -3425,7 +3425,7 @@ version = "18.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb" dependencies = [ - "futures 0.3.29", + "futures 0.3.30", "futures-executor", "futures-util", "log", @@ -3646,9 +3646,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -3703,8 +3703,8 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ - "ahash 0.8.6", - "metrics-macros 0.7.0", + "ahash 0.8.7", + "metrics-macros 0.7.1", "portable-atomic", ] @@ -3729,13 +3729,13 @@ dependencies = [ [[package]] name = "metrics-macros" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" +checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4228,9 +4228,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -4265,9 +4265,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.61" +version = "0.10.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" +checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" dependencies = [ "bitflags 2.4.1", "cfg-if 1.0.0", @@ -4286,7 +4286,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4306,9 +4306,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.97" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -4571,7 +4571,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4679,12 +4679,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -4779,9 +4779,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" dependencies = [ "unicode-ident", ] @@ -4882,7 +4882,7 @@ dependencies = [ "prost 0.12.3", "prost-types 0.12.3", "regex", - "syn 2.0.42", + "syn 2.0.46", "tempfile", "which", ] @@ -4923,7 +4923,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -5014,9 +5014,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -5479,7 +5479,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.20", + "semver 1.0.21", ] [[package]] @@ -5640,11 +5640,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5838,9 +5838,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" dependencies = [ "serde", ] @@ -5862,31 +5862,31 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.12" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" +checksum = "8b8497c313fd43ab992087548117643f6fcd935cbf36f176ffda0aacf9591734" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.194" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -5902,9 +5902,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" dependencies = [ "itoa", "ryu", @@ -5935,13 +5935,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" +checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -5985,9 +5985,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.28" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9269cfafc7e0257ee4a42f3f68a307f458c63d9e7c8ba4b58c5d15f1b7d7e8d3" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ "indexmap 2.1.0", "itoa", @@ -6023,7 +6023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92761393ee4dc3ff8f4af487bd58f4307c9329bbedea02cac0089ad9c411e153" dependencies = [ "dashmap", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.12.1", @@ -6037,7 +6037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "538c30747ae860d6fb88330addbbd3e0ddbe46d662d032855596d8a8ca260611" dependencies = [ "dashmap", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.12.1", @@ -6051,7 +6051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" dependencies = [ "dashmap", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.12.1", @@ -6109,7 +6109,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6376,7 +6376,7 @@ checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ "base64 0.13.1", "bytes 1.5.0", - "futures 0.3.29", + "futures 0.3.30", "httparse", "log", "rand 0.8.5", @@ -6438,13 +6438,13 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d6753e460c998bbd4cd8c6f0ed9a64346fcca0723d6e75e52fdc351c5d2169d" dependencies = [ - "ahash 0.8.6", + "ahash 0.8.7", "atoi", "byteorder", "bytes 1.5.0", "chrono", "crc", - "crossbeam-queue 0.3.9", + "crossbeam-queue 0.3.10", "dotenvy", "either", "event-listener", @@ -6692,6 +6692,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89a286a7e3b5720b9a477b23253bc50debac207c8d21505f8e70b36792f11b5" + [[package]] name = "strum" version = "0.24.1" @@ -6707,6 +6713,18 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +[[package]] +name = "strum_macros" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e61bb0be289045cb80bfce000512e32d09f8337e54c186725da381377ad1f8d5" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "strum_macros" version = "0.24.3" @@ -6730,7 +6748,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6752,9 +6770,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.42" +version = "2.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b7d0a2c048d661a1a59fcd7355baa232f7ed34e0ee4df2eef3c1c1c0d3852d8" +checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" dependencies = [ "proc-macro2", "quote", @@ -6770,7 +6788,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6845,15 +6863,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if 1.0.0", "fastrand", "redox_syscall 0.4.1", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -6928,7 +6946,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -6952,7 +6970,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "test-case-core", ] @@ -6963,7 +6981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "055831a02a4f5aa28fede67f2902014273eb8c21b958ac5ebbd59b71ef30dbc3" dependencies = [ "async-trait", - "futures 0.3.29", + "futures 0.3.30", "test-context-macros", ] @@ -6988,22 +7006,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -7103,7 +7121,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -7265,7 +7283,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -7652,7 +7670,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "wasm-bindgen-shared", ] @@ -7686,7 +7704,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7719,7 +7737,7 @@ dependencies = [ "derive_more", "ethabi", "ethereum-types 0.14.1", - "futures 0.3.29", + "futures 0.3.30", "futures-timer", "headers", "hex", @@ -7848,11 +7866,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -8055,9 +8073,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.30" +version = "0.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b5c3db89721d50d0e2a673f5043fc4722f76dcc352d7b1ab8b8288bed4ed2c5" +checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c" dependencies = [ "memchr", ] @@ -8099,9 +8117,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7dae5072fe1f8db8f8d29059189ac175196e410e40ba42d5d4684ae2f750995" +checksum = "914566e6413e7fa959cc394fb30e563ba80f3541fbd40816d4c05a0fc3f2a0f1" dependencies = [ "libc", "linux-raw-sys", @@ -8129,7 +8147,7 @@ dependencies = [ "diesel", "diesel_migrations", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "libsqlite3-sys", @@ -8145,7 +8163,7 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-net", "ya-persistence", @@ -8165,24 +8183,24 @@ dependencies = [ "regex", "serde", "serde_json", - "serde_yaml 0.9.28", + "serde_yaml 0.9.30", "shlex 1.2.0", "tempdir", "thiserror", - "ya-client-model", + "ya-client-model 0.6.0", ] [[package]] name = "ya-client" -version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=afdc1daf1f17a5128d911f8fff9d55c4f213e9af#afdc1daf1f17a5128d911f8fff9d55c4f213e9af" +version = "0.8.0" +source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" dependencies = [ "actix-codec", "awc", "bytes 1.5.0", "chrono", "envy", - "futures 0.3.29", + "futures 0.3.30", "heck 0.4.1", "hex", "log", @@ -8193,13 +8211,30 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.6.0", ] [[package]] name = "ya-client-model" version = "0.5.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=afdc1daf1f17a5128d911f8fff9d55c4f213e9af#afdc1daf1f17a5128d911f8fff9d55c4f213e9af" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a4a2f46bc12b1c747495c7911a844e7dc2c879826993426d568b9b594c26a1" +dependencies = [ + "bigdecimal 0.2.2", + "chrono", + "derive_more", + "rand 0.8.5", + "serde", + "serde_json", + "strum 0.19.5", + "strum_macros 0.19.4", + "thiserror", +] + +[[package]] +name = "ya-client-model" +version = "0.6.0" +source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -8242,7 +8277,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.6.0", "ya-service-bus", ] @@ -8264,13 +8299,13 @@ dependencies = [ "anyhow", "bigdecimal 0.2.2", "chrono", - "futures 0.3.29", + "futures 0.3.30", "log", "maplit", "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.5.0", "ya-core-model", "ya-payment-driver", "ya-persistence", @@ -8295,7 +8330,7 @@ dependencies = [ "ethabi", "ethereum-tx-sign", "ethereum-types 0.14.1", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "log", @@ -8316,7 +8351,7 @@ dependencies = [ "url", "uuid 0.8.2", "web3", - "ya-client-model", + "ya-client-model 0.6.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -8341,7 +8376,7 @@ dependencies = [ "ethereum-tx-sign", "ethereum-types 0.14.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "log", @@ -8360,7 +8395,7 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model", + "ya-client-model 0.6.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -8384,7 +8419,7 @@ dependencies = [ "dotenv", "env_logger 0.7.1", "flexi_logger 0.22.6", - "futures 0.3.29", + "futures 0.3.30", "graphene-sgx", "hex", "ipnet", @@ -8414,7 +8449,7 @@ dependencies = [ "url", "winapi 0.3.9", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.6.0", "ya-compile-time-utils", "ya-core-model", "ya-manifest-utils", @@ -8465,7 +8500,7 @@ dependencies = [ "bytes 1.5.0", "crossterm 0.26.1", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "gftp", "globset", "h2", @@ -8487,7 +8522,7 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-service-bus", "ya-utils-futures", @@ -8520,7 +8555,7 @@ dependencies = [ "ctor", "env_logger 0.10.1", "flexbuffers", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "serde", @@ -8530,7 +8565,7 @@ dependencies = [ "thiserror", "tokio", "uuid 1.6.1", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-persistence", "ya-service-api", @@ -8557,7 +8592,7 @@ dependencies = [ "dotenv", "env_logger 0.7.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "log", "promptly", "r2d2", @@ -8570,7 +8605,7 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-persistence", "ya-sb-router", @@ -8608,10 +8643,10 @@ dependencies = [ "pretty_assertions", "regex", "schemars", - "semver 1.0.20", + "semver 1.0.21", "serde", "serde_json", - "serde_yaml 0.9.28", + "serde_yaml 0.9.30", "serial_test 2.0.0", "shlex 1.2.0", "snailquote", @@ -8623,7 +8658,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.6.0", "ya-manifest-test-utils", "ya-utils-path", ] @@ -8647,7 +8682,7 @@ dependencies = [ "diesel_migrations", "digest 0.8.1", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "humantime 2.1.0", "lazy_static", "libsqlite3-sys", @@ -8709,7 +8744,7 @@ dependencies = [ "anyhow", "awc", "bigdecimal 0.2.2", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "metrics 0.16.0", @@ -8736,7 +8771,7 @@ dependencies = [ "chrono", "env_logger 0.7.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "humantime 2.1.0", "lazy_static", "log", @@ -8753,7 +8788,7 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", @@ -8803,7 +8838,7 @@ dependencies = [ "env_logger 0.7.1", "erc20_payment_lib", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "humantime 2.1.0", "lazy_static", @@ -8823,7 +8858,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", @@ -8851,7 +8886,7 @@ dependencies = [ "diesel_migrations", "ethereum-types 0.14.1", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "log", "num-bigint 0.3.3", @@ -8861,7 +8896,7 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-persistence", "ya-service-bus", @@ -8885,7 +8920,7 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-service-api", "ya-service-api-interfaces", @@ -8910,7 +8945,7 @@ dependencies = [ "dialoguer", "directories", "dotenv", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "golem-certificate", "hex", @@ -8950,7 +8985,7 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-compile-time-utils", "ya-core-model", "ya-file-logging", @@ -8975,7 +9010,7 @@ dependencies = [ "chrono", "derive_more", "educe", - "futures 0.3.29", + "futures 0.3.30", "hex", "humantime 2.1.0", "log", @@ -9004,7 +9039,7 @@ dependencies = [ "derive_more", "digest 0.9.0", "ethsign", - "futures 0.3.29", + "futures 0.3.30", "hex", "lazy_static", "log", @@ -9019,7 +9054,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.5.0", "ya-relay-proto", "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", ] @@ -9032,7 +9067,7 @@ dependencies = [ "anyhow", "bytes 1.5.0", "derive_more", - "futures 0.3.29", + "futures 0.3.30", "hex", "prost 0.12.3", "prost-build 0.12.3", @@ -9050,7 +9085,7 @@ version = "0.5.0" source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" dependencies = [ "derive_more", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "managed", @@ -9070,7 +9105,7 @@ version = "0.5.0" source = "git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5#c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" dependencies = [ "derive_more", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "managed", @@ -9110,7 +9145,7 @@ dependencies = [ "anyhow", "bytes 1.5.0", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "log", "prost 0.10.4", "prost-build 0.10.4", @@ -9149,7 +9184,7 @@ dependencies = [ "anyhow", "bitflags 1.3.2", "chrono", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "parking_lot 0.11.2", @@ -9172,7 +9207,7 @@ source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=44c942c72f6 dependencies = [ "actix", "bitflags 1.3.2", - "futures 0.3.29", + "futures 0.3.30", "pin-project 0.4.30", ] @@ -9213,7 +9248,7 @@ version = "0.2.0" dependencies = [ "actix-web", "anyhow", - "futures 0.3.29", + "futures 0.3.30", ] [[package]] @@ -9228,7 +9263,7 @@ dependencies = [ "anyhow", "awc", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "log", "serde", "structopt", @@ -9251,7 +9286,7 @@ source = "git+https://github.com/golemfactory/ya-service-bus.git?rev=44c942c72f6 dependencies = [ "actix", "flexbuffers", - "futures 0.3.29", + "futures 0.3.30", "lazy_static", "log", "miniz_oxide 0.5.4", @@ -9274,7 +9309,7 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-service-bus", ] @@ -9335,7 +9370,7 @@ dependencies = [ "bytes 1.5.0", "crossterm 0.26.1", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "gftp", "globset", "h2", @@ -9360,7 +9395,7 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-exe-unit", "ya-framework-basic", @@ -9379,7 +9414,7 @@ dependencies = [ "actix-rt", "anyhow", "chrono", - "futures 0.3.29", + "futures 0.3.30", "log", "tokio", ] @@ -9393,14 +9428,14 @@ dependencies = [ "prettytable-rs", "serde", "serde_json", - "serde_yaml 0.9.28", + "serde_yaml 0.9.30", ] [[package]] name = "ya-utils-futures" version = "0.3.0" dependencies = [ - "futures 0.3.29", + "futures 0.3.30", "tokio", ] @@ -9409,7 +9444,7 @@ name = "ya-utils-networking" version = "0.2.0" dependencies = [ "anyhow", - "futures 0.3.29", + "futures 0.3.30", "ipnet", "lazy_static", "log", @@ -9437,7 +9472,7 @@ dependencies = [ "anyhow", "derive_more", "fs2", - "futures 0.3.29", + "futures 0.3.30", "futures-util", "libc", "nix 0.22.3", @@ -9482,7 +9517,7 @@ dependencies = [ "anyhow", "bytes 1.5.0", "env_logger 0.7.1", - "futures 0.3.29", + "futures 0.3.30", "hex", "ipnet", "lazy_static", @@ -9499,7 +9534,7 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-core-model", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", @@ -9523,7 +9558,7 @@ dependencies = [ "chrono", "directories", "dotenv", - "futures 0.3.29", + "futures 0.3.30", "gftp", "lazy_static", "libsqlite3-sys", @@ -9541,7 +9576,7 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model", + "ya-client-model 0.6.0", "ya-compile-time-utils", "ya-core-model", "ya-dummy-driver", @@ -9592,22 +9627,22 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -9627,7 +9662,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.42", + "syn 2.0.46", ] [[package]] @@ -9654,26 +9689,25 @@ checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ "byteorder", "crc32fast", - "crossbeam-utils 0.8.17", + "crossbeam-utils 0.8.18", "flate2", ] [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ - "libc", "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 9f7b923261..fa48057add 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,8 +60,8 @@ ya-utils-networking = "0.2" ya-fd-metrics = { path = "utils/fd-metrics" } ya-version = "0.2" ya-vpn = "0.2" -ya-client = "0.7" -ya-client-model = "0.5" +ya-client = "0.8" +ya-client-model = "0.6" gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "afdc1daf1f17a5128d911f8fff9d55c4f213e9af" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "afdc1daf1f17a5128d911f8fff9d55c4f213e9af" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } ## RELAY and networking stack diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index df7415bac8..78c15149a2 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -15,8 +15,8 @@ path = "src/main.rs" [dependencies] ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } -ya-client = { version = "0.7", features = ['cli'] } -ya-client-model = "0.5" +ya-client = { version = "0.8", features = ['cli'] } +ya-client-model = "0.6" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ['activity', 'payment'] } ya-file-logging = "0.1" diff --git a/core/activity/Cargo.toml b/core/activity/Cargo.toml index 9f1633af05..c64fc428ea 100644 --- a/core/activity/Cargo.toml +++ b/core/activity/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] ya-core-model = { version = "0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.5", features = ["sgx"] } +ya-client-model = { version = "0.6", features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/core/gsb-api/Cargo.toml b/core/gsb-api/Cargo.toml index 8d1db889b9..df32d0c860 100644 --- a/core/gsb-api/Cargo.toml +++ b/core/gsb-api/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index c78db1c600..afd523c56b 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = { version = "0.5", features = ["with-diesel"] } +ya-client-model = { version = "0.6", features = ["with-diesel"] } ya-core-model = { version = "^0.9", features = ["identity", "appkey"] } ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index 5eb13843d6..9b7cd586dc 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -11,7 +11,7 @@ bcast-singleton = [] [dependencies] ya-agreement-utils = { workspace = true } -ya-client = "0.7" +ya-client = "0.8" ya-core-model = { version = "^0.9", features = ["market", "net"] } ya-diesel-utils = { version = "0.1" } ya-market-resolver = "0.2" diff --git a/core/model/Cargo.toml b/core/model/Cargo.toml index 8d6f99ad92..0b026e02d2 100644 --- a/core/model/Cargo.toml +++ b/core/model/Cargo.toml @@ -35,7 +35,7 @@ sgx = ['graphene-sgx'] version = [] [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-service-bus = "0.6.1" bigdecimal = { version = "0.2", features = ["serde"], optional = true } diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index 0d51a93bab..e4f3824923 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -15,7 +15,7 @@ packet-trace-enable = [ ] [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = ["net", "identity"] } ya-relay-client = { workspace = true } diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index 25fcf53c2f..ed80dd6da0 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -29,7 +29,7 @@ thiserror = "1.0" tokio = { version = "1", features = ["macros"] } ## yagna dependencies -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = [ "driver", "identity", diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index 55a54b5749..089e1de6a8 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -43,7 +43,7 @@ url = { workspace = true } trust-dns-resolver = { workspace = true } ## yagna dependencies ya-payment-driver = "0.3" -ya-client-model = "0.5" +ya-client-model = "0.6" ya-service-api-interfaces = "0.2" ya-utils-futures = "0.3" ya-utils-networking = "0.2" diff --git a/core/payment-driver/erc20next/Cargo.toml b/core/payment-driver/erc20next/Cargo.toml index 0a7f7da89b..b4aa656d15 100644 --- a/core/payment-driver/erc20next/Cargo.toml +++ b/core/payment-driver/erc20next/Cargo.toml @@ -43,7 +43,7 @@ rust_decimal = "1" ## yagna dependencies ya-payment-driver = "0.3" -ya-client-model = "0.5" +ya-client-model = "0.6" ya-service-api-interfaces = "0.2" ya-utils-futures = "0.3" ya-utils-networking = "0.2" diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index b021dbbd61..766ade21ef 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -9,7 +9,7 @@ default = [] [dependencies] ya-agreement-utils = { workspace = true } -ya-client-model = { version = "0.5", features = ["with-diesel"] } +ya-client-model = { version = "0.6", features = ["with-diesel"] } ya-core-model = { version = "^0.9", features = [ "activity", "driver", @@ -59,7 +59,7 @@ humantime = "2.0.1" erc20_payment_lib = { workspace = true } [dev-dependencies] -ya-client = "0.7" +ya-client = "0.8" ya-dummy-driver = "0.3" ya-erc20-driver = "0.4" ya-erc20next-driver = "0.4" diff --git a/core/persistence/Cargo.toml b/core/persistence/Cargo.toml index 21cf823a7e..6013ec618e 100644 --- a/core/persistence/Cargo.toml +++ b/core/persistence/Cargo.toml @@ -9,7 +9,7 @@ default = [] service = ["ya-service-api", "ya-service-api-interfaces", "ya-utils-process", "structopt"] [dependencies] -ya-client-model = { version = "0.5", features = [ "with-diesel" ] } +ya-client-model = { version = "0.6", features = [ "with-diesel" ] } ya-core-model = { version = "0.9" } ya-service-api = { version = "0.1", optional = true } ya-service-api-interfaces = { version = "0.2", optional = true } diff --git a/core/serv-api/web/Cargo.toml b/core/serv-api/web/Cargo.toml index 1148c348e2..2a1424e2ca 100644 --- a/core/serv-api/web/Cargo.toml +++ b/core/serv-api/web/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.7" +ya-client = "0.8" ya-core-model = { version = "^0.9", features = ["appkey"] } ya-service-api = "0.1" ya-service-bus = "0.6.1" diff --git a/core/sgx/Cargo.toml b/core/sgx/Cargo.toml index 79f64cf9cd..4fc30157ea 100644 --- a/core/sgx/Cargo.toml +++ b/core/sgx/Cargo.toml @@ -7,6 +7,6 @@ edition = "2018" [dependencies] anyhow = "1.0" graphene-sgx = { version = "0.3.3", features = [ "ias" ] } -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = ["sgx"] } ya-service-bus = "0.6.1" diff --git a/core/version/Cargo.toml b/core/version/Cargo.toml index 7e4653fc4f..b177d4cc31 100644 --- a/core/version/Cargo.toml +++ b/core/version/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.7" +ya-client = "0.8" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ["version"] } ya-persistence = "0.3" diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index e53f46653e..5cf5676f9f 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] ya-core-model = { version = "^0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.5", features = ["sgx"] } +ya-client-model = { version = "0.6", features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" @@ -42,7 +42,7 @@ packet-trace-enable = ["ya-packet-trace/enable"] default = [] [dev-dependencies] -ya-client = "0.7" +ya-client = "0.8" actix-rt = "2.7" sha3 = "0.8.2" structopt = "0.3" diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index e5ff92faf1..af1f4357ae 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -39,7 +39,7 @@ winapi = { version = "0.3.8", features = ["jobapi2", "processthreadsapi"] } [dependencies] ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } -ya-client-model = "0.5" +ya-client-model = "0.6" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index d735528f07..e980e0ccc4 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9" } ya-service-bus = "0.6.1" ya-utils-path = { version ="0.1", path = "../../../utils/path" } diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index b30befac1c..1ab485cc81 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -26,22 +26,20 @@ use futures::prelude::*; use futures::task::{Context, Poll}; use url::Url; -use crate::error::Error; - pub use crate::archive::{archive, extract, ArchiveFormat}; pub use crate::container::ContainerTransferProvider; +use crate::error::Error; pub use crate::file::{DirTransferProvider, FileTransferProvider}; pub use crate::gftp::GftpTransferProvider; +use crate::hash::with_hash_stream; pub use crate::http::HttpTransferProvider; pub use crate::location::{TransferUrl, UrlExt}; +use crate::progress::progress_report_channel; pub use crate::progress::{wrap_sink_with_progress_reporting, wrap_stream_with_progress_reporting}; pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; -use crate::hash::with_hash_stream; -use crate::progress::progress_report_channel; -use crate::transfer::Progress; -use ya_client_model::activity::TransferArgs; +use ya_client_model::activity::{CommandProgress, TransferArgs}; /// Transfers data from `stream` to a `TransferSink` pub async fn transfer(stream: S, mut sink: TransferSink) -> Result<(), Error> @@ -299,7 +297,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, - pub report: Arc>>>, + pub report: Arc>>>, } impl TransferContext { @@ -315,11 +313,11 @@ impl TransferContext { } } - pub fn register_reporter(&self, report: Option>) { + pub fn register_reporter(&self, report: Option>) { *self.report.lock().unwrap() = report; } - pub fn take_reporter(&self) -> Option> { + pub fn take_reporter(&self) -> Option> { self.report.lock().unwrap().take() } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 5c0ba866f5..2ba4c9bd9c 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -1,10 +1,12 @@ use crate::error::Error; -use crate::transfer::Progress; use crate::{abortable_sink, abortable_stream, TransferSink, TransferStream}; use crate::{TransferContext, TransferData}; + use futures::{SinkExt, StreamExt, TryFutureExt}; use tokio::task::spawn_local; +use ya_client_model::activity::CommandProgress; + type Stream = TransferStream; /// Wraps a stream to report progress. @@ -53,7 +55,7 @@ pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { let report = ctx.take_reporter(); wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { if let Some(report) = &report { - report.send(Progress { progress, size }).ok(); + report.send(CommandProgress { progress, size }).ok(); } }) } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 615dd17bf1..7629125a15 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -5,7 +5,6 @@ use std::rc::Rc; use actix::prelude::*; use futures::future::Abortable; -use serde::{Deserialize, Serialize}; use url::Url; use crate::cache::{Cache, CachePath}; @@ -16,6 +15,7 @@ use crate::{ HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; +pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; use ya_runtime_api::deploy::ContainerVolume; use ya_utils_futures::abort::Abort; @@ -36,12 +36,6 @@ macro_rules! actor_try { }; } -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct Progress { - pub progress: u64, - pub size: Option, -} - #[derive(Debug, Message)] #[rtype(result = "Result<()>")] pub struct TransferResource { @@ -51,7 +45,7 @@ pub struct TransferResource { /// Channel for watching for transfer progress. `None` means that there /// will be no progress updates. - pub progress: Option>, + pub progress: Option>, } #[derive(Message)] @@ -70,7 +64,7 @@ pub struct DeployImage { pub task_package: Option, /// Channel for watching for deploy progress. `None` means that there /// will be no progress updates. - pub progress: Option>, + pub progress: Option>, } #[derive(Clone, Debug, Message)] @@ -196,7 +190,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, - progress: Option>, + progress: Option>, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 323bb318a6..ab7c0e8bcd 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -10,7 +10,7 @@ use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; use ya_transfer::transfer::{ - AbortTransfers, DeployImage, Progress, TransferService, TransferServiceContext, + AbortTransfers, CommandProgress, DeployImage, TransferService, TransferServiceContext, }; /// When re-deploying image, `TransferService` should uses partially downloaded image. @@ -126,7 +126,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let addr = TransferService::new(exe_ctx).start(); log::info!("[>>] Deployment with hash verification"); - let (tx, mut rx) = tokio::sync::watch::channel::(Progress::default()); + let (tx, mut rx) = tokio::sync::watch::channel(CommandProgress::default()); tokio::task::spawn_local(async move { let _result = addr diff --git a/exe-unit/src/state.rs b/exe-unit/src/state.rs index b4e957fc66..7b98b8b72b 100644 --- a/exe-unit/src/state.rs +++ b/exe-unit/src/state.rs @@ -164,6 +164,7 @@ impl Batch { ..event }) } + RuntimeEventKind::Progress(_) => Some(event), }; if let Some(evt) = stream_event { diff --git a/golem_cli/Cargo.toml b/golem_cli/Cargo.toml index 1fde23a1b2..63c9525804 100644 --- a/golem_cli/Cargo.toml +++ b/golem_cli/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = { version = "0.7", features = ['cli'] } +ya-client = { version = "0.8", features = ['cli'] } ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features=["payment", "version"] } ya-provider = "0.3" diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index e3039d162b..09eceeca36 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" ya-core-model = { version = "^0.9" } ya-service-bus = "0.6.1" ya-utils-path = { version ="0.1", path = "../../../utils/path" } diff --git a/utils/agreement-utils/Cargo.toml b/utils/agreement-utils/Cargo.toml index 43e0aadabc..bf677d70c9 100644 --- a/utils/agreement-utils/Cargo.toml +++ b/utils/agreement-utils/Cargo.toml @@ -13,7 +13,7 @@ keywords=["golem", "yagna"] default = [] [dependencies] -ya-client-model = "0.5" +ya-client-model = "0.6" chrono = "0.4" regex = "1.5.4" diff --git a/utils/manifest-utils/Cargo.toml b/utils/manifest-utils/Cargo.toml index c5d584cd5e..4b08b3c87e 100644 --- a/utils/manifest-utils/Cargo.toml +++ b/utils/manifest-utils/Cargo.toml @@ -22,7 +22,7 @@ schema = ["schemars"] [dependencies] ya-agreement-utils = { workspace = true } ya-utils-path = "0.1" -ya-client-model = "0.5" +ya-client-model = "0.6" golem-certificate = "0.1.1" regex = "1.5" From 7008049a9933a58b2a78e9943693ce0f435bb25b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 4 Jan 2024 13:17:12 +0100 Subject: [PATCH 007/125] Disable spamming ExeUnit logs during transfer --- exe-unit/src/logger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exe-unit/src/logger.rs b/exe-unit/src/logger.rs index d9cd42565d..9d36cd3473 100644 --- a/exe-unit/src/logger.rs +++ b/exe-unit/src/logger.rs @@ -6,7 +6,7 @@ const ENV_VAR_LOG_DIR: &str = "EXE_UNIT_LOG_DIR"; const ENV_VAR_FILE_LOG_LEVEL: &str = "EXE_UNIT_FILE_LOG_LEVEL"; const DEFAULT_LOG_LEVEL: &str = "info"; -const DEFAULT_FILE_LOG_LEVEL: &str = "debug"; +const DEFAULT_FILE_LOG_LEVEL: &str = "debug,tokio_util=off,h2=info"; const DEFAULT_LOG_DIR: &str = "logs"; const DEFAULT_LOG_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.3f%z"; From 1ffaf5a7e078a261f47981dbe6b47cc3f7704d5c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 5 Jan 2024 18:52:57 +0100 Subject: [PATCH 008/125] Use new progress model from ya-client --- Cargo.lock | 97 +++++++++++--------- Cargo.toml | 4 +- exe-unit/components/transfer/Cargo.toml | 1 + exe-unit/components/transfer/src/lib.rs | 34 ++++++- exe-unit/components/transfer/src/progress.rs | 6 +- exe-unit/components/transfer/src/transfer.rs | 53 +++++++++-- exe-unit/src/lib.rs | 18 +++- exe-unit/src/message.rs | 15 ++- 8 files changed, 163 insertions(+), 65 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd96f033fa..91f0543b9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -154,7 +154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -336,7 +336,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -603,7 +603,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -941,7 +941,7 @@ dependencies = [ "proc-macro-crate 2.0.1", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "syn_derive", ] @@ -1270,9 +1270,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -1612,7 +1612,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1816,6 +1816,15 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "duration-string" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcc1d9ae294a15ed05aeae8e11ee5f2b3fe971c077d45a42fb20825fba6ee13" +dependencies = [ + "serde", +] + [[package]] name = "dyn-clone" version = "1.0.16" @@ -1914,7 +1923,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -1927,7 +1936,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -2573,7 +2582,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -2711,7 +2720,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -3735,7 +3744,7 @@ checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4286,7 +4295,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4505,9 +4514,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" dependencies = [ "memchr", "thiserror", @@ -4571,7 +4580,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4684,7 +4693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -4779,9 +4788,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.74" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de98502f212cfcea8d0bb305bd0f49d7ebdd75b64ba0a68f937d888f4e0d6db" +checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" dependencies = [ "unicode-ident", ] @@ -4882,7 +4891,7 @@ dependencies = [ "prost 0.12.3", "prost-types 0.12.3", "regex", - "syn 2.0.46", + "syn 2.0.48", "tempfile", "which", ] @@ -4923,7 +4932,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -5886,7 +5895,7 @@ checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -5902,9 +5911,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.110" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fbd975230bada99c8bb618e0c365c2eefa219158d5c6c29610fd09ff1833257" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -5941,7 +5950,7 @@ checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6109,7 +6118,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6748,7 +6757,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6770,9 +6779,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.46" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89456b690ff72fddcecf231caedbe615c59480c93358a93dfae7fc29e3ebbf0e" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -6788,7 +6797,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6946,7 +6955,7 @@ dependencies = [ "cfg-if 1.0.0", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -6970,7 +6979,7 @@ checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "test-case-core", ] @@ -7021,7 +7030,7 @@ checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -7121,7 +7130,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -7283,7 +7292,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -7670,7 +7679,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -7704,7 +7713,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -8073,9 +8082,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.31" +version = "0.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a4882e6b134d6c28953a387571f1acdd3496830d5e36c5e3a1075580ea641c" +checksum = "8434aeec7b290e8da5c3f0d628cb0eac6cabcb31d14bb74f779a08109a5914d6" dependencies = [ "memchr", ] @@ -8193,7 +8202,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" +source = "git+https://github.com/golemfactory/ya-client.git?rev=b313c3f79a5160c22bab3deea903f1e8ae009164#b313c3f79a5160c22bab3deea903f1e8ae009164" dependencies = [ "actix-codec", "awc", @@ -8234,12 +8243,13 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=1a923517af842ccf16809f79484a0ad4ecd86880#1a923517af842ccf16809f79484a0ad4ecd86880" +source = "git+https://github.com/golemfactory/ya-client.git?rev=b313c3f79a5160c22bab3deea903f1e8ae009164#b313c3f79a5160c22bab3deea903f1e8ae009164" dependencies = [ "bigdecimal 0.2.2", "chrono", "derive_more", "diesel", + "duration-string", "hex", "openssl", "rand 0.8.5", @@ -9391,6 +9401,7 @@ dependencies = [ "test-context", "thiserror", "tokio", + "tokio-stream", "tokio-tar", "tokio-util", "url", @@ -9642,7 +9653,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] @@ -9662,7 +9673,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.46", + "syn 2.0.48", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fa48057add..c4fd1b3ec7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "1a923517af842ccf16809f79484a0ad4ecd86880" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "b313c3f79a5160c22bab3deea903f1e8ae009164" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "b313c3f79a5160c22bab3deea903f1e8ae009164" } ## RELAY and networking stack diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index e980e0ccc4..9e9de1e517 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -42,6 +42,7 @@ tokio-util = { version = "0.7", features = ["io"] } url = "2.1.1" walkdir = "2.3.1" async-trait = "0.1.74" +tokio-stream = "0.1.14" [target.'cfg(target_family = "unix")'.dependencies] awc = { version = "3", features = ["openssl"] } diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index 1ab485cc81..ec227bff97 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -66,6 +66,8 @@ where let src = src.as_ref(); let dst = dst.as_ref(); + ctx.report(CommandProgress::TransferStarted); + loop { let fut = async { dst.prepare_destination(&dst_url.url, ctx).await?; @@ -81,10 +83,15 @@ where }; match fut.await { - Ok(val) => return Ok(val), + Ok(val) => { + ctx.report(CommandProgress::TransferFinished); + return Ok(val); + } Err(err) => match ctx.state.delay(&err) { Some(delay) => { log::warn!("Retrying in {}s because: {}", delay.as_secs_f32(), err); + + ctx.report_retry(err.to_string(), delay); tokio::time::sleep(delay).await; } None => return Err(err), @@ -297,7 +304,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, - pub report: Arc>>>, + pub report: Arc>>>, } impl TransferContext { @@ -313,12 +320,29 @@ impl TransferContext { } } - pub fn register_reporter(&self, report: Option>) { + pub fn register_reporter( + &self, + report: Option>, + ) { *self.report.lock().unwrap() = report; } - pub fn take_reporter(&self) -> Option> { - self.report.lock().unwrap().take() + pub fn reporter(&self) -> Option> { + self.report.lock().unwrap().clone() + } + + pub fn report_retry(&self, message: String, delay: Duration) { + self.report(CommandProgress::Retry(message, delay)) + } + + pub fn report_fetching_cached(&self) { + self.report(CommandProgress::FetchingFromCache) + } + + pub fn report(&self, status: CommandProgress) { + if let Some(reporter) = self.reporter() { + reporter.send(status).ok(); + } } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 2ba4c9bd9c..e4133a3087 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -52,10 +52,12 @@ where type Sink = TransferSink; pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { - let report = ctx.take_reporter(); + let report = ctx.reporter(); wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { if let Some(report) = &report { - report.send(CommandProgress { progress, size }).ok(); + report + .send(CommandProgress::TransferProgress(progress, size)) + .ok(); } }) } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 7629125a15..8b5ccb718b 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -5,6 +5,7 @@ use std::rc::Rc; use actix::prelude::*; use futures::future::Abortable; +use futures::{Sink, StreamExt, TryStreamExt}; use url::Url; use crate::cache::{Cache, CachePath}; @@ -15,6 +16,7 @@ use crate::{ HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; +use ya_client_model::activity::exe_script_command::ProgressArgs; pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; use ya_runtime_api::deploy::ContainerVolume; @@ -36,16 +38,49 @@ macro_rules! actor_try { }; } -#[derive(Debug, Message)] +#[derive(Debug, Clone)] +pub struct ProgressConfig { + /// Channel for watching for transfer progress. + pub progress: tokio::sync::broadcast::Sender, + pub progress_args: ProgressArgs, +} + +#[derive(Debug, Message, Default)] #[rtype(result = "Result<()>")] pub struct TransferResource { pub from: String, pub to: String, pub args: TransferArgs, + /// Progress reporting configuration. `None` means that there will be no progress updates. + pub progress_args: Option, +} - /// Channel for watching for transfer progress. `None` means that there - /// will be no progress updates. - pub progress: Option>, +impl TransferResource { + pub fn forward_progress( + &mut self, + args: &ProgressArgs, + sender: impl Sink + 'static, + ) { + let rx = match &self.progress_args { + None => { + let (tx, rx) = tokio::sync::broadcast::channel(50); + self.progress_args = Some(ProgressConfig { + progress: tx, + progress_args: args.clone(), + }); + rx + } + Some(args) => args.progress.subscribe(), + }; + + tokio::task::spawn_local(async move { + tokio_stream::wrappers::BroadcastStream::new(rx) + .map_err(|e| Error::Other(e.to_string())) + .forward(sender) + .await + .ok() + }); + } } #[derive(Message)] @@ -62,9 +97,10 @@ impl AddVolumes { #[rtype(result = "Result>")] pub struct DeployImage { pub task_package: Option, + pub progress_args: ProgressArgs, /// Channel for watching for deploy progress. `None` means that there /// will be no progress updates. - pub progress: Option>, + pub progress: Option>, } #[derive(Clone, Debug, Message)] @@ -168,7 +204,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, - progress: Option>, + progress: Option>, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) @@ -190,7 +226,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, - progress: Option>, + progress: Option>, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); @@ -222,6 +258,7 @@ impl TransferService { let fut = async move { if path.exists() { log::info!("Deploying cached image: {:?}", path); + ctx.report_fetching_cached(); return Ok(Some(path)); } @@ -299,7 +336,7 @@ impl Handler for TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); - ctx.register_reporter(msg.progress); + ctx.register_reporter(msg.progress_args.map(|args| args.progress)); let (abort, reg) = Abort::new_pair(); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 56a70f27ac..e3983044ca 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -303,20 +303,30 @@ impl RuntimeRef { transfer_service: &Addr, ) -> Result<()> { match &runtime_cmd.command { - ExeScriptCommand::Transfer { from, to, args } => { - let msg = TransferResource { + ExeScriptCommand::Transfer { + from, + to, + args, + progress, + } => { + let mut msg = TransferResource { from: from.clone(), to: to.clone(), args: args.clone(), - progress: None, + progress_args: None, }; + + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } transfer_service.send(msg).await??; } - ExeScriptCommand::Deploy { net, hosts } => { + ExeScriptCommand::Deploy { net, hosts, .. } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. let task_package = transfer_service .send(DeployImage { task_package: None, + progress_args: Default::default(), progress: None, }) .await??; diff --git a/exe-unit/src/message.rs b/exe-unit/src/message.rs index 27be13d80b..8c19e9c3ee 100644 --- a/exe-unit/src/message.rs +++ b/exe-unit/src/message.rs @@ -5,6 +5,7 @@ use crate::Result; use actix::prelude::*; use futures::channel::mpsc; +use futures::{Sink, SinkExt}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::PathBuf; @@ -12,7 +13,9 @@ use std::path::PathBuf; use ya_client_model::activity; use ya_client_model::activity::activity_state::{State, StatePair}; use ya_client_model::activity::exe_script_command::Network; -use ya_client_model::activity::{CommandOutput, ExeScriptCommand, ExeScriptCommandResult}; +use ya_client_model::activity::{ + CommandOutput, CommandProgress, ExeScriptCommand, ExeScriptCommandResult, +}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Message)] #[rtype(result = "Result>")] @@ -110,6 +113,16 @@ impl ExecuteCommand { }, ) } + + pub fn progress_sink(&self) -> impl Sink { + let CommandContext { batch_id, idx, .. } = self.clone().split().1; + self.tx.clone().with(move |item| { + let batch_id = batch_id.clone(); + futures::future::ok(RuntimeEvent::Process(activity::RuntimeEvent::progress( + batch_id, idx, item, + ))) + }) + } } #[derive(Clone, Debug)] From 1675c21565084bd9fc88a70aa5aeb5d48327bcc6 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 5 Jan 2024 19:01:13 +0100 Subject: [PATCH 009/125] Fix ya-client dependencies versions problems (still not compiling) --- Cargo.lock | 38 +++++++++++++++++++------------------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91f0543b9e..09af9f2c59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9011,8 +9011,8 @@ dependencies = [ [[package]] name = "ya-relay-client" -version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +version = "0.6.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "anyhow", "async-trait", @@ -9036,13 +9036,13 @@ dependencies = [ "ya-packet-trace 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ya-relay-core", "ya-relay-proto", - "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-stack 0.5.1", ] [[package]] name = "ya-relay-core" -version = "0.4.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +version = "0.4.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "anyhow", "chrono", @@ -9064,15 +9064,15 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.5.0", + "ya-client-model 0.6.0", "ya-relay-proto", - "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-stack 0.5.1", ] [[package]] name = "ya-relay-proto" -version = "0.4.2" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +version = "0.4.3" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "anyhow", "bytes 1.5.0", @@ -9086,13 +9086,13 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a)", ] [[package]] name = "ya-relay-stack" version = "0.5.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5#c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" dependencies = [ "derive_more", "futures 0.3.30", @@ -9102,17 +9102,17 @@ dependencies = [ "num-derive", "num-traits", "rand 0.8.5", - "smoltcp", "thiserror", "tokio", "tokio-stream", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a)", + "ya-relay-util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ya-smoltcp", ] [[package]] name = "ya-relay-stack" -version = "0.5.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5#c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" +version = "0.5.1" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "derive_more", "futures 0.3.30", @@ -9122,11 +9122,11 @@ dependencies = [ "num-derive", "num-traits", "rand 0.8.5", + "smoltcp", "thiserror", "tokio", "tokio-stream", - "ya-relay-util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ya-smoltcp", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a)", ] [[package]] @@ -9142,7 +9142,7 @@ dependencies = [ [[package]] name = "ya-relay-util" version = "0.1.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=785c9c4271f514d47fe18018efb9438007ae611a#785c9c4271f514d47fe18018efb9438007ae611a" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=8d544dd766e14479519437d0bf4f2879e6a6de5a#8d544dd766e14479519437d0bf4f2879e6a6de5a" dependencies = [ "bytes 1.5.0", "derive_more", @@ -9463,7 +9463,7 @@ dependencies = [ "thiserror", "trust-dns-resolver 0.22.0", "url", - "ya-relay-stack 0.5.0 (git+https://github.com/golemfactory/ya-relay.git?rev=c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5)", + "ya-relay-stack 0.5.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c4fd1b3ec7..b15202f204 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -241,7 +241,7 @@ trust-dns-resolver = "0.22" derive_more = "0.99.11" ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "785c9c4271f514d47fe18018efb9438007ae611a" } +ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "8d544dd766e14479519437d0bf4f2879e6a6de5a" } gftp = { version = "0.4.0", path = "core/gftp" } ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } From 49f23ae0cc118a646f3f3454f35cad5ff1f91e47 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 8 Jan 2024 18:34:32 +0100 Subject: [PATCH 010/125] Use new ya-client Progress structs; Move towards streaming implementation --- Cargo.lock | 14 ++- Cargo.toml | 4 +- exe-unit/components/transfer/src/lib.rs | 35 ++----- exe-unit/components/transfer/src/progress.rs | 93 ++++++++++++++++++- exe-unit/components/transfer/src/transfer.rs | 13 +-- .../components/transfer/tests/test_deploy.rs | 26 +++--- .../transfer/tests/test_transfer_abort.rs | 2 +- .../transfer/tests/test_transfer_resume.rs | 2 +- .../transfer/tests/test_transfer_service.rs | 8 +- exe-unit/src/manifest.rs | 5 + 10 files changed, 138 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6377cc411..c1d6749a48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1805,6 +1805,15 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "duration-string" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcc1d9ae294a15ed05aeae8e11ee5f2b3fe971c077d45a42fb20825fba6ee13" +dependencies = [ + "serde", +] + [[package]] name = "dyn-clone" version = "1.0.16" @@ -8169,7 +8178,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db#a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" +source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" dependencies = [ "actix-codec", "awc", @@ -8193,12 +8202,13 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db#a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" +source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" dependencies = [ "bigdecimal 0.2.2", "chrono", "derive_more", "diesel", + "duration-string", "hex", "openssl", "rand 0.8.5", diff --git a/Cargo.toml b/Cargo.toml index 21e586e210..e9ffd15bd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "a3ee3cd0a5ac5c1a6022fa0f106a2d83e62844db" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } ## RELAY and networking stack diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index ec227bff97..4f05b41396 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -15,7 +15,6 @@ mod traverse; use std::cell::RefCell; use std::pin::Pin; use std::rc::Rc; -use std::sync::Arc; use std::time::Duration; use bytes::Bytes; @@ -34,7 +33,7 @@ pub use crate::gftp::GftpTransferProvider; use crate::hash::with_hash_stream; pub use crate::http::HttpTransferProvider; pub use crate::location::{TransferUrl, UrlExt}; -use crate::progress::progress_report_channel; +use crate::progress::{progress_report_channel, ProgressReporter}; pub use crate::progress::{wrap_sink_with_progress_reporting, wrap_stream_with_progress_reporting}; pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; @@ -66,8 +65,6 @@ where let src = src.as_ref(); let dst = dst.as_ref(); - ctx.report(CommandProgress::TransferStarted); - loop { let fut = async { dst.prepare_destination(&dst_url.url, ctx).await?; @@ -84,14 +81,14 @@ where match fut.await { Ok(val) => { - ctx.report(CommandProgress::TransferFinished); return Ok(val); } Err(err) => match ctx.state.delay(&err) { Some(delay) => { - log::warn!("Retrying in {}s because: {}", delay.as_secs_f32(), err); + let msg = format!("Retry in {}s because of error: {err}", delay.as_secs_f32()); + log::warn!("{}", msg); - ctx.report_retry(err.to_string(), delay); + ctx.progress.report_message(msg); tokio::time::sleep(delay).await; } None => return Err(err), @@ -304,7 +301,7 @@ impl From> for TransferData { pub struct TransferContext { pub state: TransferState, pub args: TransferArgs, - pub report: Arc>>>, + pub progress: ProgressReporter, } impl TransferContext { @@ -316,7 +313,7 @@ impl TransferContext { Self { args, state, - report: Arc::new(std::sync::Mutex::new(None)), + progress: ProgressReporter::default(), } } @@ -324,25 +321,11 @@ impl TransferContext { &self, report: Option>, ) { - *self.report.lock().unwrap() = report; - } - - pub fn reporter(&self) -> Option> { - self.report.lock().unwrap().clone() - } - - pub fn report_retry(&self, message: String, delay: Duration) { - self.report(CommandProgress::Retry(message, delay)) - } - - pub fn report_fetching_cached(&self) { - self.report(CommandProgress::FetchingFromCache) + self.progress.register_reporter(report); } - pub fn report(&self, status: CommandProgress) { - if let Some(reporter) = self.reporter() { - reporter.send(status).ok(); - } + pub fn reporter(&self) -> ProgressReporter { + self.progress.clone() } } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index e4133a3087..bed4fd7b47 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -3,12 +3,99 @@ use crate::{abortable_sink, abortable_stream, TransferSink, TransferStream}; use crate::{TransferContext, TransferData}; use futures::{SinkExt, StreamExt, TryFutureExt}; +use std::sync::Arc; use tokio::task::spawn_local; +use tokio::time::Instant; +use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::CommandProgress; type Stream = TransferStream; +#[derive(Debug, Clone)] +pub struct ProgressConfig { + /// Channel for watching for transfer progress. + pub progress: tokio::sync::broadcast::Sender, + pub progress_args: ProgressArgs, +} + +#[derive(Default, Clone)] +pub struct ProgressReporter { + config: ProgressArgs, + inner: Arc>>, +} + +struct ProgressImpl { + pub report: tokio::sync::broadcast::Sender, + pub last: CommandProgress, + pub last_update: Instant, +} + +impl ProgressReporter { + pub fn next_step(&self) { + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last.step.0 += 1; + inner.last.progress = (0, None); + inner.last_update = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok() + }); + } + + /// TODO: implement `update_interval` and `step` + pub fn report_progress(&self, progress: u64, size: Option) { + let _update_interval = self.config.update_interval; + let _step = self.config.step; + + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last.progress = (progress, size); + inner.last_update = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok() + }); + } + + pub fn report_message(&self, message: String) { + self.inner.lock().unwrap().as_mut().map(|inner| { + inner.last_update = Instant::now(); + inner + .report + .send(CommandProgress { + message: Some(message), + ..inner.last.clone() + }) + .ok() + }); + } + + pub fn report_fetching_cached(&self) { + self.report_message("Fetched file from cache".to_string()) + } + + pub fn register_reporter( + &self, + report: Option>, + ) { + if let Some(report) = report { + self.inner + .lock() + .unwrap() + .as_mut() + .map(|inner| inner.report = report); + } + } +} + /// Wraps a stream to report progress. /// The `report` function is called with the current offset and the total size. /// The total size is 0 if the size is unknown. (For example, when the source is a directory.) @@ -54,11 +141,7 @@ type Sink = TransferSink; pub fn progress_report_channel(dest: Sink, ctx: &TransferContext) -> Sink { let report = ctx.reporter(); wrap_sink_with_progress_reporting(dest, ctx, move |progress, size| { - if let Some(report) = &report { - report - .send(CommandProgress::TransferProgress(progress, size)) - .ok(); - } + report.report_progress(progress, size) }) } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 8b5ccb718b..335c82fd74 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -16,6 +16,7 @@ use crate::{ HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; +use crate::progress::ProgressConfig; use ya_client_model::activity::exe_script_command::ProgressArgs; pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; @@ -38,13 +39,6 @@ macro_rules! actor_try { }; } -#[derive(Debug, Clone)] -pub struct ProgressConfig { - /// Channel for watching for transfer progress. - pub progress: tokio::sync::broadcast::Sender, - pub progress_args: ProgressArgs, -} - #[derive(Debug, Message, Default)] #[rtype(result = "Result<()>")] pub struct TransferResource { @@ -93,7 +87,7 @@ impl AddVolumes { } } -#[derive(Debug, Message)] +#[derive(Debug, Message, Default)] #[rtype(result = "Result>")] pub struct DeployImage { pub task_package: Option, @@ -258,7 +252,8 @@ impl TransferService { let fut = async move { if path.exists() { log::info!("Deploying cached image: {:?}", path); - ctx.report_fetching_cached(); + ctx.reporter() + .report_message("Deployed image from cache".to_string()); return Ok(Some(path)); } diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index ab7c0e8bcd..57bf9656d4 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -3,15 +3,14 @@ use std::env; use std::time::Duration; use test_context::test_context; use tokio::time::sleep; +use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{ - AbortTransfers, CommandProgress, DeployImage, TransferService, TransferServiceContext, -}; +use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -66,7 +65,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re let result = addr .send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await?; log::info!("Deployment stopped"); @@ -76,7 +75,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re log::info!("Re-deploying the same image"); addr.send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await??; @@ -126,13 +125,14 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let addr = TransferService::new(exe_ctx).start(); log::info!("[>>] Deployment with hash verification"); - let (tx, mut rx) = tokio::sync::watch::channel(CommandProgress::default()); + let (tx, mut rx) = tokio::sync::broadcast::channel(15); tokio::task::spawn_local(async move { let _result = addr .send(DeployImage { task_package: task_package.clone(), progress: Some(tx), + progress_args: ProgressArgs::default(), }) .await??; log::info!("Deployment stopped"); @@ -140,18 +140,16 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< }); let mut last_progress = 0u64; - while let Ok(_) = rx.changed().await { - let progress = rx.borrow_and_update(); + while let Ok(progress) = rx.recv().await { + assert_eq!(progress.progress.1.unwrap(), file_size); + assert!(progress.progress.0 >= last_progress); - assert_eq!(progress.size.unwrap(), file_size); - assert!(progress.progress >= last_progress); - - last_progress = progress.progress; + last_progress = progress.progress.0; log::info!( "Progress: {}/{}", - progress.progress, - progress.size.unwrap_or(0) + progress.progress.0, + progress.progress.1.unwrap_or(0) ); } diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index 63f167d3a2..a41ad24269 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -38,7 +38,7 @@ async fn interrupted_transfer( from: src.to_owned(), to: dest.to_owned(), args: TransferArgs::default(), - progress: None, + progress_args: None, }) .await?; diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 9b561d2ef2..044e29384b 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -113,7 +113,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress: None, + progress_args: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index e92d5809b3..f339fecb4e 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -31,7 +31,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress: None, + progress_args: None, }) .await??; @@ -99,7 +99,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment with hash verification"); addr.send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await??; log::warn!("Deployment complete"); @@ -108,7 +108,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu log::warn!("[>>] Deployment from cache"); addr.send(DeployImage { task_package: task_package.clone(), - progress: None, + ..DeployImage::default() }) .await??; log::warn!("Deployment from cache complete"); @@ -157,7 +157,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu } #[ignore] -#[cfg_attr(not(feature = "framework-test"), ignore)] +//#[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/src/manifest.rs b/exe-unit/src/manifest.rs index 15aa919e49..21602bdc79 100644 --- a/exe-unit/src/manifest.rs +++ b/exe-unit/src/manifest.rs @@ -503,6 +503,10 @@ mod tests { ExeScriptCommand::Deploy { net: Default::default(), hosts: Default::default(), + hostname: None, + volumes: vec![], + env: Default::default(), + progress: None, }, ExeScriptCommand::Start { args: Default::default(), @@ -657,6 +661,7 @@ mod tests { from: "/src/0x0add".to_string(), to: "/dst/0x0add".to_string(), args: Default::default(), + progress: None, }]; let validator: ScriptValidator = r#"{ From b40c4e6ed9ea2b1b4d257707317cc78478701d20 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 8 Jan 2024 19:10:29 +0100 Subject: [PATCH 011/125] Use ProgressReporter in Deploy and Transfer flow --- exe-unit/components/transfer/src/lib.rs | 9 +- exe-unit/components/transfer/src/progress.rs | 25 +++-- exe-unit/components/transfer/src/transfer.rs | 101 +++++++++++++----- .../components/transfer/tests/test_deploy.rs | 10 +- .../transfer/tests/test_transfer_abort.rs | 2 +- .../transfer/tests/test_transfer_resume.rs | 2 +- .../transfer/tests/test_transfer_service.rs | 2 +- exe-unit/src/lib.rs | 28 +++-- 8 files changed, 115 insertions(+), 64 deletions(-) diff --git a/exe-unit/components/transfer/src/lib.rs b/exe-unit/components/transfer/src/lib.rs index 4f05b41396..4cca1ff186 100644 --- a/exe-unit/components/transfer/src/lib.rs +++ b/exe-unit/components/transfer/src/lib.rs @@ -38,7 +38,7 @@ pub use crate::progress::{wrap_sink_with_progress_reporting, wrap_stream_with_pr pub use crate::retry::Retry; pub use crate::traverse::PathTraverse; -use ya_client_model::activity::{CommandProgress, TransferArgs}; +use ya_client_model::activity::TransferArgs; /// Transfers data from `stream` to a `TransferSink` pub async fn transfer(stream: S, mut sink: TransferSink) -> Result<(), Error> @@ -317,13 +317,6 @@ impl TransferContext { } } - pub fn register_reporter( - &self, - report: Option>, - ) { - self.progress.register_reporter(report); - } - pub fn reporter(&self) -> ProgressReporter { self.progress.clone() } diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index bed4fd7b47..bbddbe9de8 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -78,20 +78,23 @@ impl ProgressReporter { }); } - pub fn report_fetching_cached(&self) { - self.report_message("Fetched file from cache".to_string()) - } - pub fn register_reporter( &self, - report: Option>, + args: Option, + steps: usize, + unit: Option, ) { - if let Some(report) = report { - self.inner - .lock() - .unwrap() - .as_mut() - .map(|inner| inner.report = report); + if let Some(args) = args { + *(self.inner.lock().unwrap()) = Some(ProgressImpl { + report: args.progress, + last: CommandProgress { + step: (0, steps), + message: None, + progress: (0, None), + unit, + }, + last_update: Instant::now(), + }); } } } diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 335c82fd74..55022a73d2 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -11,12 +11,12 @@ use url::Url; use crate::cache::{Cache, CachePath}; use crate::error::Error; use crate::error::Error as TransferError; +pub use crate::progress::ProgressConfig; use crate::{ transfer_with, ContainerTransferProvider, FileTransferProvider, GftpTransferProvider, HttpTransferProvider, Retry, TransferContext, TransferData, TransferProvider, TransferUrl, }; -use crate::progress::ProgressConfig; use ya_client_model::activity::exe_script_command::ProgressArgs; pub use ya_client_model::activity::CommandProgress; use ya_client_model::activity::TransferArgs; @@ -46,19 +46,40 @@ pub struct TransferResource { pub to: String, pub args: TransferArgs, /// Progress reporting configuration. `None` means that there will be no progress updates. - pub progress_args: Option, + pub progress_config: Option, } -impl TransferResource { - pub fn forward_progress( +#[derive(Message)] +#[rtype(result = "Result<()>")] +pub struct AddVolumes(Vec); + +impl AddVolumes { + pub fn new(vols: Vec) -> Self { + AddVolumes(vols) + } +} + +#[derive(Debug, Message, Default)] +#[rtype(result = "Result>")] +pub struct DeployImage { + pub task_package: Option, + /// Progress reporting configuration. `None` means that there will be no progress updates. + pub progress_config: Option, +} + +pub trait ForwardProgressToSink { + fn progress_config_mut(&mut self) -> &mut Option; + + fn forward_progress( &mut self, args: &ProgressArgs, sender: impl Sink + 'static, ) { - let rx = match &self.progress_args { + let progress_args = self.progress_config_mut(); + let rx = match progress_args { None => { let (tx, rx) = tokio::sync::broadcast::channel(50); - self.progress_args = Some(ProgressConfig { + *progress_args = Some(ProgressConfig { progress: tx, progress_args: args.clone(), }); @@ -77,24 +98,44 @@ impl TransferResource { } } -#[derive(Message)] -#[rtype(result = "Result<()>")] -pub struct AddVolumes(Vec); +impl ForwardProgressToSink for DeployImage { + fn progress_config_mut(&mut self) -> &mut Option { + &mut self.progress_config + } +} -impl AddVolumes { - pub fn new(vols: Vec) -> Self { - AddVolumes(vols) +impl ForwardProgressToSink for TransferResource { + fn progress_config_mut(&mut self) -> &mut Option { + &mut self.progress_config } } -#[derive(Debug, Message, Default)] -#[rtype(result = "Result>")] -pub struct DeployImage { - pub task_package: Option, - pub progress_args: ProgressArgs, - /// Channel for watching for deploy progress. `None` means that there - /// will be no progress updates. - pub progress: Option>, +impl DeployImage { + pub fn forward_progress( + &mut self, + args: &ProgressArgs, + sender: impl Sink + 'static, + ) { + let rx = match &self.progress_config { + None => { + let (tx, rx) = tokio::sync::broadcast::channel(50); + self.progress_config = Some(ProgressConfig { + progress: tx, + progress_args: args.clone(), + }); + rx + } + Some(args) => args.progress.subscribe(), + }; + + tokio::task::spawn_local(async move { + tokio_stream::wrappers::BroadcastStream::new(rx) + .map_err(|e| Error::Other(e.to_string())) + .forward(sender) + .await + .ok() + }); + } } #[derive(Clone, Debug, Message)] @@ -198,7 +239,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, - progress: Option>, + ctx: TransferContext, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) @@ -220,7 +261,7 @@ impl TransferService { src_url: TransferUrl, src_name: CachePath, path: PathBuf, - progress: Option>, + ctx: TransferContext, ) -> ActorResponse>> { let path_tmp = self.cache.to_temp_path(&src_name).to_path_buf(); @@ -231,10 +272,6 @@ impl TransferService { hash: None, }; - let ctx = TransferContext::default(); - ctx.state.retry_with(self.deploy_retry.clone()); - ctx.register_reporter(progress); - // Using partially downloaded image from previous executions could speed up deploy // process, but it comes with the cost: If image under URL changed, Requestor will get // error on the end. This can result with Provider being perceived as unreliable. @@ -312,11 +349,16 @@ impl Handler for TransferService { log::info!("Deploying from {:?} to {:?}", src_url.url, path); + let ctx = TransferContext::default(); + ctx.state.retry_with(self.deploy_retry.clone()); + ctx.progress + .register_reporter(deploy.progress_config, 1, Some("Bytes".to_string())); + #[cfg(not(feature = "sgx"))] - return self.deploy_no_sgx(src_url, src_name, path, deploy.progress); + return self.deploy_no_sgx(src_url, src_name, path, ctx); #[cfg(feature = "sgx")] - return self.deploy_sgx(src_url, src_name, path, deploy.progress); + return self.deploy_sgx(src_url, src_name, path, ctx); } } @@ -331,7 +373,8 @@ impl Handler for TransferService { let ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); - ctx.register_reporter(msg.progress_args.map(|args| args.progress)); + ctx.progress + .register_reporter(msg.progress_config, 1, Some("Bytes".to_string())); let (abort, reg) = Abort::new_pair(); diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 57bf9656d4..509c8fc0d3 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -10,7 +10,9 @@ use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; +use ya_transfer::transfer::{ + AbortTransfers, DeployImage, ProgressConfig, TransferService, TransferServiceContext, +}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -131,8 +133,10 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let _result = addr .send(DeployImage { task_package: task_package.clone(), - progress: Some(tx), - progress_args: ProgressArgs::default(), + progress_config: Some(ProgressConfig { + progress: tx, + progress_args: ProgressArgs::default(), + }), }) .await??; log::info!("Deployment stopped"); diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index a41ad24269..3b8e7e7ec6 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -38,7 +38,7 @@ async fn interrupted_transfer( from: src.to_owned(), to: dest.to_owned(), args: TransferArgs::default(), - progress_args: None, + progress_config: None, }) .await?; diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 044e29384b..7253453199 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -113,7 +113,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress_args: None, + progress_config: None, }) .await??; diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index f339fecb4e..fe2cf29d8e 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -31,7 +31,7 @@ async fn transfer_with_args( from: from.to_owned(), to: to.to_owned(), args, - progress_args: None, + progress_config: None, }) .await??; diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index e3983044ca..a2cfb911a6 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -18,7 +18,8 @@ use ya_core_model::activity::local::Credentials; use ya_runtime_api::deploy; use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; use ya_transfer::transfer::{ - AddVolumes, DeployImage, TransferResource, TransferService, TransferServiceContext, + AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, + TransferServiceContext, }; use crate::acl::Acl; @@ -313,7 +314,7 @@ impl RuntimeRef { from: from.clone(), to: to.clone(), args: args.clone(), - progress_args: None, + progress_config: None, }; if let Some(args) = progress { @@ -321,15 +322,22 @@ impl RuntimeRef { } transfer_service.send(msg).await??; } - ExeScriptCommand::Deploy { net, hosts, .. } => { + ExeScriptCommand::Deploy { + net, + hosts, + progress, + .. + } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. - let task_package = transfer_service - .send(DeployImage { - task_package: None, - progress_args: Default::default(), - progress: None, - }) - .await??; + let mut msg = DeployImage { + task_package: None, + progress_config: None, + }; + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + + let task_package = transfer_service.send(msg).await??; runtime .send(UpdateDeployment { task_package, From 1916012f6b239cc93587850e034c4aba07bd1adb Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 8 Jan 2024 19:34:08 +0100 Subject: [PATCH 012/125] More user friendly Deploy api in TransferService --- exe-unit/components/transfer/src/transfer.rs | 9 ++++ .../components/transfer/tests/test_deploy.rs | 49 ++++++++----------- .../transfer/tests/test_transfer_service.rs | 18 +++---- exe-unit/src/lib.rs | 5 +- 4 files changed, 36 insertions(+), 45 deletions(-) diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 55022a73d2..251f87f930 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -67,6 +67,15 @@ pub struct DeployImage { pub progress_config: Option, } +impl DeployImage { + pub fn with_package(task_package: &str) -> DeployImage { + DeployImage { + task_package: Some(task_package.to_string()), + progress_config: None, + } + } +} + pub trait ForwardProgressToSink { fn progress_config_mut(&mut self) -> &mut Option; diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 509c8fc0d3..ae32972974 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -1,18 +1,20 @@ use actix::Actor; +use futures::channel::mpsc; +use futures::SinkExt; use std::env; use std::time::Duration; use test_context::test_context; use tokio::time::sleep; +use tokio_stream::StreamExt; use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::CommandProgress; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; -use ya_transfer::transfer::{ - AbortTransfers, DeployImage, ProgressConfig, TransferService, TransferServiceContext, -}; +use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, TransferServiceContext}; /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. @@ -42,10 +44,10 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -64,22 +66,14 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re }); log::info!("[>>] Deployment with hash verification"); - let result = addr - .send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await?; + let result = addr.send(DeployImage::with_package(&task_package)).await?; log::info!("Deployment stopped"); assert!(result.is_err()); log::info!("Re-deploying the same image"); - addr.send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; Ok(()) } @@ -113,10 +107,10 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -127,24 +121,21 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let addr = TransferService::new(exe_ctx).start(); log::info!("[>>] Deployment with hash verification"); - let (tx, mut rx) = tokio::sync::broadcast::channel(15); + let (tx, mut rx) = mpsc::channel::(15); + let mut msg = DeployImage::with_package(&task_package); + msg.forward_progress( + &ProgressArgs::default(), + tx.sink_map_err(|e| ya_transfer::error::Error::Other(e.to_string())), + ); tokio::task::spawn_local(async move { - let _result = addr - .send(DeployImage { - task_package: task_package.clone(), - progress_config: Some(ProgressConfig { - progress: tx, - progress_args: ProgressArgs::default(), - }), - }) - .await??; + let _result = addr.send(msg).await??; log::info!("Deployment stopped"); anyhow::Ok(()) }); let mut last_progress = 0u64; - while let Ok(progress) = rx.recv().await { + while let Some(progress) = rx.next().await { assert_eq!(progress.progress.1.unwrap(), file_size); assert!(progress.progress.0 >= last_progress); diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index fe2cf29d8e..80efccc800 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -79,10 +79,10 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu .await .expect("unable to start http servers"); - let task_package = Some(format!( + let task_package = format!( "hash://sha3:{}:http://127.0.0.1:8001/rnd", hex::encode(hash) - )); + ); log::debug!("Starting TransferService"); let exe_ctx = TransferServiceContext { @@ -97,20 +97,14 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu println!(); log::warn!("[>>] Deployment with hash verification"); - addr.send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; log::warn!("Deployment complete"); println!(); log::warn!("[>>] Deployment from cache"); - addr.send(DeployImage { - task_package: task_package.clone(), - ..DeployImage::default() - }) - .await??; + addr.send(DeployImage::with_package(&task_package)) + .await??; log::warn!("Deployment from cache complete"); println!(); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index a2cfb911a6..c2d28d6908 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -329,10 +329,7 @@ impl RuntimeRef { .. } => { // TODO: We should pass `task_package` here not in `TransferService` initialization. - let mut msg = DeployImage { - task_package: None, - progress_config: None, - }; + let mut msg = DeployImage::default(); if let Some(args) = progress { msg.forward_progress(args, runtime_cmd.progress_sink()) } From f07e678bfcd1414dd7e9ee62a0da53dedabf48a5 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 9 Jan 2024 17:09:41 +0100 Subject: [PATCH 013/125] Update ya-client --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- exe-unit/components/transfer/src/progress.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1d6749a48..ffd3f7cb13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8178,7 +8178,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" +source = "git+https://github.com/golemfactory/ya-client.git?rev=f2fb8e5392c5fdee0f965309625a561c330a8eae#f2fb8e5392c5fdee0f965309625a561c330a8eae" dependencies = [ "actix-codec", "awc", @@ -8202,7 +8202,7 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=3238c2f5b50c72eb1342a1dbf2c574603c9139f6#3238c2f5b50c72eb1342a1dbf2c574603c9139f6" +source = "git+https://github.com/golemfactory/ya-client.git?rev=f2fb8e5392c5fdee0f965309625a561c330a8eae#f2fb8e5392c5fdee0f965309625a561c330a8eae" dependencies = [ "bigdecimal 0.2.2", "chrono", diff --git a/Cargo.toml b/Cargo.toml index e9ffd15bd3..63d9331e2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -283,8 +283,8 @@ ya-sb-util = { git = "https://github.com/golemfactory/ya-service-bus.git", rev = #ya-sb-util = { path = "../ya-service-bus/crates/util" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "3238c2f5b50c72eb1342a1dbf2c574603c9139f6" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "f2fb8e5392c5fdee0f965309625a561c330a8eae" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "f2fb8e5392c5fdee0f965309625a561c330a8eae" } golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } ## RELAY and networking stack diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index bbddbe9de8..d29cd8de71 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -50,7 +50,7 @@ impl ProgressReporter { /// TODO: implement `update_interval` and `step` pub fn report_progress(&self, progress: u64, size: Option) { let _update_interval = self.config.update_interval; - let _step = self.config.step; + let _update_step = self.config.update_step; self.inner.lock().unwrap().as_mut().map(|inner| { inner.last.progress = (progress, size); From 10e62dbb18a4d0579f669a3cd08c45279bde2c55 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 11 Jan 2024 18:41:29 +0100 Subject: [PATCH 014/125] Limit speed of progress updates to avoid unresponsive exe-unit --- exe-unit/components/transfer/src/progress.rs | 33 ++++++++++++-------- exe-unit/src/lib.rs | 1 + 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index d29cd8de71..91932d8e1c 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -4,6 +4,7 @@ use crate::{TransferContext, TransferData}; use futures::{SinkExt, StreamExt, TryFutureExt}; use std::sync::Arc; +use std::time::Duration; use tokio::task::spawn_local; use tokio::time::Instant; @@ -28,7 +29,7 @@ pub struct ProgressReporter { struct ProgressImpl { pub report: tokio::sync::broadcast::Sender, pub last: CommandProgress, - pub last_update: Instant, + pub last_send: Instant, } impl ProgressReporter { @@ -36,7 +37,7 @@ impl ProgressReporter { self.inner.lock().unwrap().as_mut().map(|inner| { inner.last.step.0 += 1; inner.last.progress = (0, None); - inner.last_update = Instant::now(); + inner.last_send = Instant::now(); inner .report .send(CommandProgress { @@ -49,25 +50,31 @@ impl ProgressReporter { /// TODO: implement `update_interval` and `step` pub fn report_progress(&self, progress: u64, size: Option) { - let _update_interval = self.config.update_interval; + let update_interval: Duration = self + .config + .update_interval + .map(Into::into) + .unwrap_or(Duration::from_secs(1)); let _update_step = self.config.update_step; self.inner.lock().unwrap().as_mut().map(|inner| { inner.last.progress = (progress, size); - inner.last_update = Instant::now(); - inner - .report - .send(CommandProgress { - message: None, - ..inner.last.clone() - }) - .ok() + if inner.last_send + update_interval <= Instant::now() { + inner.last_send = Instant::now(); + inner + .report + .send(CommandProgress { + message: None, + ..inner.last.clone() + }) + .ok(); + } }); } pub fn report_message(&self, message: String) { self.inner.lock().unwrap().as_mut().map(|inner| { - inner.last_update = Instant::now(); + inner.last_send = Instant::now(); inner .report .send(CommandProgress { @@ -93,7 +100,7 @@ impl ProgressReporter { progress: (0, None), unit, }, - last_update: Instant::now(), + last_send: Instant::now(), }); } } diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index c2d28d6908..3bd6322d36 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -98,6 +98,7 @@ impl ExeUnit { let supervisor_template = OfferTemplate::new(serde_json::json!({ "golem.com.usage.vector": MetricsService::usage_vector(), "golem.activity.caps.transfer.protocol": TransferService::schemes(), + "golem.activity.caps.transfer.report-progress": true, })); Ok(supervisor_template.patch(runtime_template)) From d9f9bbc39221cbd8b2e7b85b3e9fd0fe3fa9eb3e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jan 2024 16:43:14 +0100 Subject: [PATCH 015/125] ProgressReporter update interval test --- Cargo.lock | 1 + exe-unit/components/transfer/Cargo.toml | 1 + exe-unit/components/transfer/src/progress.rs | 78 +++++++++++++++++++- exe-unit/components/transfer/src/transfer.rs | 4 +- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ffd3f7cb13..1bcd56c82c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9337,6 +9337,7 @@ dependencies = [ "awc", "bytes 1.5.0", "crossterm 0.26.1", + "duration-string", "env_logger 0.7.1", "futures 0.3.30", "gftp", diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 9e9de1e517..e9e67ddae0 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -72,6 +72,7 @@ actix-files = "0.6" actix-web = "4" anyhow = "1.0" crossterm = "0.26.1" +duration-string = "0.3" env_logger = "0.7" secp256k1 = { version = "0.27.0" } serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 91932d8e1c..e9814d6d43 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -48,7 +48,7 @@ impl ProgressReporter { }); } - /// TODO: implement `update_interval` and `step` + /// TODO: implement `update_step` pub fn report_progress(&self, progress: u64, size: Option) { let update_interval: Duration = self .config @@ -86,12 +86,13 @@ impl ProgressReporter { } pub fn register_reporter( - &self, + &mut self, args: Option, steps: usize, unit: Option, ) { if let Some(args) = args { + self.config = args.progress_args; *(self.inner.lock().unwrap()) = Some(ProgressImpl { report: args.progress, last: CommandProgress { @@ -198,3 +199,76 @@ where sink } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Instant; + + use duration_string::DurationString; + use tokio::time::Duration; + + #[actix_rt::test] + async fn test_progress_reporter_interval() { + let mut report = ProgressReporter::default(); + let (tx, mut rx) = tokio::sync::broadcast::channel(10); + report.register_reporter( + Some(ProgressConfig { + progress: tx, + progress_args: ProgressArgs { + update_interval: Some("500ms".parse::().unwrap()), + update_step: None, + }, + }), + 2, + Some("Bytes".to_string()), + ); + + let size = 200; + let mut before = Instant::now(); + tokio::task::spawn_local(async move { + for _step in 0..2 { + tokio::time::sleep(Duration::from_millis(25)).await; + + for i in 0..=size { + report.report_progress(i, Some(size)); + tokio::time::sleep(Duration::from_millis(50)).await; + } + report.next_step(); + } + }); + + let mut counter = 0; + let mut step = 0; + while let Ok(progress) = rx.recv().await { + //println!("{progress:?}"); + + counter += 1; + let update = Instant::now().duration_since(before); + before = Instant::now(); + let diff = if update > Duration::from_millis(525) { + update - Duration::from_millis(525) + } else { + Duration::from_millis(525) - update + }; + + assert!(diff <= Duration::from_millis(20)); + + // `ProgressReporter` should ignore 10 messages in each loop. + assert_eq!(progress.progress.0, counter * 10); + assert_eq!(progress.progress.1, Some(size)); + assert_eq!(progress.step, (step, 2)); + assert_eq!(progress.unit, Some("Bytes".to_string())); + assert_eq!(progress.message, None); + + if counter == 20 { + counter = 0; + step += 1; + + // Skip step change event + rx.recv().await.unwrap(); + before = Instant::now(); + } + } + } +} diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index 251f87f930..b9e9f55739 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -358,7 +358,7 @@ impl Handler for TransferService { log::info!("Deploying from {:?} to {:?}", src_url.url, path); - let ctx = TransferContext::default(); + let mut ctx = TransferContext::default(); ctx.state.retry_with(self.deploy_retry.clone()); ctx.progress .register_reporter(deploy.progress_config, 1, Some("Bytes".to_string())); @@ -380,7 +380,7 @@ impl Handler for TransferService { let src = actor_try!(self.provider(&src_url)); let dst = actor_try!(self.provider(&dst_url)); - let ctx = TransferContext::default(); + let mut ctx = TransferContext::default(); ctx.state.retry_with(self.transfer_retry.clone()); ctx.progress .register_reporter(msg.progress_config, 1, Some("Bytes".to_string())); From 7ad5482dd438a37c4a5ed34f55fe089633d28ad3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 16 Jan 2024 16:55:30 +0100 Subject: [PATCH 016/125] Test sending progress message --- exe-unit/components/transfer/src/progress.rs | 33 ++++++++++++++------ 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index e9814d6d43..289ed049b2 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -227,21 +227,24 @@ mod tests { let size = 200; let mut before = Instant::now(); tokio::task::spawn_local(async move { - for _step in 0..2 { + for step in 0..2 { tokio::time::sleep(Duration::from_millis(25)).await; for i in 0..=size { report.report_progress(i, Some(size)); tokio::time::sleep(Duration::from_millis(50)).await; } - report.next_step(); + if step == 0 { + report.next_step(); + } } + report.report_message("Finished".to_string()); }); let mut counter = 0; let mut step = 0; - while let Ok(progress) = rx.recv().await { - //println!("{progress:?}"); + while let Ok(event) = rx.recv().await { + //println!("{event:?}"); counter += 1; let update = Instant::now().duration_since(before); @@ -255,13 +258,17 @@ mod tests { assert!(diff <= Duration::from_millis(20)); // `ProgressReporter` should ignore 10 messages in each loop. - assert_eq!(progress.progress.0, counter * 10); - assert_eq!(progress.progress.1, Some(size)); - assert_eq!(progress.step, (step, 2)); - assert_eq!(progress.unit, Some("Bytes".to_string())); - assert_eq!(progress.message, None); + assert_eq!(event.progress.0, counter * 10); + assert_eq!(event.progress.1, Some(size)); + assert_eq!(event.step, (step, 2)); + assert_eq!(event.unit, Some("Bytes".to_string())); + assert_eq!(event.message, None); if counter == 20 { + if step == 1 { + break; + } + counter = 0; step += 1; @@ -270,5 +277,13 @@ mod tests { before = Instant::now(); } } + + // Reporting message will result in event containing progress adn step from previous event. + let last = rx.recv().await.unwrap(); + //println!("{last:?}"); + assert_eq!(last.message, Some("Finished".to_string())); + assert_eq!(last.progress.0, size); + assert_eq!(last.progress.1, Some(size)); + assert_eq!(last.step, (1, 2)); } } From f3b76b1e8226cc5400b3e6434c3d8010ac55bd0d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 17 Jan 2024 13:53:48 +0100 Subject: [PATCH 017/125] Make ExeUnit instantiable in test --- exe-unit/components/transfer/src/transfer.rs | 2 +- exe-unit/src/bin.rs | 337 +------- exe-unit/src/exe_unit.rs | 579 +++++++++++++ exe-unit/src/handlers/local.rs | 6 +- exe-unit/src/lib.rs | 816 +++++++------------ 5 files changed, 875 insertions(+), 865 deletions(-) create mode 100644 exe-unit/src/exe_unit.rs diff --git a/exe-unit/components/transfer/src/transfer.rs b/exe-unit/components/transfer/src/transfer.rs index b9e9f55739..afca8cb6e0 100644 --- a/exe-unit/components/transfer/src/transfer.rs +++ b/exe-unit/components/transfer/src/transfer.rs @@ -248,7 +248,7 @@ impl TransferService { src_url: TransferUrl, _src_name: CachePath, path: PathBuf, - ctx: TransferContext, + _ctx: TransferContext, ) -> ActorResponse>> { let fut = async move { let resp = reqwest::get(src_url.url) diff --git a/exe-unit/src/bin.rs b/exe-unit/src/bin.rs index 53b1e37077..4dc9e0b7bc 100644 --- a/exe-unit/src/bin.rs +++ b/exe-unit/src/bin.rs @@ -1,191 +1,7 @@ -use actix::{Actor, Addr}; -use anyhow::{bail, Context}; -use futures::channel::oneshot; -use std::convert::TryFrom; -use std::path::PathBuf; -use structopt::{clap, StructOpt}; +use structopt::StructOpt; -use ya_client_model::activity::ExeScriptCommand; -use ya_service_bus::RpcEnvelope; - -use ya_core_model::activity; -use ya_exe_unit::agreement::Agreement; use ya_exe_unit::logger::*; -use ya_exe_unit::manifest::ManifestContext; -use ya_exe_unit::message::{GetState, GetStateResponse, Register}; -use ya_exe_unit::runtime::process::RuntimeProcess; -use ya_exe_unit::service::metrics::MetricsService; -use ya_exe_unit::service::signal::SignalMonitor; -use ya_exe_unit::state::Supervision; -use ya_exe_unit::{ExeUnit, ExeUnitContext}; -use ya_transfer::transfer::TransferService; -use ya_utils_path::normalize_path; - -#[derive(structopt::StructOpt, Debug)] -#[structopt(global_setting = clap::AppSettings::ColoredHelp)] -#[structopt(version = ya_compile_time_utils::version_describe!())] -struct Cli { - /// Runtime binary path - #[structopt(long, short)] - binary: PathBuf, - #[structopt(flatten)] - supervise: SuperviseCli, - /// Additional runtime arguments - #[structopt( - long, - short, - set = clap::ArgSettings::Global, - number_of_values = 1, - )] - runtime_arg: Vec, - /// Enclave secret key used in secure communication - #[structopt( - long, - env = "EXE_UNIT_SEC_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, - )] - #[allow(dead_code)] - sec_key: Option, - /// Requestor public key used in secure communication - #[structopt( - long, - env = "EXE_UNIT_REQUESTOR_PUB_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, - )] - #[allow(dead_code)] - requestor_pub_key: Option, - #[structopt(subcommand)] - command: Command, -} - -#[derive(structopt::StructOpt, Debug)] -struct SuperviseCli { - /// Hardware resources are handled by the runtime - #[structopt( - long = "runtime-managed-hardware", - alias = "cap-handoff", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, - )] - hardware: bool, - /// Images are handled by the runtime - #[structopt( - long = "runtime-managed-image", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, - )] - image: bool, -} - -#[derive(structopt::StructOpt, Debug)] -#[structopt(global_setting = clap::AppSettings::DeriveDisplayOrder)] -enum Command { - /// Execute commands from file - FromFile { - /// ExeUnit daemon GSB URL - #[structopt(long)] - report_url: Option, - /// ExeUnit service ID - #[structopt(long)] - service_id: Option, - /// Command file path - input: PathBuf, - #[structopt(flatten)] - args: RunArgs, - }, - /// Bind to Service Bus - ServiceBus { - /// ExeUnit service ID - service_id: String, - /// ExeUnit daemon GSB URL - report_url: String, - #[structopt(flatten)] - args: RunArgs, - }, - /// Print an offer template in JSON format - OfferTemplate, - /// Run runtime's test command - Test, -} - -#[derive(structopt::StructOpt, Debug)] -struct RunArgs { - /// Agreement file path - #[structopt(long, short)] - agreement: PathBuf, - /// Working directory - #[structopt(long, short)] - work_dir: PathBuf, - /// Common cache directory - #[structopt(long, short)] - cache_dir: PathBuf, -} - -fn create_path(path: &PathBuf) -> anyhow::Result { - if let Err(error) = std::fs::create_dir_all(path) { - match &error.kind() { - std::io::ErrorKind::AlreadyExists => (), - _ => bail!("Can't create directory: {}, {}", path.display(), error), - } - } - Ok(normalize_path(path)?) -} - -#[cfg(feature = "sgx")] -fn init_crypto( - sec_key: Option, - req_key: Option, -) -> anyhow::Result { - use ya_exe_unit::crypto::Crypto; - - let req_key = req_key.ok_or_else(|| anyhow::anyhow!("Missing requestor public key"))?; - match sec_key { - Some(key) => Ok(Crypto::try_with_keys(key, req_key)?), - None => { - log::info!("Generating a new key pair..."); - Ok(Crypto::try_new(req_key)?) - } - } -} - -async fn send_script( - exe_unit: Addr>, - activity_id: Option, - exe_script: Vec, -) { - use std::time::Duration; - use ya_exe_unit::state::{State, StatePair}; - - let delay = Duration::from_secs_f32(0.5); - loop { - match exe_unit.send(GetState).await { - Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, - Ok(GetStateResponse(StatePair(State::Terminated, _))) - | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) - | Err(_) => { - return log::error!("ExeUnit has terminated"); - } - _ => tokio::time::sleep(delay).await, - } - } - - log::debug!("Executing commands: {:?}", exe_script); - - let msg = activity::Exec { - activity_id: activity_id.unwrap_or_default(), - batch_id: hex::encode(rand::random::<[u8; 16]>()), - exe_script, - timeout: None, - }; - if let Err(e) = exe_unit - .send(RpcEnvelope::with_caller(String::new(), msg)) - .await - { - log::error!("Unable to execute exe script: {:?}", e); - } -} +use ya_exe_unit::{run, Cli}; #[cfg(feature = "packet-trace-enable")] fn init_packet_trace() -> anyhow::Result<()> { @@ -197,147 +13,6 @@ fn init_packet_trace() -> anyhow::Result<()> { Ok(()) } -async fn run() -> anyhow::Result<()> { - dotenv::dotenv().ok(); - - #[cfg(feature = "packet-trace-enable")] - init_packet_trace()?; - - #[allow(unused_mut)] - let mut cli: Cli = Cli::from_args(); - if !cli.binary.exists() { - bail!("Runtime binary does not exist: {}", cli.binary.display()); - } - - let mut commands = None; - let ctx_activity_id; - let ctx_report_url; - - let args = match &cli.command { - Command::FromFile { - args, - service_id, - report_url, - input, - } => { - let contents = std::fs::read_to_string(input).map_err(|e| { - anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) - })?; - let contents = serde_json::from_str(&contents).map_err(|e| { - anyhow::anyhow!( - "Cannot deserialize commands from file {}: {e}", - input.display(), - ) - })?; - ctx_activity_id = service_id.clone(); - ctx_report_url = report_url.clone(); - commands = Some(contents); - args - } - Command::ServiceBus { - args, - service_id, - report_url, - } => { - ctx_activity_id = Some(service_id.clone()); - ctx_report_url = Some(report_url.clone()); - args - } - Command::OfferTemplate => { - let args = cli.runtime_arg.clone(); - let offer_template = ExeUnit::::offer_template(cli.binary, args)?; - println!("{}", serde_json::to_string(&offer_template)?); - return Ok(()); - } - Command::Test => { - let args = cli.runtime_arg.clone(); - let output = ExeUnit::::test(cli.binary, args)?; - println!("{}", String::from_utf8_lossy(&output.stdout)); - eprintln!("{}", String::from_utf8_lossy(&output.stderr)); - if !output.status.success() { - bail!("Test failed"); - } - return Ok(()); - } - }; - - if !args.agreement.exists() { - bail!( - "Agreement file does not exist: {}", - args.agreement.display() - ); - } - let work_dir = create_path(&args.work_dir).map_err(|e| { - anyhow::anyhow!( - "Cannot create the working directory {}: {e}", - args.work_dir.display(), - ) - })?; - let cache_dir = create_path(&args.cache_dir).map_err(|e| { - anyhow::anyhow!( - "Cannot create the cache directory {}: {e}", - args.work_dir.display(), - ) - })?; - let mut agreement = Agreement::try_from(&args.agreement).map_err(|e| { - anyhow::anyhow!( - "Error parsing the agreement from {}: {e}", - args.agreement.display(), - ) - })?; - - log::info!("Attempting to read app manifest .."); - - let manifest_ctx = - ManifestContext::try_new(&agreement.inner).context("Invalid app manifest")?; - agreement.task_package = manifest_ctx - .payload() - .or_else(|| agreement.task_package.take()); - - log::info!("Manifest-enabled features: {:?}", manifest_ctx.features()); - log::info!("User-provided payload: {:?}", agreement.task_package); - - let ctx = ExeUnitContext { - supervise: Supervision { - hardware: cli.supervise.hardware, - image: cli.supervise.image, - manifest: manifest_ctx, - }, - activity_id: ctx_activity_id.clone(), - report_url: ctx_report_url, - agreement, - work_dir, - cache_dir, - runtime_args: cli.runtime_arg.clone(), - acl: Default::default(), - credentials: None, - #[cfg(feature = "sgx")] - crypto: init_crypto( - cli.sec_key.replace("".into()), - cli.requestor_pub_key.clone(), - )?, - }; - - log::debug!("CLI args: {:?}", cli); - log::debug!("ExeUnitContext args: {:?}", ctx); - - let (tx, rx) = oneshot::channel(); - - let metrics = MetricsService::try_new(&ctx, Some(10000), ctx.supervise.hardware)?.start(); - let transfers = TransferService::new((&ctx).into()).start(); - let runtime = RuntimeProcess::new(&ctx, cli.binary).start(); - let exe_unit = ExeUnit::new(tx, ctx, metrics, transfers, runtime).start(); - let signals = SignalMonitor::new(exe_unit.clone()).start(); - exe_unit.send(Register(signals)).await?; - - if let Some(exe_script) = commands { - tokio::task::spawn(send_script(exe_unit, ctx_activity_id, exe_script)); - } - - rx.await??; - Ok(()) -} - #[actix_rt::main] async fn main() { let panic_hook = std::panic::take_hook(); @@ -351,7 +26,13 @@ async fn main() { log::warn!("Using fallback logging due to an error: {:?}", error); }; - std::process::exit(match run().await { + dotenv::dotenv().ok(); + #[cfg(feature = "packet-trace-enable")] + init_packet_trace()?; + + let cli: Cli = Cli::from_args(); + + std::process::exit(match run(cli).await { Ok(_) => 0, Err(error) => { log::error!("{}", error); diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs new file mode 100644 index 0000000000..b666fdd259 --- /dev/null +++ b/exe-unit/src/exe_unit.rs @@ -0,0 +1,579 @@ +use actix::dev::IntervalFunc; +use actix::{ + Actor, ActorFutureExt, ActorStreamExt, Addr, AsyncContext, Context, ContextFutureSpawner, + Handler, ResponseFuture, Running, StreamHandler, WrapFuture, +}; +use chrono::Utc; +use futures::channel::{mpsc, oneshot}; +use futures::{FutureExt, SinkExt}; +use std::path::PathBuf; +use std::time::Duration; + +use ya_agreement_utils::OfferTemplate; +use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; +use ya_core_model::activity; +use ya_core_model::activity::local::Credentials; +use ya_runtime_api::deploy; +use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; +use ya_transfer::transfer::{ + AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, + TransferServiceContext, +}; + +use crate::acl::Acl; +use crate::agreement::Agreement; +use crate::error::Error; +use crate::message::{ + ExecuteCommand, GetMetrics, GetStdOut, Initialize, RuntimeEvent, SetState, Shutdown, + ShutdownReason, SignExeScript, Stop, UpdateDeployment, +}; +use crate::runtime::{Runtime, RuntimeMode}; +use crate::service::metrics::MetricsService; +use crate::service::{ServiceAddr, ServiceControl}; +use crate::state::{ExeUnitState, StateError, Supervision}; + +lazy_static::lazy_static! { + static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); +} + +pub struct ExeUnit { + pub(crate) ctx: ExeUnitContext, + pub(crate) state: ExeUnitState, + pub(crate) events: Channel, + pub(crate) runtime: Addr, + pub(crate) metrics: Addr, + pub(crate) transfers: Addr, + pub(crate) services: Vec>, + pub(crate) shutdown_tx: Option>>, +} + +impl ExeUnit { + pub fn new( + shutdown_tx: oneshot::Sender>, + ctx: ExeUnitContext, + metrics: Addr, + transfers: Addr, + runtime: Addr, + ) -> Self { + ExeUnit { + ctx, + state: ExeUnitState::default(), + events: Channel::default(), + runtime: runtime.clone(), + metrics: metrics.clone(), + transfers: transfers.clone(), + services: vec![ + Box::new(ServiceAddr::new(metrics)), + Box::new(ServiceAddr::new(transfers)), + Box::new(ServiceAddr::new(runtime)), + ], + shutdown_tx: Some(shutdown_tx), + } + } + + pub fn offer_template(binary: PathBuf, args: Vec) -> crate::Result { + use crate::runtime::process::RuntimeProcess; + + let runtime_template = RuntimeProcess::offer_template(binary, args)?; + let supervisor_template = OfferTemplate::new(serde_json::json!({ + "golem.com.usage.vector": MetricsService::usage_vector(), + "golem.activity.caps.transfer.protocol": TransferService::schemes(), + "golem.activity.caps.transfer.report-progress": true, + })); + + Ok(supervisor_template.patch(runtime_template)) + } + + pub fn test(binary: PathBuf, args: Vec) -> crate::Result { + use crate::runtime::process::RuntimeProcess; + RuntimeProcess::test(binary, args) + } + + fn report_usage(&mut self, context: &mut Context) { + if self.ctx.activity_id.is_none() || self.ctx.report_url.is_none() { + return; + } + let fut = report_usage( + self.ctx.report_url.clone().unwrap(), + self.ctx.activity_id.clone().unwrap(), + context.address(), + self.metrics.clone(), + ); + context.spawn(fut.into_actor(self)); + } + + pub(crate) async fn stop_runtime(runtime: Addr, reason: ShutdownReason) { + if let Err(e) = runtime + .send(Shutdown(reason)) + .timeout(Duration::from_secs(5u64)) + .await + { + log::warn!("Unable to stop the runtime: {:?}", e); + } + } +} + +#[derive(Clone)] +pub struct RuntimeRef(Addr>); + +impl RuntimeRef { + pub fn from_ctx(ctx: &Context>) -> Self { + RuntimeRef(ctx.address()) + } +} + +impl std::ops::Deref for RuntimeRef { + type Target = Addr>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl RuntimeRef { + pub async fn exec( + self, + exec: activity::Exec, + runtime: Addr, + transfers: Addr, + mut events: mpsc::Sender, + mut control: oneshot::Receiver<()>, + ) { + let batch_id = exec.batch_id.clone(); + for (idx, command) in exec.exe_script.into_iter().enumerate() { + if let Ok(Some(_)) = control.try_recv() { + log::warn!("Batch {} execution aborted", batch_id); + break; + } + + let runtime_cmd = ExecuteCommand { + batch_id: batch_id.clone(), + command: command.clone(), + tx: events.clone(), + idx, + }; + + let evt = RuntimeEvent::started(batch_id.clone(), idx, command.clone()); + if let Err(e) = events.send(evt).await { + log::error!("Unable to report event: {:?}", e); + } + + let (return_code, message) = match { + if runtime_cmd.stateless() { + self.exec_stateless(&runtime_cmd).await + } else { + self.exec_stateful(runtime_cmd, &runtime, &transfers).await + } + } { + Ok(_) => (0, None), + Err(ref err) => match err { + Error::CommandExitCodeError(c) => (*c, Some(err.to_string())), + _ => (-1, Some(err.to_string())), + }, + }; + + let evt = RuntimeEvent::finished(batch_id.clone(), idx, return_code, message.clone()); + if let Err(e) = events.send(evt).await { + log::error!("Unable to report event: {:?}", e); + } + + if return_code != 0 { + let message = message.unwrap_or_else(|| "reason unspecified".into()); + log::warn!("Batch {} execution interrupted: {}", batch_id, message); + break; + } + } + } + + async fn exec_stateless(&self, runtime_cmd: &ExecuteCommand) -> crate::Result<()> { + match runtime_cmd.command { + ExeScriptCommand::Sign {} => { + let batch_id = runtime_cmd.batch_id.clone(); + let signature = self.send(SignExeScript { batch_id }).await??; + let stdout = serde_json::to_string(&signature)?; + + runtime_cmd + .tx + .clone() + .send(RuntimeEvent::stdout( + runtime_cmd.batch_id.clone(), + runtime_cmd.idx, + CommandOutput::Bin(stdout.into_bytes()), + )) + .await + .map_err(|e| Error::runtime(format!("Unable to send stdout event: {:?}", e)))?; + } + ExeScriptCommand::Terminate {} => { + log::debug!("Terminating running ExeScripts"); + let exclude_batches = vec![runtime_cmd.batch_id.clone()]; + self.send(Stop { exclude_batches }).await??; + self.send(SetState::from(State::Initialized)).await?; + } + _ => (), + } + Ok(()) + } + + async fn exec_stateful( + &self, + runtime_cmd: ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + let state = self.send(crate::message::GetState {}).await?.0; + let state_pre = match (&state.0, &state.1) { + (_, Some(_)) => { + return Err(StateError::Busy(state).into()); + } + (State::New, _) | (State::Terminated, _) => { + return Err(StateError::InvalidState(state).into()); + } + (State::Initialized, _) => match &runtime_cmd.command { + ExeScriptCommand::Deploy { .. } => { + StatePair(State::Initialized, Some(State::Deployed)) + } + _ => return Err(StateError::InvalidState(state).into()), + }, + (State::Deployed, _) => match &runtime_cmd.command { + ExeScriptCommand::Start { .. } => StatePair(State::Deployed, Some(State::Ready)), + _ => return Err(StateError::InvalidState(state).into()), + }, + (s, _) => match &runtime_cmd.command { + ExeScriptCommand::Deploy { .. } | ExeScriptCommand::Start { .. } => { + return Err(StateError::InvalidState(state).into()); + } + _ => StatePair(*s, Some(*s)), + }, + }; + self.send(SetState::from(state_pre)).await?; + + log::info!("Executing command: {:?}", runtime_cmd.command); + + let result = async { + self.pre_runtime(&runtime_cmd, runtime, transfer_service) + .await?; + + let exit_code = runtime.send(runtime_cmd.clone()).await??; + if exit_code != 0 { + return Err(Error::CommandExitCodeError(exit_code)); + } + + self.post_runtime(&runtime_cmd, runtime, transfer_service) + .await?; + + Ok(()) + } + .await; + + let state_cur = self.send(crate::message::GetState {}).await?.0; + if state_cur != state_pre { + return Err(StateError::UnexpectedState { + current: state_cur, + expected: state_pre, + } + .into()); + } + + self.send(SetState::from(state_pre.1.unwrap())).await?; + result + } + + async fn pre_runtime( + &self, + runtime_cmd: &ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + match &runtime_cmd.command { + ExeScriptCommand::Transfer { + from, + to, + args, + progress, + } => { + let mut msg = TransferResource { + from: from.clone(), + to: to.clone(), + args: args.clone(), + progress_config: None, + }; + + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + transfer_service.send(msg).await??; + } + ExeScriptCommand::Deploy { + net, + hosts, + progress, + .. + } => { + // TODO: We should pass `task_package` here not in `TransferService` initialization. + let mut msg = DeployImage::default(); + if let Some(args) = progress { + msg.forward_progress(args, runtime_cmd.progress_sink()) + } + + let task_package = transfer_service.send(msg).await??; + runtime + .send(UpdateDeployment { + task_package, + networks: Some(net.clone()), + hosts: Some(hosts.clone()), + ..Default::default() + }) + .await??; + } + _ => (), + } + Ok(()) + } + + async fn post_runtime( + &self, + runtime_cmd: &ExecuteCommand, + runtime: &Addr, + transfer_service: &Addr, + ) -> crate::Result<()> { + if let ExeScriptCommand::Deploy { .. } = &runtime_cmd.command { + let mut runtime_mode = RuntimeMode::ProcessPerCommand; + let stdout = self + .send(GetStdOut { + batch_id: runtime_cmd.batch_id.clone(), + idx: runtime_cmd.idx, + }) + .await?; + + if let Some(output) = stdout { + let deployment = deploy::DeployResult::from_bytes(output).map_err(|e| { + log::error!("Deployment failed: {}", e); + Error::CommandError(e.to_string()) + })?; + transfer_service + .send(AddVolumes::new(deployment.vols)) + .await??; + runtime_mode = deployment.start_mode.into(); + } + runtime + .send(UpdateDeployment { + runtime_mode: Some(runtime_mode), + ..Default::default() + }) + .await??; + } + Ok(()) + } +} + +impl Actor for ExeUnit { + type Context = Context; + + fn started(&mut self, ctx: &mut Self::Context) { + let rx = self.events.rx.take().unwrap(); + Self::add_stream(rx, ctx); + + let addr = ctx.address(); + if let Some(activity_id) = &self.ctx.activity_id { + let srv_id = activity::exeunit::bus_id(activity_id); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + + #[cfg(feature = "sgx")] + { + actix_rpc::bind::( + &srv_id, + addr.clone().recipient(), + ); + } + #[cfg(not(feature = "sgx"))] + { + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::bind::(&srv_id, addr.clone().recipient()); + actix_rpc::binds::( + &srv_id, + addr.clone().recipient(), + ); + } + } + + IntervalFunc::new(*DEFAULT_REPORT_INTERVAL, Self::report_usage) + .finish() + .spawn(ctx); + + log::info!("Initializing manifests"); + self.ctx + .supervise + .manifest + .build_validators() + .into_actor(self) + .map(|result, this, ctx| match result { + Ok(validators) => { + this.ctx.supervise.manifest.add_validators(validators); + log::info!("Manifest initialization complete"); + } + Err(e) => { + let err = Error::Other(format!("manifest initialization error: {}", e)); + log::error!("Supervisor is shutting down due to {}", err); + ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); + } + }) + .wait(ctx); + + let addr_ = addr.clone(); + async move { + addr.send(Initialize).await?.map_err(Error::from)?; + addr.send(SetState::from(State::Initialized)).await?; + Ok::<_, Error>(()) + } + .then(|result| async move { + match result { + Ok(_) => log::info!("Supervisor initialized"), + Err(e) => { + let err = Error::Other(format!("initialization error: {}", e)); + log::error!("Supervisor is shutting down due to {}", err); + let _ = addr_.send(Shutdown(ShutdownReason::Error(err))).await; + } + } + }) + .into_actor(self) + .spawn(ctx); + } + + fn stopping(&mut self, _: &mut Self::Context) -> Running { + if self.state.inner.0 == State::Terminated { + return Running::Stop; + } + Running::Continue + } + + fn stopped(&mut self, _: &mut Self::Context) { + if let Some(tx) = self.shutdown_tx.take() { + let _ = tx.send(Ok(())); + } + } +} + +#[derive(derivative::Derivative)] +#[derivative(Debug)] +pub struct ExeUnitContext { + pub supervise: Supervision, + pub activity_id: Option, + pub report_url: Option, + pub agreement: Agreement, + pub work_dir: PathBuf, + pub cache_dir: PathBuf, + pub runtime_args: Vec, + pub acl: Acl, + pub credentials: Option, + #[cfg(feature = "sgx")] + #[derivative(Debug = "ignore")] + pub crypto: crate::crypto::Crypto, +} + +impl ExeUnitContext { + pub fn verify_activity_id(&self, activity_id: &str) -> crate::Result<()> { + match &self.activity_id { + Some(act_id) => match act_id == activity_id { + true => Ok(()), + false => Err(Error::RemoteServiceError(format!( + "Forbidden! Invalid activity id: {}", + activity_id + ))), + }, + None => Ok(()), + } + } +} + +impl From<&ExeUnitContext> for TransferServiceContext { + fn from(val: &ExeUnitContext) -> Self { + TransferServiceContext { + task_package: val.agreement.task_package.clone(), + deploy_retry: None, + cache_dir: val.cache_dir.clone(), + work_dir: val.work_dir.clone(), + transfer_retry: None, + } + } +} + +pub struct Channel { + pub(crate) tx: mpsc::Sender, + rx: Option>, +} + +impl Default for Channel { + fn default() -> Self { + let (tx, rx) = mpsc::channel(8); + Channel { tx, rx: Some(rx) } + } +} + +pub async fn report(url: S, msg: M) -> bool +where + M: RpcMessage + Unpin + 'static, + S: AsRef, +{ + let url = url.as_ref(); + match ya_service_bus::typed::service(url).send(msg).await { + Err(ya_service_bus::Error::Timeout(msg)) => { + log::warn!("Timed out reporting to {}: {}", url, msg); + true + } + Err(e) => { + log::error!("Error reporting to {}: {:?}", url, e); + false + } + Ok(Err(e)) => { + log::error!("Error response while reporting to {}: {:?}", url, e); + false + } + Ok(Ok(_)) => true, + } +} + +async fn report_usage( + report_url: String, + activity_id: String, + exe_unit: Addr>, + metrics: Addr, +) { + match metrics.send(GetMetrics).await { + Ok(resp) => match resp { + Ok(data) => { + let msg = activity::local::SetUsage { + activity_id, + usage: ActivityUsage { + current_usage: Some(data), + timestamp: Utc::now().timestamp(), + }, + timeout: None, + }; + if !report(&report_url, msg).await { + exe_unit.do_send(Shutdown(ShutdownReason::Error(Error::RuntimeError( + format!("Reporting endpoint '{}' is not available", report_url), + )))); + } + } + Err(err) => match err { + Error::UsageLimitExceeded(info) => { + log::warn!("Usage limit exceeded: {}", info); + exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); + } + error => log::warn!("Unable to retrieve metrics: {:?}", error), + }, + }, + Err(e) => log::warn!("Unable to report activity usage: {:?}", e), + } +} + +impl Handler for TransferService { + type Result = ResponseFuture>; + + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + let addr = ctx.address(); + async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() + } +} diff --git a/exe-unit/src/handlers/local.rs b/exe-unit/src/handlers/local.rs index 2a02222ae4..3236695ecf 100644 --- a/exe-unit/src/handlers/local.rs +++ b/exe-unit/src/handlers/local.rs @@ -1,11 +1,13 @@ +use actix::prelude::*; +use futures::FutureExt; + use crate::error::Error; use crate::message::*; use crate::runtime::Runtime; use crate::service::ServiceAddr; use crate::state::State; use crate::{report, ExeUnit}; -use actix::prelude::*; -use futures::FutureExt; + use ya_client_model::activity; use ya_core_model::activity::local::SetState as SetActivityState; diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 3bd6322d36..388d81f06a 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -1,35 +1,27 @@ #[macro_use] extern crate derive_more; -use std::path::PathBuf; -use std::time::Duration; - use actix::prelude::*; -use chrono::Utc; -use futures::channel::{mpsc, oneshot}; -use futures::{FutureExt, SinkExt}; +use anyhow::{bail, Context}; +use futures::channel::oneshot; +use std::convert::TryFrom; +use std::path::PathBuf; +use structopt::clap; -use ya_agreement_utils::agreement::OfferTemplate; -use ya_client_model::activity::{ - activity_state::StatePair, ActivityUsage, CommandOutput, ExeScriptCommand, State, -}; +use ya_client_model::activity::ExeScriptCommand; use ya_core_model::activity; -use ya_core_model::activity::local::Credentials; -use ya_runtime_api::deploy; -use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; -use ya_transfer::transfer::{ - AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, - TransferServiceContext, -}; +use ya_service_bus::RpcEnvelope; +use ya_transfer::transfer::TransferService; +use ya_utils_path::normalize_path; -use crate::acl::Acl; use crate::agreement::Agreement; use crate::error::Error; -use crate::message::*; -use crate::runtime::*; +use crate::manifest::ManifestContext; +use crate::message::{GetState, GetStateResponse, Register}; +use crate::runtime::process::RuntimeProcess; use crate::service::metrics::MetricsService; -use crate::service::{ServiceAddr, ServiceControl}; -use crate::state::{ExeUnitState, StateError, Supervision}; +use crate::service::signal::SignalMonitor; +use crate::state::Supervision; mod acl; pub mod agreement; @@ -50,550 +42,306 @@ pub mod service; pub mod state; mod dns; -pub type Result = std::result::Result; - -lazy_static::lazy_static! { - static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); -} - -pub struct ExeUnit { - ctx: ExeUnitContext, - state: ExeUnitState, - events: Channel, - runtime: Addr, - metrics: Addr, - transfers: Addr, - services: Vec>, - shutdown_tx: Option>>, -} - -impl ExeUnit { - pub fn new( - shutdown_tx: oneshot::Sender>, - ctx: ExeUnitContext, - metrics: Addr, - transfers: Addr, - runtime: Addr, - ) -> Self { - ExeUnit { - ctx, - state: ExeUnitState::default(), - events: Channel::default(), - runtime: runtime.clone(), - metrics: metrics.clone(), - transfers: transfers.clone(), - services: vec![ - Box::new(ServiceAddr::new(metrics)), - Box::new(ServiceAddr::new(transfers)), - Box::new(ServiceAddr::new(runtime)), - ], - shutdown_tx: Some(shutdown_tx), - } - } - - pub fn offer_template(binary: PathBuf, args: Vec) -> Result { - use crate::runtime::process::RuntimeProcess; - - let runtime_template = RuntimeProcess::offer_template(binary, args)?; - let supervisor_template = OfferTemplate::new(serde_json::json!({ - "golem.com.usage.vector": MetricsService::usage_vector(), - "golem.activity.caps.transfer.protocol": TransferService::schemes(), - "golem.activity.caps.transfer.report-progress": true, - })); +mod exe_unit; - Ok(supervisor_template.patch(runtime_template)) - } - - pub fn test(binary: PathBuf, args: Vec) -> Result { - use crate::runtime::process::RuntimeProcess; - RuntimeProcess::test(binary, args) - } +pub use exe_unit::{report, ExeUnit, ExeUnitContext, RuntimeRef}; - fn report_usage(&mut self, context: &mut Context) { - if self.ctx.activity_id.is_none() || self.ctx.report_url.is_none() { - return; - } - let fut = report_usage( - self.ctx.report_url.clone().unwrap(), - self.ctx.activity_id.clone().unwrap(), - context.address(), - self.metrics.clone(), - ); - context.spawn(fut.into_actor(self)); - } +pub type Result = std::result::Result; - async fn stop_runtime(runtime: Addr, reason: ShutdownReason) { - if let Err(e) = runtime - .send(Shutdown(reason)) - .timeout(Duration::from_secs(5u64)) - .await - { - log::warn!("Unable to stop the runtime: {:?}", e); - } - } +#[derive(structopt::StructOpt, Debug)] +#[structopt(global_setting = clap::AppSettings::ColoredHelp)] +#[structopt(version = ya_compile_time_utils::version_describe!())] +pub struct Cli { + /// Runtime binary path + #[structopt(long, short)] + binary: PathBuf, + #[structopt(flatten)] + supervise: SuperviseCli, + /// Additional runtime arguments + #[structopt( + long, + short, + set = clap::ArgSettings::Global, + number_of_values = 1, + )] + runtime_arg: Vec, + /// Enclave secret key used in secure communication + #[structopt( + long, + env = "EXE_UNIT_SEC_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, + )] + #[allow(dead_code)] + sec_key: Option, + /// Requestor public key used in secure communication + #[structopt( + long, + env = "EXE_UNIT_REQUESTOR_PUB_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, + )] + #[allow(dead_code)] + requestor_pub_key: Option, + #[structopt(subcommand)] + command: Command, } -#[derive(Clone)] -struct RuntimeRef(Addr>); - -impl RuntimeRef { - fn from_ctx(ctx: &Context>) -> Self { - RuntimeRef(ctx.address()) - } +#[derive(structopt::StructOpt, Debug)] +pub struct SuperviseCli { + /// Hardware resources are handled by the runtime + #[structopt( + long = "runtime-managed-hardware", + alias = "cap-handoff", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, + )] + hardware: bool, + /// Images are handled by the runtime + #[structopt( + long = "runtime-managed-image", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, + )] + image: bool, } -impl std::ops::Deref for RuntimeRef { - type Target = Addr>; - - fn deref(&self) -> &Self::Target { - &self.0 - } +#[derive(structopt::StructOpt, Debug)] +#[structopt(global_setting = clap::AppSettings::DeriveDisplayOrder)] +pub enum Command { + /// Execute commands from file + FromFile { + /// ExeUnit daemon GSB URL + #[structopt(long)] + report_url: Option, + /// ExeUnit service ID + #[structopt(long)] + service_id: Option, + /// Command file path + input: PathBuf, + #[structopt(flatten)] + args: RunArgs, + }, + /// Bind to Service Bus + ServiceBus { + /// ExeUnit service ID + service_id: String, + /// ExeUnit daemon GSB URL + report_url: String, + #[structopt(flatten)] + args: RunArgs, + }, + /// Print an offer template in JSON format + OfferTemplate, + /// Run runtime's test command + Test, } -impl RuntimeRef { - async fn exec( - self, - exec: activity::Exec, - runtime: Addr, - transfers: Addr, - mut events: mpsc::Sender, - mut control: oneshot::Receiver<()>, - ) { - let batch_id = exec.batch_id.clone(); - for (idx, command) in exec.exe_script.into_iter().enumerate() { - if let Ok(Some(_)) = control.try_recv() { - log::warn!("Batch {} execution aborted", batch_id); - break; - } - - let runtime_cmd = ExecuteCommand { - batch_id: batch_id.clone(), - command: command.clone(), - tx: events.clone(), - idx, - }; - - let evt = RuntimeEvent::started(batch_id.clone(), idx, command.clone()); - if let Err(e) = events.send(evt).await { - log::error!("Unable to report event: {:?}", e); - } - - let (return_code, message) = match { - if runtime_cmd.stateless() { - self.exec_stateless(&runtime_cmd).await - } else { - self.exec_stateful(runtime_cmd, &runtime, &transfers).await - } - } { - Ok(_) => (0, None), - Err(ref err) => match err { - Error::CommandExitCodeError(c) => (*c, Some(err.to_string())), - _ => (-1, Some(err.to_string())), - }, - }; - - let evt = RuntimeEvent::finished(batch_id.clone(), idx, return_code, message.clone()); - if let Err(e) = events.send(evt).await { - log::error!("Unable to report event: {:?}", e); - } - - if return_code != 0 { - let message = message.unwrap_or_else(|| "reason unspecified".into()); - log::warn!("Batch {} execution interrupted: {}", batch_id, message); - break; - } - } - } - - async fn exec_stateless(&self, runtime_cmd: &ExecuteCommand) -> Result<()> { - match runtime_cmd.command { - ExeScriptCommand::Sign {} => { - let batch_id = runtime_cmd.batch_id.clone(); - let signature = self.send(SignExeScript { batch_id }).await??; - let stdout = serde_json::to_string(&signature)?; - - runtime_cmd - .tx - .clone() - .send(RuntimeEvent::stdout( - runtime_cmd.batch_id.clone(), - runtime_cmd.idx, - CommandOutput::Bin(stdout.into_bytes()), - )) - .await - .map_err(|e| Error::runtime(format!("Unable to send stdout event: {:?}", e)))?; - } - ExeScriptCommand::Terminate {} => { - log::debug!("Terminating running ExeScripts"); - let exclude_batches = vec![runtime_cmd.batch_id.clone()]; - self.send(Stop { exclude_batches }).await??; - self.send(SetState::from(State::Initialized)).await?; - } - _ => (), - } - Ok(()) - } - - async fn exec_stateful( - &self, - runtime_cmd: ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - let state = self.send(GetState {}).await?.0; - let state_pre = match (&state.0, &state.1) { - (_, Some(_)) => { - return Err(StateError::Busy(state).into()); - } - (State::New, _) | (State::Terminated, _) => { - return Err(StateError::InvalidState(state).into()); - } - (State::Initialized, _) => match &runtime_cmd.command { - ExeScriptCommand::Deploy { .. } => { - StatePair(State::Initialized, Some(State::Deployed)) - } - _ => return Err(StateError::InvalidState(state).into()), - }, - (State::Deployed, _) => match &runtime_cmd.command { - ExeScriptCommand::Start { .. } => StatePair(State::Deployed, Some(State::Ready)), - _ => return Err(StateError::InvalidState(state).into()), - }, - (s, _) => match &runtime_cmd.command { - ExeScriptCommand::Deploy { .. } | ExeScriptCommand::Start { .. } => { - return Err(StateError::InvalidState(state).into()); - } - _ => StatePair(*s, Some(*s)), - }, - }; - self.send(SetState::from(state_pre)).await?; - - log::info!("Executing command: {:?}", runtime_cmd.command); - - let result = async { - self.pre_runtime(&runtime_cmd, runtime, transfer_service) - .await?; - - let exit_code = runtime.send(runtime_cmd.clone()).await??; - if exit_code != 0 { - return Err(Error::CommandExitCodeError(exit_code)); - } - - self.post_runtime(&runtime_cmd, runtime, transfer_service) - .await?; - - Ok(()) - } - .await; - - let state_cur = self.send(GetState {}).await?.0; - if state_cur != state_pre { - return Err(StateError::UnexpectedState { - current: state_cur, - expected: state_pre, - } - .into()); - } - - self.send(SetState::from(state_pre.1.unwrap())).await?; - result - } - - async fn pre_runtime( - &self, - runtime_cmd: &ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - match &runtime_cmd.command { - ExeScriptCommand::Transfer { - from, - to, - args, - progress, - } => { - let mut msg = TransferResource { - from: from.clone(), - to: to.clone(), - args: args.clone(), - progress_config: None, - }; - - if let Some(args) = progress { - msg.forward_progress(args, runtime_cmd.progress_sink()) - } - transfer_service.send(msg).await??; - } - ExeScriptCommand::Deploy { - net, - hosts, - progress, - .. - } => { - // TODO: We should pass `task_package` here not in `TransferService` initialization. - let mut msg = DeployImage::default(); - if let Some(args) = progress { - msg.forward_progress(args, runtime_cmd.progress_sink()) - } +#[derive(structopt::StructOpt, Debug)] +pub struct RunArgs { + /// Agreement file path + #[structopt(long, short)] + agreement: PathBuf, + /// Working directory + #[structopt(long, short)] + work_dir: PathBuf, + /// Common cache directory + #[structopt(long, short)] + cache_dir: PathBuf, +} - let task_package = transfer_service.send(msg).await??; - runtime - .send(UpdateDeployment { - task_package, - networks: Some(net.clone()), - hosts: Some(hosts.clone()), - ..Default::default() - }) - .await??; - } - _ => (), +fn create_path(path: &PathBuf) -> anyhow::Result { + if let Err(error) = std::fs::create_dir_all(path) { + match &error.kind() { + std::io::ErrorKind::AlreadyExists => (), + _ => bail!("Can't create directory: {}, {}", path.display(), error), } - Ok(()) } + Ok(normalize_path(path)?) +} - async fn post_runtime( - &self, - runtime_cmd: &ExecuteCommand, - runtime: &Addr, - transfer_service: &Addr, - ) -> Result<()> { - if let ExeScriptCommand::Deploy { .. } = &runtime_cmd.command { - let mut runtime_mode = RuntimeMode::ProcessPerCommand; - let stdout = self - .send(GetStdOut { - batch_id: runtime_cmd.batch_id.clone(), - idx: runtime_cmd.idx, - }) - .await?; - - if let Some(output) = stdout { - let deployment = deploy::DeployResult::from_bytes(output).map_err(|e| { - log::error!("Deployment failed: {}", e); - Error::CommandError(e.to_string()) - })?; - transfer_service - .send(AddVolumes::new(deployment.vols)) - .await??; - runtime_mode = deployment.start_mode.into(); - } - runtime - .send(UpdateDeployment { - runtime_mode: Some(runtime_mode), - ..Default::default() - }) - .await??; +#[cfg(feature = "sgx")] +fn init_crypto( + sec_key: Option, + req_key: Option, +) -> anyhow::Result { + let req_key = req_key.ok_or_else(|| anyhow::anyhow!("Missing requestor public key"))?; + match sec_key { + Some(key) => Ok(crate::crypto::Crypto::try_with_keys(key, req_key)?), + None => { + log::info!("Generating a new key pair..."); + Ok(crate::crypto::Crypto::try_new(req_key)?) } - Ok(()) } } -impl Actor for ExeUnit { - type Context = Context; - - fn started(&mut self, ctx: &mut Self::Context) { - let rx = self.events.rx.take().unwrap(); - Self::add_stream(rx, ctx); - - let addr = ctx.address(); - if let Some(activity_id) = &self.ctx.activity_id { - let srv_id = activity::exeunit::bus_id(activity_id); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - - #[cfg(feature = "sgx")] - { - actix_rpc::bind::( - &srv_id, - addr.clone().recipient(), - ); - } - #[cfg(not(feature = "sgx"))] - { - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::bind::(&srv_id, addr.clone().recipient()); - actix_rpc::binds::( - &srv_id, - addr.clone().recipient(), - ); +async fn send_script( + exe_unit: Addr>, + activity_id: Option, + exe_script: Vec, +) { + use crate::state::{State, StatePair}; + use std::time::Duration; + + let delay = Duration::from_secs_f32(0.5); + loop { + match exe_unit.send(GetState).await { + Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, + Ok(GetStateResponse(StatePair(State::Terminated, _))) + | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) + | Err(_) => { + return log::error!("ExeUnit has terminated"); } + _ => tokio::time::sleep(delay).await, } - - IntervalFunc::new(*DEFAULT_REPORT_INTERVAL, Self::report_usage) - .finish() - .spawn(ctx); - - log::info!("Initializing manifests"); - self.ctx - .supervise - .manifest - .build_validators() - .into_actor(self) - .map(|result, this, ctx| match result { - Ok(validators) => { - this.ctx.supervise.manifest.add_validators(validators); - log::info!("Manifest initialization complete"); - } - Err(e) => { - let err = Error::Other(format!("manifest initialization error: {}", e)); - log::error!("Supervisor is shutting down due to {}", err); - ctx.address().do_send(Shutdown(ShutdownReason::Error(err))); - } - }) - .wait(ctx); - - let addr_ = addr.clone(); - async move { - addr.send(Initialize).await?.map_err(Error::from)?; - addr.send(SetState::from(State::Initialized)).await?; - Ok::<_, Error>(()) - } - .then(|result| async move { - match result { - Ok(_) => log::info!("Supervisor initialized"), - Err(e) => { - let err = Error::Other(format!("initialization error: {}", e)); - log::error!("Supervisor is shutting down due to {}", err); - let _ = addr_.send(Shutdown(ShutdownReason::Error(err))).await; - } - } - }) - .into_actor(self) - .spawn(ctx); } - fn stopping(&mut self, _: &mut Self::Context) -> Running { - if self.state.inner.0 == State::Terminated { - return Running::Stop; - } - Running::Continue - } - - fn stopped(&mut self, _: &mut Self::Context) { - if let Some(tx) = self.shutdown_tx.take() { - let _ = tx.send(Ok(())); - } + log::debug!("Executing commands: {:?}", exe_script); + + let msg = activity::Exec { + activity_id: activity_id.unwrap_or_default(), + batch_id: hex::encode(rand::random::<[u8; 16]>()), + exe_script, + timeout: None, + }; + if let Err(e) = exe_unit + .send(RpcEnvelope::with_caller(String::new(), msg)) + .await + { + log::error!("Unable to execute exe script: {:?}", e); } } -#[derive(derivative::Derivative)] -#[derivative(Debug)] -pub struct ExeUnitContext { - pub supervise: Supervision, - pub activity_id: Option, - pub report_url: Option, - pub agreement: Agreement, - pub work_dir: PathBuf, - pub cache_dir: PathBuf, - pub runtime_args: Vec, - pub acl: Acl, - pub credentials: Option, - #[cfg(feature = "sgx")] - #[derivative(Debug = "ignore")] - pub crypto: crypto::Crypto, -} - -impl ExeUnitContext { - pub fn verify_activity_id(&self, activity_id: &str) -> Result<()> { - match &self.activity_id { - Some(act_id) => match act_id == activity_id { - true => Ok(()), - false => Err(Error::RemoteServiceError(format!( - "Forbidden! Invalid activity id: {}", - activity_id - ))), - }, - None => Ok(()), - } +pub async fn run(mut cli: Cli) -> anyhow::Result<()> { + if !cli.binary.exists() { + bail!("Runtime binary does not exist: {}", cli.binary.display()); } -} -impl From<&ExeUnitContext> for TransferServiceContext { - fn from(val: &ExeUnitContext) -> Self { - TransferServiceContext { - task_package: val.agreement.task_package.clone(), - deploy_retry: None, - cache_dir: val.cache_dir.clone(), - work_dir: val.work_dir.clone(), - transfer_retry: None, + let mut commands = None; + let ctx_activity_id; + let ctx_report_url; + + let args = match &cli.command { + Command::FromFile { + args, + service_id, + report_url, + input, + } => { + let contents = std::fs::read_to_string(input).map_err(|e| { + anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) + })?; + let contents = serde_json::from_str(&contents).map_err(|e| { + anyhow::anyhow!( + "Cannot deserialize commands from file {}: {e}", + input.display(), + ) + })?; + ctx_activity_id = service_id.clone(); + ctx_report_url = report_url.clone(); + commands = Some(contents); + args } - } -} - -struct Channel { - tx: mpsc::Sender, - rx: Option>, -} - -impl Default for Channel { - fn default() -> Self { - let (tx, rx) = mpsc::channel(8); - Channel { tx, rx: Some(rx) } - } -} - -pub(crate) async fn report(url: S, msg: M) -> bool -where - M: RpcMessage + Unpin + 'static, - S: AsRef, -{ - let url = url.as_ref(); - match ya_service_bus::typed::service(url).send(msg).await { - Err(ya_service_bus::Error::Timeout(msg)) => { - log::warn!("Timed out reporting to {}: {}", url, msg); - true + Command::ServiceBus { + args, + service_id, + report_url, + } => { + ctx_activity_id = Some(service_id.clone()); + ctx_report_url = Some(report_url.clone()); + args } - Err(e) => { - log::error!("Error reporting to {}: {:?}", url, e); - false + Command::OfferTemplate => { + let args = cli.runtime_arg.clone(); + let offer_template = ExeUnit::::offer_template(cli.binary, args)?; + println!("{}", serde_json::to_string(&offer_template)?); + return Ok(()); } - Ok(Err(e)) => { - log::error!("Error response while reporting to {}: {:?}", url, e); - false + Command::Test => { + let args = cli.runtime_arg.clone(); + let output = ExeUnit::::test(cli.binary, args)?; + println!("{}", String::from_utf8_lossy(&output.stdout)); + eprintln!("{}", String::from_utf8_lossy(&output.stderr)); + if !output.status.success() { + bail!("Test failed"); + } + return Ok(()); } - Ok(Ok(_)) => true, - } -} + }; -async fn report_usage( - report_url: String, - activity_id: String, - exe_unit: Addr>, - metrics: Addr, -) { - match metrics.send(GetMetrics).await { - Ok(resp) => match resp { - Ok(data) => { - let msg = activity::local::SetUsage { - activity_id, - usage: ActivityUsage { - current_usage: Some(data), - timestamp: Utc::now().timestamp(), - }, - timeout: None, - }; - if !report(&report_url, msg).await { - exe_unit.do_send(Shutdown(ShutdownReason::Error(Error::RuntimeError( - format!("Reporting endpoint '{}' is not available", report_url), - )))); - } - } - Err(err) => match err { - Error::UsageLimitExceeded(info) => { - log::warn!("Usage limit exceeded: {}", info); - exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); - } - error => log::warn!("Unable to retrieve metrics: {:?}", error), - }, + if !args.agreement.exists() { + bail!( + "Agreement file does not exist: {}", + args.agreement.display() + ); + } + let work_dir = create_path(&args.work_dir).map_err(|e| { + anyhow::anyhow!( + "Cannot create the working directory {}: {e}", + args.work_dir.display(), + ) + })?; + let cache_dir = create_path(&args.cache_dir).map_err(|e| { + anyhow::anyhow!( + "Cannot create the cache directory {}: {e}", + args.work_dir.display(), + ) + })?; + let mut agreement = Agreement::try_from(&args.agreement).map_err(|e| { + anyhow::anyhow!( + "Error parsing the agreement from {}: {e}", + args.agreement.display(), + ) + })?; + + log::info!("Attempting to read app manifest .."); + + let manifest_ctx = + ManifestContext::try_new(&agreement.inner).context("Invalid app manifest")?; + agreement.task_package = manifest_ctx + .payload() + .or_else(|| agreement.task_package.take()); + + log::info!("Manifest-enabled features: {:?}", manifest_ctx.features()); + log::info!("User-provided payload: {:?}", agreement.task_package); + + let ctx = ExeUnitContext { + supervise: Supervision { + hardware: cli.supervise.hardware, + image: cli.supervise.image, + manifest: manifest_ctx, }, - Err(e) => log::warn!("Unable to report activity usage: {:?}", e), + activity_id: ctx_activity_id.clone(), + report_url: ctx_report_url, + agreement, + work_dir, + cache_dir, + runtime_args: cli.runtime_arg.clone(), + acl: Default::default(), + credentials: None, + #[cfg(feature = "sgx")] + crypto: init_crypto( + cli.sec_key.replace("".into()), + cli.requestor_pub_key.clone(), + )?, + }; + + log::debug!("CLI args: {:?}", cli); + log::debug!("ExeUnitContext args: {:?}", ctx); + + let (tx, rx) = oneshot::channel(); + + let metrics = MetricsService::try_new(&ctx, Some(10000), ctx.supervise.hardware)?.start(); + let transfers = TransferService::new((&ctx).into()).start(); + let runtime = RuntimeProcess::new(&ctx, cli.binary).start(); + let exe_unit = ExeUnit::new(tx, ctx, metrics, transfers, runtime).start(); + let signals = SignalMonitor::new(exe_unit.clone()).start(); + exe_unit.send(Register(signals)).await?; + + if let Some(exe_script) = commands { + tokio::task::spawn(send_script(exe_unit, ctx_activity_id, exe_script)); } -} - -impl Handler for TransferService { - type Result = ResponseFuture>; - fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { - let addr = ctx.address(); - async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() - } + rx.await??; + Ok(()) } From 7e6d099e849de2ea70a9fbc0ea5c23a3ac4a1a1c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 17 Jan 2024 13:55:02 +0100 Subject: [PATCH 018/125] Fix mut cli warning for non sgx build --- exe-unit/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 388d81f06a..edda6509ca 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -212,6 +212,8 @@ async fn send_script( } } +// We need this mut for conditional compilation for sgx +#[allow(unused_mut)] pub async fn run(mut cli: Cli) -> anyhow::Result<()> { if !cli.binary.exists() { bail!("Runtime binary does not exist: {}", cli.binary.display()); From 5be9840f73bb81c139e006d016b739b6871ea552 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 18 Jan 2024 17:57:22 +0100 Subject: [PATCH 019/125] Add Exeunit system test using mock runtime (not passing yet) --- Cargo.lock | 45 +++++- Cargo.toml | 1 + exe-unit/Cargo.toml | 5 + exe-unit/components/mock-runtime/Cargo.toml | 30 ++++ .../resources/mock-runtime-descriptor.json | 9 ++ .../mock-runtime/src/lib.rs} | 57 +------ exe-unit/components/mock-runtime/src/main.rs | 42 ++++++ exe-unit/runtime-api/Cargo.toml | 5 - exe-unit/src/exe_unit.rs | 29 +++- exe-unit/src/lib.rs | 140 +++++++++++------- exe-unit/tests/resources/agreement.json | 80 ++++++++++ exe-unit/tests/test_exe_unit_basic.rs | 90 +++++++++++ .../test-framework/framework-basic/src/lib.rs | 2 +- .../framework-basic/src/temp.rs | 23 --- .../framework-basic/src/test_dirs.rs | 90 +++++++++++ 15 files changed, 504 insertions(+), 144 deletions(-) create mode 100644 exe-unit/components/mock-runtime/Cargo.toml create mode 100644 exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json rename exe-unit/{runtime-api/examples/runtime-server-mock.rs => components/mock-runtime/src/lib.rs} (58%) create mode 100644 exe-unit/components/mock-runtime/src/main.rs create mode 100644 exe-unit/tests/resources/agreement.json create mode 100644 exe-unit/tests/test_exe_unit_basic.rs delete mode 100644 test-utils/test-framework/framework-basic/src/temp.rs create mode 100644 test-utils/test-framework/framework-basic/src/test_dirs.rs diff --git a/Cargo.lock b/Cargo.lock index 1bcd56c82c..70158d2cf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8404,12 +8404,14 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.8.26", + "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", "sha3 0.8.2", "shell-words", "signal-hook", "socket2 0.4.10", "structopt", "tempdir", + "test-context", "thiserror", "tokio", "tokio-stream", @@ -8421,9 +8423,11 @@ dependencies = [ "ya-client-model", "ya-compile-time-utils", "ya-core-model", + "ya-framework-basic", "ya-manifest-utils", + "ya-mock-runtime", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-sb-router", "ya-service-bus", "ya-std-utils", @@ -8729,6 +8733,24 @@ dependencies = [ "ya-service-bus", ] +[[package]] +name = "ya-mock-runtime" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes 1.5.0", + "env_logger 0.10.1", + "futures 0.3.30", + "log", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "url", + "ya-runtime-api 0.7.0", +] + [[package]] name = "ya-net" version = "0.3.0" @@ -9106,6 +9128,25 @@ dependencies = [ "derive_more", ] +[[package]] +name = "ya-runtime-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0db25811f107d62be6c6ac7444d9c6c3e39714b6f76d72798b66ecce47506f" +dependencies = [ + "anyhow", + "bytes 1.5.0", + "futures 0.3.30", + "log", + "prost 0.10.4", + "prost-build 0.10.4", + "serde", + "serde_json", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "ya-runtime-api" version = "0.7.1" @@ -9369,7 +9410,7 @@ dependencies = [ "ya-core-model", "ya-exe-unit", "ya-framework-basic", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-service-bus", "ya-utils-futures", "ya-utils-path", diff --git a/Cargo.toml b/Cargo.toml index 63d9331e2f..99643039c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,6 +206,7 @@ members = [ "exe-unit/runtime-api", "exe-unit/tokio-process-ns", "exe-unit/components/transfer", + "exe-unit/components/mock-runtime", "golem_cli", "utils/actix_utils", "utils/agreement-utils", diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index af1f4357ae..27ceca7cf7 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -109,6 +109,11 @@ actix-files = "0.6" actix-web = "4" env_logger = "0.7" rustyline = "7.0.0" +serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} sha3 = "0.8.2" shell-words = "1.0.0" tempdir = "0.3.7" +test-context = "0.1.4" + +ya-framework-basic = { version = "0.1" } +ya-mock-runtime = { path = "components/mock-runtime"} diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml new file mode 100644 index 0000000000..9d6b190227 --- /dev/null +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "ya-mock-runtime" +version = "0.1.0" +authors = ["Golem Factory "] +edition = "2021" +description = "Mock runtime for testing purposes and set of libraries for testing ExeUnits in tests." + + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "ya-mock-runtime" +path = "src/main.rs" + + +[dependencies] +anyhow = "1.0.31" +bytes = "1.0" +futures = { version = "0.3" } +log = "0.4" +env_logger = "0.10" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +thiserror = "1.0" +tokio = { version = "1", features = ["io-std", "rt", "process", "sync", "macros", "time"] } +tokio-util = { version = "0.7", features = ["codec"] } +url = "2.3" + +ya-runtime-api = "0.7" diff --git a/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json b/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json new file mode 100644 index 0000000000..93f43f2d54 --- /dev/null +++ b/exe-unit/components/mock-runtime/resources/mock-runtime-descriptor.json @@ -0,0 +1,9 @@ +[ + { + "name": "ya-mock-runtime", + "version": "0.1.0", + "supervisor-path": "exe-unit", + "runtime-path": "ya-mock-runtime", + "description": "Mock runtime for testing purposes" + } +] \ No newline at end of file diff --git a/exe-unit/runtime-api/examples/runtime-server-mock.rs b/exe-unit/components/mock-runtime/src/lib.rs similarity index 58% rename from exe-unit/runtime-api/examples/runtime-server-mock.rs rename to exe-unit/components/mock-runtime/src/lib.rs index 496b741a17..c4e3b352c4 100644 --- a/exe-unit/runtime-api/examples/runtime-server-mock.rs +++ b/exe-unit/components/mock-runtime/src/lib.rs @@ -2,19 +2,16 @@ use futures::future::BoxFuture; use futures::prelude::*; use futures::FutureExt; use std::clone::Clone; -use std::env; use std::sync::{Arc, Mutex}; use std::time::Duration; use ya_runtime_api::server::*; -// server - -struct RuntimeMock +pub struct RuntimeMock where H: RuntimeHandler, { - handler: H, + pub handler: H, } impl RuntimeService for RuntimeMock { @@ -61,14 +58,15 @@ impl RuntimeService for RuntimeMock { // client // holds last received status -struct EventMock(Arc>); +#[derive(Clone)] +pub struct EventMock(Arc>); impl EventMock { - fn new() -> Self { + pub fn new() -> Self { Self(Arc::new(Mutex::new(Default::default()))) } - fn get_last_status(&self) -> ProcessStatus { + pub fn get_last_status(&self) -> ProcessStatus { self.0.lock().unwrap().clone() } } @@ -84,46 +82,3 @@ impl RuntimeHandler for EventMock { future::ready(()).boxed() } } - -impl Clone for EventMock { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - if env::var("RUST_LOG").is_err() { - env::set_var("RUST_LOG", "debug") - } - env_logger::init(); - if env::var("X_SERVER").is_ok() { - run(|event_emitter| RuntimeMock { - handler: event_emitter, - }) - .await - } else { - use tokio::process::Command; - let exe = env::current_exe().unwrap(); - - let mut cmd = Command::new(exe); - cmd.env("X_SERVER", "1"); - let events = EventMock::new(); - let c = spawn(cmd, events.clone()).await?; - log::debug!("hello_result={:?}", c.hello("0.0.0x").await); - let run = RunProcess { - bin: "sleep".to_owned(), - args: vec!["10".to_owned()], - ..Default::default() - }; - let sleep_1 = c.run_process(run.clone()); - let sleep_2 = c.run_process(run.clone()); - let sleep_3 = c.run_process(run); - log::info!("start sleep1"); - log::info!("sleep1={:?}", sleep_1.await); - log::info!("start sleep2 sleep3"); - log::info!("sleep23={:?}", future::join(sleep_2, sleep_3).await); - log::info!("last status: {:?}", events.get_last_status()); - } - Ok(()) -} diff --git a/exe-unit/components/mock-runtime/src/main.rs b/exe-unit/components/mock-runtime/src/main.rs new file mode 100644 index 0000000000..509263c203 --- /dev/null +++ b/exe-unit/components/mock-runtime/src/main.rs @@ -0,0 +1,42 @@ +use futures::future; +use std::env; + +use ya_mock_runtime::{EventMock, RuntimeMock}; +use ya_runtime_api::server::{run, spawn, RunProcess, RuntimeService}; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + if env::var("RUST_LOG").is_err() { + env::set_var("RUST_LOG", "debug") + } + env_logger::init(); + if env::var("X_SERVER").is_ok() { + run(|event_emitter| RuntimeMock { + handler: event_emitter, + }) + .await + } else { + use tokio::process::Command; + let exe = env::current_exe().unwrap(); + + let mut cmd = Command::new(exe); + cmd.env("X_SERVER", "1"); + let events = EventMock::new(); + let c = spawn(cmd, events.clone()).await?; + log::debug!("hello_result={:?}", c.hello("0.0.0x").await); + let run = RunProcess { + bin: "sleep".to_owned(), + args: vec!["10".to_owned()], + ..Default::default() + }; + let sleep_1 = c.run_process(run.clone()); + let sleep_2 = c.run_process(run.clone()); + let sleep_3 = c.run_process(run); + log::info!("start sleep1"); + log::info!("sleep1={:?}", sleep_1.await); + log::info!("start sleep2 sleep3"); + log::info!("sleep23={:?}", future::join(sleep_2, sleep_3).await); + log::info!("last status: {:?}", events.get_last_status()); + } + Ok(()) +} diff --git a/exe-unit/runtime-api/Cargo.toml b/exe-unit/runtime-api/Cargo.toml index 9de7a43bfc..316a74f2fb 100644 --- a/exe-unit/runtime-api/Cargo.toml +++ b/exe-unit/runtime-api/Cargo.toml @@ -8,12 +8,7 @@ license = "GPL-3.0" homepage = "https://github.com/golemfactory/yagna/tree/master/exe-unit/runtime-api" repository = "https://github.com/golemfactory/yagna" -[[example]] -name = "runtime-server-mock" -required-features = ["server"] - [features] -default = ['server'] codec = [] server = ['prost', 'futures', 'tokio', 'tokio-util'] diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index b666fdd259..edaa86a8e7 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -1,13 +1,14 @@ use actix::dev::IntervalFunc; use actix::{ Actor, ActorFutureExt, ActorStreamExt, Addr, AsyncContext, Context, ContextFutureSpawner, - Handler, ResponseFuture, Running, StreamHandler, WrapFuture, + Handler, Message, ResponseFuture, Running, StreamHandler, WrapFuture, }; use chrono::Utc; use futures::channel::{mpsc, oneshot}; use futures::{FutureExt, SinkExt}; use std::path::PathBuf; use std::time::Duration; +use tokio::sync::broadcast; use ya_agreement_utils::OfferTemplate; use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; @@ -36,6 +37,10 @@ lazy_static::lazy_static! { static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); } +#[derive(Clone, Debug, Default, Message)] +#[rtype(result = "()")] +pub struct AwaitFinish {} + pub struct ExeUnit { pub(crate) ctx: ExeUnitContext, pub(crate) state: ExeUnitState, @@ -44,17 +49,17 @@ pub struct ExeUnit { pub(crate) metrics: Addr, pub(crate) transfers: Addr, pub(crate) services: Vec>, - pub(crate) shutdown_tx: Option>>, + pub(crate) shutdown_tx: broadcast::Sender<()>, } impl ExeUnit { pub fn new( - shutdown_tx: oneshot::Sender>, ctx: ExeUnitContext, metrics: Addr, transfers: Addr, runtime: Addr, ) -> Self { + let (shutdown_tx, _) = broadcast::channel(1); ExeUnit { ctx, state: ExeUnitState::default(), @@ -67,7 +72,7 @@ impl ExeUnit { Box::new(ServiceAddr::new(transfers)), Box::new(ServiceAddr::new(runtime)), ], - shutdown_tx: Some(shutdown_tx), + shutdown_tx, } } @@ -449,9 +454,7 @@ impl Actor for ExeUnit { } fn stopped(&mut self, _: &mut Self::Context) { - if let Some(tx) = self.shutdown_tx.take() { - let _ = tx.send(Ok(())); - } + self.shutdown_tx.send(()).ok(); } } @@ -569,6 +572,18 @@ async fn report_usage( } } +impl Handler for ExeUnit { + type Result = ResponseFuture<()>; + + fn handle(&mut self, _msg: AwaitFinish, _: &mut Self::Context) -> Self::Result { + let mut rx = self.shutdown_tx.subscribe(); + async move { + rx.recv().await.ok(); + } + .boxed_local() + } +} + impl Handler for TransferService { type Result = ResponseFuture>; diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index edda6509ca..f98231aaac 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -3,7 +3,6 @@ extern crate derive_more; use actix::prelude::*; use anyhow::{bail, Context}; -use futures::channel::oneshot; use std::convert::TryFrom; use std::path::PathBuf; use structopt::clap; @@ -44,7 +43,7 @@ pub mod state; mod dns; mod exe_unit; -pub use exe_unit::{report, ExeUnit, ExeUnitContext, RuntimeRef}; +pub use exe_unit::{report, AwaitFinish, ExeUnit, ExeUnitContext, RuntimeRef}; pub type Result = std::result::Result; @@ -54,56 +53,56 @@ pub type Result = std::result::Result; pub struct Cli { /// Runtime binary path #[structopt(long, short)] - binary: PathBuf, + pub binary: PathBuf, #[structopt(flatten)] - supervise: SuperviseCli, + pub supervise: SuperviseCli, /// Additional runtime arguments #[structopt( - long, - short, - set = clap::ArgSettings::Global, - number_of_values = 1, + long, + short, + set = clap::ArgSettings::Global, + number_of_values = 1, )] - runtime_arg: Vec, + pub runtime_arg: Vec, /// Enclave secret key used in secure communication #[structopt( - long, - env = "EXE_UNIT_SEC_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, + long, + env = "EXE_UNIT_SEC_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, )] #[allow(dead_code)] - sec_key: Option, + pub sec_key: Option, /// Requestor public key used in secure communication #[structopt( - long, - env = "EXE_UNIT_REQUESTOR_PUB_KEY", - hide_env_values = true, - set = clap::ArgSettings::Global, + long, + env = "EXE_UNIT_REQUESTOR_PUB_KEY", + hide_env_values = true, + set = clap::ArgSettings::Global, )] #[allow(dead_code)] - requestor_pub_key: Option, + pub requestor_pub_key: Option, #[structopt(subcommand)] - command: Command, + pub command: Command, } -#[derive(structopt::StructOpt, Debug)] +#[derive(structopt::StructOpt, Debug, Clone)] pub struct SuperviseCli { /// Hardware resources are handled by the runtime #[structopt( - long = "runtime-managed-hardware", - alias = "cap-handoff", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, + long = "runtime-managed-hardware", + alias = "cap-handoff", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, )] - hardware: bool, + pub hardware: bool, /// Images are handled by the runtime #[structopt( - long = "runtime-managed-image", - parse(from_flag = std::ops::Not::not), - set = clap::ArgSettings::Global, + long = "runtime-managed-image", + parse(from_flag = std::ops::Not::not), + set = clap::ArgSettings::Global, )] - image: bool, + pub image: bool, } #[derive(structopt::StructOpt, Debug)] @@ -137,17 +136,17 @@ pub enum Command { Test, } -#[derive(structopt::StructOpt, Debug)] +#[derive(structopt::StructOpt, Debug, Clone)] pub struct RunArgs { /// Agreement file path #[structopt(long, short)] - agreement: PathBuf, + pub agreement: PathBuf, /// Working directory #[structopt(long, short)] - work_dir: PathBuf, + pub work_dir: PathBuf, /// Common cache directory #[structopt(long, short)] - cache_dir: PathBuf, + pub cache_dir: PathBuf, } fn create_path(path: &PathBuf) -> anyhow::Result { @@ -175,7 +174,7 @@ fn init_crypto( } } -async fn send_script( +pub async fn send_script( exe_unit: Addr>, activity_id: Option, exe_script: Vec, @@ -215,6 +214,8 @@ async fn send_script( // We need this mut for conditional compilation for sgx #[allow(unused_mut)] pub async fn run(mut cli: Cli) -> anyhow::Result<()> { + log::debug!("CLI args: {:?}", cli); + if !cli.binary.exists() { bail!("Runtime binary does not exist: {}", cli.binary.display()); } @@ -223,14 +224,14 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { let ctx_activity_id; let ctx_report_url; - let args = match &cli.command { + let args = match cli.command { Command::FromFile { args, service_id, report_url, input, } => { - let contents = std::fs::read_to_string(input).map_err(|e| { + let contents = std::fs::read_to_string(&input).map_err(|e| { anyhow::anyhow!("Cannot read commands from file {}: {e}", input.display()) })?; let contents = serde_json::from_str(&contents).map_err(|e| { @@ -271,6 +272,43 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { } }; + let exe_unit = exe_unit(ExeUnitConfig { + report_url: ctx_report_url, + service_id: ctx_activity_id.clone(), + runtime_args: cli.runtime_arg, + binary: cli.binary, + supervise: cli.supervise, + sec_key: cli.sec_key, + args, + requestor_pub_key: cli.requestor_pub_key, + }) + .await?; + + if let Some(exe_script) = commands { + tokio::task::spawn(send_script(exe_unit.clone(), ctx_activity_id, exe_script)); + } + + exe_unit.send(AwaitFinish {}).await?; + Ok(()) +} + +#[derive(Debug, Clone)] +pub struct ExeUnitConfig { + pub args: RunArgs, + pub binary: PathBuf, + pub runtime_args: Vec, + pub service_id: Option, + pub report_url: Option, + pub supervise: SuperviseCli, + + #[allow(dead_code)] + pub sec_key: Option, + #[allow(dead_code)] + pub requestor_pub_key: Option, +} + +pub async fn exe_unit(config: ExeUnitConfig) -> anyhow::Result>> { + let args = config.args; if !args.agreement.exists() { bail!( "Agreement file does not exist: {}", @@ -309,41 +347,33 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { let ctx = ExeUnitContext { supervise: Supervision { - hardware: cli.supervise.hardware, - image: cli.supervise.image, + hardware: config.supervise.hardware, + image: config.supervise.image, manifest: manifest_ctx, }, - activity_id: ctx_activity_id.clone(), - report_url: ctx_report_url, + activity_id: config.service_id.clone(), + report_url: config.report_url, agreement, work_dir, cache_dir, - runtime_args: cli.runtime_arg.clone(), + runtime_args: config.runtime_args, acl: Default::default(), credentials: None, #[cfg(feature = "sgx")] crypto: init_crypto( - cli.sec_key.replace("".into()), - cli.requestor_pub_key.clone(), + config.sec_key.replace("".into()), + config.requestor_pub_key.clone(), )?, }; - log::debug!("CLI args: {:?}", cli); log::debug!("ExeUnitContext args: {:?}", ctx); - let (tx, rx) = oneshot::channel(); - let metrics = MetricsService::try_new(&ctx, Some(10000), ctx.supervise.hardware)?.start(); let transfers = TransferService::new((&ctx).into()).start(); - let runtime = RuntimeProcess::new(&ctx, cli.binary).start(); - let exe_unit = ExeUnit::new(tx, ctx, metrics, transfers, runtime).start(); + let runtime = RuntimeProcess::new(&ctx, config.binary).start(); + let exe_unit = ExeUnit::new(ctx, metrics, transfers, runtime).start(); let signals = SignalMonitor::new(exe_unit.clone()).start(); exe_unit.send(Register(signals)).await?; - if let Some(exe_script) = commands { - tokio::task::spawn(send_script(exe_unit, ctx_activity_id, exe_script)); - } - - rx.await??; - Ok(()) + Ok(exe_unit) } diff --git a/exe-unit/tests/resources/agreement.json b/exe-unit/tests/resources/agreement.json new file mode 100644 index 0000000000..9dbbccfea4 --- /dev/null +++ b/exe-unit/tests/resources/agreement.json @@ -0,0 +1,80 @@ +{ + "agreementId": "0ec929f5acc8f98a47ab72d61a2c2f343d45d8438d3aa4ccdc84e717c219e185", + "proposedSignature": "NoSignature", + "state": "Pending", + "timestamp": "2022-05-22T10:41:42.564784259Z", + "validTo": "2022-05-22T11:41:42.562457Z", + + "offer": { + "properties": { + "golem.activity.caps.transfer.protocol": [ + "gftp", + "https", + "http" + ], + "golem.activity.caps.transfer.report-progress": true, + "golem.com.payment.debit-notes.accept-timeout?": 240, + "golem.com.payment.platform.erc20-goerli-tglm.address": "0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a", + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": [ + 0.001388888888888889, + 0.0002777777777777778, + 0.0 + ], + "golem.com.scheme": "payu", + "golem.com.scheme.payu.debit-note.interval-sec?": 120, + "golem.com.scheme.payu.payment-timeout-sec?": 120, + "golem.com.usage.vector": [ + "golem.usage.cpu_sec", + "golem.usage.duration_sec" + ], + "golem.inf.cpu.architecture": "x86_64", + "golem.inf.cpu.brand": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz", + "golem.inf.cpu.capabilities": [], + "golem.inf.cpu.cores": 6, + "golem.inf.cpu.model": "Stepping 10 Family 6 Model 302", + "golem.inf.cpu.threads": 11, + "golem.inf.cpu.vendor": "GenuineIntel", + "golem.inf.mem.gib": 28.0, + "golem.inf.storage.gib": 10.188121795654297, + "golem.node.id.name": "mock-provider", + "golem.node.net.is-public": true, + "golem.runtime.capabilities": [], + "golem.runtime.name": "ya-mock-runtime", + "golem.runtime.version": "0.1.0", + "golem.srv.caps.multi-activity": true, + "golem.srv.caps.payload-manifest": true + }, + "constraints": "(&\n (golem.srv.comp.expiration>1705586871777)\n)", + "offerId": "afce49b1ea5b45db91bdd6e5481479f9-9095fca9dea0a91ce95cf994125b33cdd838fcc963a1106f2be9e4b5b65a52f0", + "providerId": "0x86a269498fb5270f20bdc6fdcf6039122b0d3b23", + "timestamp": "2022-05-22T10:41:42.564784259Z" + }, + + "demand": { + "constraints": "(&(golem.com.payment.platform.erc20-goerli-tglm.address=*)\n\t(golem.com.pricing.model=linear)\n\t(&(golem.inf.mem.gib>=0.5)\n\t(golem.inf.storage.gib>=2.0)\n\t(golem.inf.cpu.threads>=1)\n\t(golem.runtime.name=ya-mock-runtime)))", + "demandId": "773035fc685c46da8e61473ac2a2568e-3f3eb86d6ef9a01708d0f57d0b19cc69fd74422150c120e33cc1b5f4a1a12b96", + "properties": { + "golem": { + "com.payment": { + "chosen-platform": "erc20-goerli-tglm", + "debit-notes.accept-timeout?": 240, + "platform.erc20-goerli-tglm.address": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7" + }, + "node": {}, + "srv": { + "caps.multi-activity": true, + "comp": { + "expiration": 1653216996555, + "task_package": "http://127.0.0.1:8001/rnd", + "vm": { + "package_format": "gvmkit-squash" + } + } + } + } + }, + "requestorId": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7", + "timestamp": "2022-05-22T10:41:42.564784259Z" + } +} diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs new file mode 100644 index 0000000000..bd2f4d57fd --- /dev/null +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -0,0 +1,90 @@ +use actix::Addr; +use test_context::test_context; +use ya_client_model::activity::ExeScriptCommand; +use ya_exe_unit::message::{Shutdown, ShutdownReason}; +use ya_exe_unit::runtime::process::RuntimeProcess; +use ya_exe_unit::{ + exe_unit, send_script, AwaitFinish, ExeUnit, ExeUnitConfig, RunArgs, SuperviseCli, +}; +use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; +use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::cargo_binary; +use ya_framework_basic::{resource, temp_dir}; + +#[derive(Debug, Clone)] +pub struct ExeUnitHandle(pub Addr>); + +#[async_trait::async_trait] +impl AsyncDroppable for ExeUnitHandle { + async fn async_drop(&self) { + self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); + self.0.send(AwaitFinish {}).await.ok(); + } +} + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("exe-unit-start-terminate")?; + let temp_dir = dir.path(); + + generate_file_with_hash(temp_dir, "rnd", 4096_usize, 10); + start_http(ctx, temp_dir.to_path_buf()) + .await + .expect("unable to start http servers"); + + let config = ExeUnitConfig { + args: RunArgs { + agreement: resource!("agreement.json"), + cache_dir: temp_dir.join("cache"), + work_dir: temp_dir.join("work"), + }, + binary: cargo_binary("ya-mock-runtime").unwrap(), + runtime_args: vec![], + supervise: SuperviseCli { + hardware: false, + image: false, + }, + sec_key: None, + requestor_pub_key: None, + service_id: None, + report_url: None, + }; + + let exe = exe_unit(config).await.unwrap(); + ctx.register(ExeUnitHandle(exe.clone())); + + log::info!("Sending [deploy, start] batch for execution."); + + send_script( + exe.clone(), + None, + vec![ + ExeScriptCommand::Deploy { + net: vec![], + progress: None, + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec![], + }, + ExeScriptCommand::Start { args: vec![] }, + ], + ) + .await; + + log::info!("Sending shutdown request."); + + exe.send(Shutdown(ShutdownReason::Finished)) + .await + .unwrap() + .unwrap(); + exe.send(AwaitFinish {}).await?; + + Ok(()) +} diff --git a/test-utils/test-framework/framework-basic/src/lib.rs b/test-utils/test-framework/framework-basic/src/lib.rs index 40350bad24..b3bb3916d9 100644 --- a/test-utils/test-framework/framework-basic/src/lib.rs +++ b/test-utils/test-framework/framework-basic/src/lib.rs @@ -3,4 +3,4 @@ pub mod file; pub mod hash; pub mod log; pub mod server_external; -pub mod temp; +pub mod test_dirs; diff --git a/test-utils/test-framework/framework-basic/src/temp.rs b/test-utils/test-framework/framework-basic/src/temp.rs deleted file mode 100644 index aea081f7ef..0000000000 --- a/test-utils/test-framework/framework-basic/src/temp.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::fs; -use tempdir::TempDir; - -pub mod macros { - #[macro_export] - macro_rules! temp_dir { - ($prefix:literal) => { - // CARGO_TARGET_TMPDIR is available in compile time only in binary modules, so we can't - // use it in this library. Thanks to using macro it will be resolved in final code not here - // and it will work. - ya_framework_basic::temp::temp_dir_(env!("CARGO_TARGET_TMPDIR"), $prefix) - }; - } -} - -pub fn temp_dir_(base_dir: &str, prefix: &str) -> anyhow::Result { - fs::create_dir_all(base_dir)?; - let dir = TempDir::new_in(base_dir, prefix)?; - let temp_dir = dir.path(); - fs::create_dir_all(temp_dir)?; - - Ok(dir) -} diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs new file mode 100644 index 0000000000..bf0e1c892b --- /dev/null +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -0,0 +1,90 @@ +use anyhow::{anyhow, bail}; +use std::fs; +use std::path::PathBuf; +use tempdir::TempDir; + +pub mod macros { + /// Creates temporary directory in cargo target directory. + #[macro_export] + macro_rules! temp_dir { + ($prefix:literal) => { + // CARGO_TARGET_TMPDIR is available in compile time only in binary modules, so we can't + // use it in this library. Thanks to using macro it will be resolved in final code not here + // and it will work. + ya_framework_basic::test_dirs::temp_dir_(env!("CARGO_TARGET_TMPDIR"), $prefix) + }; + } + + /// Returns resource from `resources` directory in tests folder. + #[macro_export] + macro_rules! resource { + ($name:literal) => { + // CARGO_MANIFEST_DIR is available in compile time only in binary modules, so we can't + // use it in this library. Thanks to using macro it will be resolved in final code not here + // and it will work. + ya_framework_basic::test_dirs::resource_(env!("CARGO_MANIFEST_DIR"), $name) + }; + } +} + +pub fn temp_dir_(base_dir: &str, prefix: &str) -> anyhow::Result { + fs::create_dir_all(base_dir)?; + let dir = TempDir::new_in(base_dir, prefix)?; + let temp_dir = dir.path(); + fs::create_dir_all(temp_dir)?; + + Ok(dir) +} + +#[cfg(debug_assertions)] +pub fn is_debug() -> bool { + true +} + +#[cfg(not(debug_assertions))] +pub fn is_debug() -> bool { + false +} + +#[cfg(target_family = "windows")] +pub fn extension() -> String { + ".exe".to_string() +} + +#[cfg(not(target_family = "windows"))] +pub fn extension() -> String { + "".to_string() +} + +/// Returns absolute path to cargo project binary. +pub fn cargo_binary(bin_name: &str) -> anyhow::Result { + let current = std::env::current_exe() + .map_err(|e| anyhow!("Failed to get path to current binary. {e}"))? + .parent() + .and_then(|path| path.parent()) + .ok_or(anyhow!("No parent dir for current binary."))? + .to_path_buf(); + let bin_name = format!("{bin_name}{}", extension()); + let bin_path = current.join(&bin_name); + if !bin_path.exists() { + bail!( + "Path doesn't exist: {}, when looking for binary: {}", + bin_path.display(), + bin_name + ); + } + + if !bin_path.is_file() { + bail!("Expected {} to be binary file.", bin_path.display()); + } + + Ok(bin_path) +} + +/// Returns resource from `resources` directory in tests. +pub fn resource_(base_dir: &str, name: &str) -> PathBuf { + PathBuf::from(base_dir) + .join("tests") + .join("resources") + .join(name) +} From fbba58008981af4beb5316c9c2eebb06b72af357 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 19 Jan 2024 13:28:21 +0100 Subject: [PATCH 020/125] Better mechanism to wait for ExeUnit finish --- exe-unit/src/exe_unit.rs | 17 +++++++---------- exe-unit/src/lib.rs | 4 ++-- exe-unit/tests/test_exe_unit_basic.rs | 11 ++++++++--- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index edaa86a8e7..aa4a426038 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -32,14 +32,15 @@ use crate::runtime::{Runtime, RuntimeMode}; use crate::service::metrics::MetricsService; use crate::service::{ServiceAddr, ServiceControl}; use crate::state::{ExeUnitState, StateError, Supervision}; +use crate::Result; lazy_static::lazy_static! { static ref DEFAULT_REPORT_INTERVAL: Duration = Duration::from_secs(1u64); } #[derive(Clone, Debug, Default, Message)] -#[rtype(result = "()")] -pub struct AwaitFinish {} +#[rtype(result = "Result>")] +pub struct FinishNotifier {} pub struct ExeUnit { pub(crate) ctx: ExeUnitContext, @@ -572,15 +573,11 @@ async fn report_usage( } } -impl Handler for ExeUnit { - type Result = ResponseFuture<()>; +impl Handler for ExeUnit { + type Result = Result>; - fn handle(&mut self, _msg: AwaitFinish, _: &mut Self::Context) -> Self::Result { - let mut rx = self.shutdown_tx.subscribe(); - async move { - rx.recv().await.ok(); - } - .boxed_local() + fn handle(&mut self, _msg: FinishNotifier, _: &mut Self::Context) -> Self::Result { + Ok(self.shutdown_tx.subscribe()) } } diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index f98231aaac..8b310d7d21 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -43,7 +43,7 @@ pub mod state; mod dns; mod exe_unit; -pub use exe_unit::{report, AwaitFinish, ExeUnit, ExeUnitContext, RuntimeRef}; +pub use exe_unit::{report, ExeUnit, ExeUnitContext, FinishNotifier, RuntimeRef}; pub type Result = std::result::Result; @@ -288,7 +288,7 @@ pub async fn run(mut cli: Cli) -> anyhow::Result<()> { tokio::task::spawn(send_script(exe_unit.clone(), ctx_activity_id, exe_script)); } - exe_unit.send(AwaitFinish {}).await?; + exe_unit.send(FinishNotifier {}).await??.recv().await?; Ok(()) } diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index bd2f4d57fd..93cec3cccb 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -4,7 +4,7 @@ use ya_client_model::activity::ExeScriptCommand; use ya_exe_unit::message::{Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; use ya_exe_unit::{ - exe_unit, send_script, AwaitFinish, ExeUnit, ExeUnitConfig, RunArgs, SuperviseCli, + exe_unit, send_script, ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli, }; use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; use ya_framework_basic::file::generate_file_with_hash; @@ -19,8 +19,11 @@ pub struct ExeUnitHandle(pub Addr>); #[async_trait::async_trait] impl AsyncDroppable for ExeUnitHandle { async fn async_drop(&self) { + let finish = self.0.send(FinishNotifier {}).await; self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); - self.0.send(AwaitFinish {}).await.ok(); + if let Ok(Ok(mut finish)) = finish { + finish.recv().await.ok(); + } } } @@ -57,6 +60,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow }; let exe = exe_unit(config).await.unwrap(); + let mut finish = exe.send(FinishNotifier {}).await??; ctx.register(ExeUnitHandle(exe.clone())); log::info!("Sending [deploy, start] batch for execution."); @@ -84,7 +88,8 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow .await .unwrap() .unwrap(); - exe.send(AwaitFinish {}).await?; + + finish.recv().await.unwrap(); Ok(()) } From f82c3edb00bf3e0f6555ea0c9ac6b3940b51a062 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 19 Jan 2024 17:31:53 +0100 Subject: [PATCH 021/125] Move ExeUnit test utils to mock-runtime --- Cargo.lock | 4 + exe-unit/components/mock-runtime/Cargo.toml | 7 ++ exe-unit/components/mock-runtime/src/lib.rs | 2 + .../components/mock-runtime/src/testing.rs | 3 + .../mock-runtime/src/testing/exe_unit_ext.rs | 75 +++++++++++++++++++ .../components/transfer/tests/test_deploy.rs | 6 +- .../transfer/tests/test_transfer_resume.rs | 4 +- .../transfer/tests/test_transfer_service.rs | 6 +- exe-unit/src/lib.rs | 9 ++- exe-unit/tests/resources/agreement.json | 2 +- exe-unit/tests/test_exe_unit_basic.rs | 59 ++++----------- .../framework-basic/src/file.rs | 57 ++++++++++++-- 12 files changed, 172 insertions(+), 62 deletions(-) create mode 100644 exe-unit/components/mock-runtime/src/testing.rs create mode 100644 exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs diff --git a/Cargo.lock b/Cargo.lock index 70158d2cf1..516887adfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8737,7 +8737,9 @@ dependencies = [ name = "ya-mock-runtime" version = "0.1.0" dependencies = [ + "actix", "anyhow", + "async-trait", "bytes 1.5.0", "env_logger 0.10.1", "futures 0.3.30", @@ -8748,6 +8750,8 @@ dependencies = [ "tokio", "tokio-util", "url", + "ya-exe-unit", + "ya-framework-basic", "ya-runtime-api 0.7.0", ] diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 9d6b190227..c73f95478f 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -28,3 +28,10 @@ tokio-util = { version = "0.7", features = ["codec"] } url = "2.3" ya-runtime-api = "0.7" + +# Dependancies for ExeUnit testing utils +ya-exe-unit = "0.4" +ya-framework-basic = { version = "0.1" } + +async-trait = "0.1.77" +actix = { version = "0.13", default-features = false } diff --git a/exe-unit/components/mock-runtime/src/lib.rs b/exe-unit/components/mock-runtime/src/lib.rs index c4e3b352c4..c818b73da4 100644 --- a/exe-unit/components/mock-runtime/src/lib.rs +++ b/exe-unit/components/mock-runtime/src/lib.rs @@ -1,3 +1,5 @@ +pub mod testing; + use futures::future::BoxFuture; use futures::prelude::*; use futures::FutureExt; diff --git a/exe-unit/components/mock-runtime/src/testing.rs b/exe-unit/components/mock-runtime/src/testing.rs new file mode 100644 index 0000000000..cb349f200a --- /dev/null +++ b/exe-unit/components/mock-runtime/src/testing.rs @@ -0,0 +1,3 @@ +mod exe_unit_ext; + +pub use exe_unit_ext::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs new file mode 100644 index 0000000000..6dee0f450c --- /dev/null +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -0,0 +1,75 @@ +use actix::Addr; +use anyhow::bail; +use std::path::Path; +use std::time::Duration; + +use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; +use ya_exe_unit::runtime::process::RuntimeProcess; +use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; +use ya_framework_basic::async_drop::AsyncDroppable; +use ya_framework_basic::test_dirs::cargo_binary; + +#[async_trait::async_trait] +pub trait ExeUnitExt { + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()>; +} + +#[derive(Debug, Clone)] +pub struct ExeUnitHandle(pub Addr>); + +#[async_trait::async_trait] +impl AsyncDroppable for ExeUnitHandle { + async fn async_drop(&self) { + let finish = self.0.send(FinishNotifier {}).await; + self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); + if let Ok(Ok(mut finish)) = finish { + finish.recv().await.ok(); + } + } +} + +pub fn exe_unit_config(temp_dir: &Path, agreement_path: &Path, binary: &str) -> ExeUnitConfig { + ExeUnitConfig { + args: RunArgs { + agreement: agreement_path.to_path_buf(), + cache_dir: temp_dir.join("cache"), + work_dir: temp_dir.join("work"), + }, + binary: cargo_binary(binary).unwrap(), + runtime_args: vec![], + supervise: SuperviseCli { + hardware: false, + image: false, + }, + sec_key: None, + requestor_pub_key: None, + service_id: None, + report_url: None, + } +} + +#[async_trait::async_trait] +impl ExeUnitExt for Addr> { + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()> { + let delay = Duration::from_secs_f32(0.5); + loop { + match self + .send(GetBatchResults { + batch_id: batch_id.to_string(), + idx: None, + }) + .await + { + Ok(results) => { + if let Some(last) = results.0.last() { + if last.is_batch_finished { + return Ok(()); + } + } + } + Err(e) => bail!("Waiting for batch: {batch_id}. Error: {e}"), + } + tokio::time::sleep(delay).await; + } + } +} diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index ae32972974..2f84fb4d24 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -10,7 +10,7 @@ use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::CommandProgress; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::temp_dir; @@ -36,7 +36,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re std::fs::create_dir_all(dir)?; } - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 1024 * 10); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 1024 * 10); log::debug!("Starting HTTP servers"); let path = temp_dir.to_path_buf(); @@ -99,7 +99,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< let chunk_size = 4096_usize; let chunk_count = 1024 * 1; let file_size = (chunk_size * chunk_count) as u64; - let hash = generate_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); + let hash = generate_random_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); log::debug!("Starting HTTP servers"); let path = temp_dir.to_path_buf(); diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 7253453199..03920246b6 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -9,7 +9,7 @@ use url::Url; use ya_client_model::activity::TransferArgs; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::hash::verify_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; @@ -162,7 +162,7 @@ async fn test_transfer_resume(ctx: &mut DroppableTestContext) -> anyhow::Result< }]; addr.send(AddVolumes::new(volumes)).await??; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 3 * 1024); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 3 * 1024); log::debug!("Starting HTTP servers"); start_http(ctx, temp_dir.to_path_buf()) diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index 80efccc800..500b21ecf8 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -5,7 +5,7 @@ use test_context::test_context; use ya_client_model::activity::TransferArgs; use ya_exe_unit::error::Error; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::file::generate_file_with_hash; +use ya_framework_basic::file::generate_random_file_with_hash; use ya_framework_basic::hash::verify_hash; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; @@ -70,7 +70,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu }, // Uncomment to enable logs ]; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); log::debug!("Starting HTTP servers"); @@ -182,7 +182,7 @@ async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Resul path: "/extract".into(), }, ]; - let hash = generate_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); + let hash = generate_random_file_with_hash(temp_dir, "rnd", 4096_usize, 256_usize); log::debug!("Starting HTTP servers"); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 8b310d7d21..38608f5561 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -178,7 +178,7 @@ pub async fn send_script( exe_unit: Addr>, activity_id: Option, exe_script: Vec, -) { +) -> anyhow::Result { use crate::state::{State, StatePair}; use std::time::Duration; @@ -189,7 +189,8 @@ pub async fn send_script( Ok(GetStateResponse(StatePair(State::Terminated, _))) | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) | Err(_) => { - return log::error!("ExeUnit has terminated"); + log::error!("ExeUnit has terminated"); + bail!("ExeUnit has terminated"); } _ => tokio::time::sleep(delay).await, } @@ -197,9 +198,10 @@ pub async fn send_script( log::debug!("Executing commands: {:?}", exe_script); + let batch_id = hex::encode(rand::random::<[u8; 16]>()); let msg = activity::Exec { activity_id: activity_id.unwrap_or_default(), - batch_id: hex::encode(rand::random::<[u8; 16]>()), + batch_id: batch_id.clone(), exe_script, timeout: None, }; @@ -209,6 +211,7 @@ pub async fn send_script( { log::error!("Unable to execute exe script: {:?}", e); } + Ok(batch_id) } // We need this mut for conditional compilation for sgx diff --git a/exe-unit/tests/resources/agreement.json b/exe-unit/tests/resources/agreement.json index 9dbbccfea4..427dbee17f 100644 --- a/exe-unit/tests/resources/agreement.json +++ b/exe-unit/tests/resources/agreement.json @@ -66,7 +66,7 @@ "caps.multi-activity": true, "comp": { "expiration": 1653216996555, - "task_package": "http://127.0.0.1:8001/rnd", + "task_package": "hash://sha3:22e08b990b6c6685a7e80ecd9a1adb52561a7d9fe9e69b915269da229be6c1ad69dea4ff8a77dc2c4973558da9150909a2be4121b1cbe1ddb04630c1f75aad4f:http://127.0.0.1:8001/image-1", "vm": { "package_format": "gvmkit-squash" } diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 93cec3cccb..4fc29b002c 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -1,63 +1,31 @@ -use actix::Addr; use test_context::test_context; + use ya_client_model::activity::ExeScriptCommand; use ya_exe_unit::message::{Shutdown, ShutdownReason}; -use ya_exe_unit::runtime::process::RuntimeProcess; -use ya_exe_unit::{ - exe_unit, send_script, ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli, -}; -use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; -use ya_framework_basic::file::generate_file_with_hash; +use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; -use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; - -#[derive(Debug, Clone)] -pub struct ExeUnitHandle(pub Addr>); - -#[async_trait::async_trait] -impl AsyncDroppable for ExeUnitHandle { - async fn async_drop(&self) { - let finish = self.0.send(FinishNotifier {}).await; - self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); - if let Ok(Ok(mut finish)) = finish { - finish.recv().await.ok(); - } - } -} +use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("exe-unit-start-terminate")?; let temp_dir = dir.path(); + let image_repo = temp_dir.join("images"); - generate_file_with_hash(temp_dir, "rnd", 4096_usize, 10); - start_http(ctx, temp_dir.to_path_buf()) + generate_image(&image_repo, "image-1", 4096_usize, 10); + start_http(ctx, image_repo) .await .expect("unable to start http servers"); - let config = ExeUnitConfig { - args: RunArgs { - agreement: resource!("agreement.json"), - cache_dir: temp_dir.join("cache"), - work_dir: temp_dir.join("work"), - }, - binary: cargo_binary("ya-mock-runtime").unwrap(), - runtime_args: vec![], - supervise: SuperviseCli { - hardware: false, - image: false, - }, - sec_key: None, - requestor_pub_key: None, - service_id: None, - report_url: None, - }; + let config = exe_unit_config(temp_dir, &resource!("agreement.json"), "ya-mock-runtime"); let exe = exe_unit(config).await.unwrap(); let mut finish = exe.send(FinishNotifier {}).await??; @@ -65,7 +33,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow log::info!("Sending [deploy, start] batch for execution."); - send_script( + let batch_id = send_script( exe.clone(), None, vec![ @@ -80,7 +48,10 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow ExeScriptCommand::Start { args: vec![] }, ], ) - .await; + .await + .unwrap(); + + exe.wait_for_batch(&batch_id).await.unwrap(); log::info!("Sending shutdown request."); diff --git a/test-utils/test-framework/framework-basic/src/file.rs b/test-utils/test-framework/framework-basic/src/file.rs index e18df9e037..89a83ecc32 100644 --- a/test-utils/test-framework/framework-basic/src/file.rs +++ b/test-utils/test-framework/framework-basic/src/file.rs @@ -1,17 +1,66 @@ +use rand::rngs::ThreadRng; use rand::Rng; use sha2::Digest; +use std::fs; use std::fs::OpenOptions; use std::io::Write; use std::path::{Path, PathBuf}; use crate::hash::HashOutput; -pub fn generate_file_with_hash( +trait ContentGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec; +} + +struct RandomGenerator(pub ThreadRng); +struct ZeroGenerator {} + +impl ContentGenerator for RandomGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec { + (0..chunk_size) + .map(|_| self.0.gen_range(0..256) as u8) + .collect() + } +} + +impl ContentGenerator for ZeroGenerator { + fn generate(&mut self, chunk_size: usize) -> Vec { + vec![0; chunk_size] + } +} + +pub fn generate_random_file_with_hash( path: &Path, name: &str, chunk_size: usize, chunk_count: usize, ) -> HashOutput { + generate_file_with_hash_( + path, + name, + chunk_size, + chunk_count, + RandomGenerator(rand::thread_rng()), + ) +} + +pub fn generate_image( + path: &Path, + name: &str, + chunk_size: usize, + chunk_count: usize, +) -> HashOutput { + generate_file_with_hash_(path, name, chunk_size, chunk_count, ZeroGenerator {}) +} + +fn generate_file_with_hash_( + path: &Path, + name: &str, + chunk_size: usize, + chunk_count: usize, + mut gen: impl ContentGenerator, +) -> HashOutput { + fs::create_dir_all(&path).ok(); let path = path.join(name); log::debug!( @@ -25,8 +74,6 @@ pub fn generate_file_with_hash( .open(path) .expect("rnd file"); - let mut rng = rand::thread_rng(); - for i in 0..chunk_count { log::trace!( "Generating chunk {i}/{chunk_count}. File size: {}/{}", @@ -34,9 +81,7 @@ pub fn generate_file_with_hash( chunk_count * chunk_size ); - let input: Vec = (0..chunk_size) - .map(|_| rng.gen_range(0..256) as u8) - .collect(); + let input: Vec = gen.generate(chunk_size); hasher.input(&input); let _ = file_src.write(&input).unwrap(); From 4a1b17bf70a0a6112105b431af43e788d1380a21 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 19 Jan 2024 17:58:22 +0100 Subject: [PATCH 022/125] Send terminate after test --- Cargo.lock | 5 +++ exe-unit/components/mock-runtime/Cargo.toml | 5 +++ .../mock-runtime/src/testing/exe_unit_ext.rs | 31 ++++++++++++++++++- exe-unit/tests/test_exe_unit_basic.rs | 7 +++-- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 516887adfd..cb5ff44d24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8743,16 +8743,21 @@ dependencies = [ "bytes 1.5.0", "env_logger 0.10.1", "futures 0.3.30", + "hex", "log", + "rand 0.8.5", "serde", "serde_json", "thiserror", "tokio", "tokio-util", "url", + "ya-client-model", + "ya-core-model", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.0", + "ya-service-bus", ] [[package]] diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index c73f95478f..6f36159b6e 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -32,6 +32,11 @@ ya-runtime-api = "0.7" # Dependancies for ExeUnit testing utils ya-exe-unit = "0.4" ya-framework-basic = { version = "0.1" } +ya-client-model = "0.6" +ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } +ya-service-bus = "0.6" async-trait = "0.1.77" actix = { version = "0.13", default-features = false } +hex = "0.4.3" +rand = "0.8.5" diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index 6dee0f450c..cad5eb993b 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -1,16 +1,24 @@ use actix::Addr; -use anyhow::bail; +use anyhow::{anyhow, bail}; use std::path::Path; use std::time::Duration; +use ya_client_model::activity::ExeScriptCommand; +use ya_core_model::activity; use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; use ya_framework_basic::async_drop::AsyncDroppable; use ya_framework_basic::test_dirs::cargo_binary; +use ya_service_bus::RpcEnvelope; #[async_trait::async_trait] pub trait ExeUnitExt { + async fn exec( + &self, + activity_id: Option, + exe_script: Vec, + ) -> anyhow::Result; async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()>; } @@ -50,6 +58,27 @@ pub fn exe_unit_config(temp_dir: &Path, agreement_path: &Path, binary: &str) -> #[async_trait::async_trait] impl ExeUnitExt for Addr> { + async fn exec( + &self, + activity_id: Option, + exe_script: Vec, + ) -> anyhow::Result { + log::debug!("Executing commands: {:?}", exe_script); + + let batch_id = hex::encode(rand::random::<[u8; 16]>()); + let msg = activity::Exec { + activity_id: activity_id.unwrap_or_default(), + batch_id: batch_id.clone(), + exe_script, + timeout: None, + }; + self.send(RpcEnvelope::with_caller(String::new(), msg)) + .await + .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))? + .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))?; + Ok(batch_id) + } + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()> { let delay = Duration::from_secs_f32(0.5); loop { diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 4fc29b002c..ee35beca46 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -55,12 +55,13 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow log::info!("Sending shutdown request."); - exe.send(Shutdown(ShutdownReason::Finished)) + exe.exec(None, vec![ExeScriptCommand::Terminate {}]) .await - .unwrap() .unwrap(); + exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); - finish.recv().await.unwrap(); + log::info!("Waiting for shutdown.."); + finish.recv().await.unwrap(); Ok(()) } From e955e5126d05c190faf5a405439c6d212f755c89 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 22 Jan 2024 17:11:16 +0100 Subject: [PATCH 023/125] Fix goth test due to chenged DebitNote ordering --- goth_tests/domain/exe_units/test_runtime_counters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goth_tests/domain/exe_units/test_runtime_counters.py b/goth_tests/domain/exe_units/test_runtime_counters.py index 6215b2e1c1..287912419f 100644 --- a/goth_tests/domain/exe_units/test_runtime_counters.py +++ b/goth_tests/domain/exe_units/test_runtime_counters.py @@ -113,7 +113,7 @@ async def test_custom_runtime_counter( ) debit_notes = await requestor.api.payment.get_debit_notes() - last_debit_note = debit_notes[0] + last_debit_note = debit_notes[len(debit_notes) - 1] logger.info("last debit note: %r", last_debit_note) assert len(last_debit_note.usage_counter_vector) == len(usage_vector) From 04b6dbf34c829bbe176b720b821c7c0f2a57c0a7 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 24 Jan 2024 16:24:26 +0100 Subject: [PATCH 024/125] Fix: Build test binary in test if it wasn't built --- Cargo.lock | 61 ++++++++++++++++ exe-unit/Cargo.toml | 1 + .../mock-runtime/src/testing/exe_unit_ext.rs | 9 ++- exe-unit/tests/test_exe_unit_basic.rs | 7 +- exe-unit/tests/test_progress.rs | 72 +++++++++++++++++++ .../test-framework/framework-basic/Cargo.toml | 2 + .../framework-basic/src/test_dirs.rs | 23 ++++-- 7 files changed, 166 insertions(+), 9 deletions(-) create mode 100644 exe-unit/tests/test_progress.rs diff --git a/Cargo.lock b/Cargo.lock index cb5ff44d24..4c8211108c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,6 +1085,52 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.21", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.21", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.0.83" @@ -6898,6 +6944,18 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "test-binary" +version = "3.0.2" +source = "git+https://github.com/golemfactory/test-binary.git#c9ebfa3e257455f8365e042b8838a518f2106169" +dependencies = [ + "camino", + "cargo_metadata 0.15.4", + "once_cell", + "paste", + "thiserror", +] + [[package]] name = "test-case" version = "2.2.2" @@ -8386,6 +8444,7 @@ dependencies = [ "derivative", "derive_more", "dotenv", + "duration-string", "env_logger 0.7.1", "flexi_logger 0.22.6", "futures 0.3.30", @@ -8471,6 +8530,7 @@ dependencies = [ "async-trait", "awc", "bytes 1.5.0", + "cargo_metadata 0.18.1", "crossterm 0.26.1", "env_logger 0.7.1", "futures 0.3.30", @@ -8488,6 +8548,7 @@ dependencies = [ "sha2 0.8.2", "sha3 0.8.2", "tempdir", + "test-binary", "test-context", "thiserror", "tokio", diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index 27ceca7cf7..5dbed9afc2 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -107,6 +107,7 @@ ya-sb-router = "0.6.1" actix-files = "0.6" actix-web = "4" +duration-string = "0.3" env_logger = "0.7" rustyline = "7.0.0" serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index cad5eb993b..6fd31ac1ea 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -9,7 +9,6 @@ use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; use ya_framework_basic::async_drop::AsyncDroppable; -use ya_framework_basic::test_dirs::cargo_binary; use ya_service_bus::RpcEnvelope; #[async_trait::async_trait] @@ -36,14 +35,18 @@ impl AsyncDroppable for ExeUnitHandle { } } -pub fn exe_unit_config(temp_dir: &Path, agreement_path: &Path, binary: &str) -> ExeUnitConfig { +pub fn exe_unit_config( + temp_dir: &Path, + agreement_path: &Path, + binary: impl AsRef, +) -> ExeUnitConfig { ExeUnitConfig { args: RunArgs { agreement: agreement_path.to_path_buf(), cache_dir: temp_dir.join("cache"), work_dir: temp_dir.join("work"), }, - binary: cargo_binary(binary).unwrap(), + binary: binary.as_ref().to_path_buf(), runtime_args: vec![], supervise: SuperviseCli { hardware: false, diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index ee35beca46..7ca451ba9f 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -7,6 +7,7 @@ use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; @@ -25,7 +26,11 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow .await .expect("unable to start http servers"); - let config = exe_unit_config(temp_dir, &resource!("agreement.json"), "ya-mock-runtime"); + let config = exe_unit_config( + temp_dir, + &resource!("agreement.json"), + cargo_binary("ya-mock-runtime")?, + ); let exe = exe_unit(config).await.unwrap(); let mut finish = exe.send(FinishNotifier {}).await??; diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs new file mode 100644 index 0000000000..4b1b0492bb --- /dev/null +++ b/exe-unit/tests/test_progress.rs @@ -0,0 +1,72 @@ +use duration_string::DurationString; +use std::str::FromStr; +use test_context::test_context; +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::ExeScriptCommand; +use ya_exe_unit::message::{Shutdown, ShutdownReason}; +use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::file::generate_image; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::server_external::start_http; +use ya_framework_basic::test_dirs::cargo_binary; +use ya_framework_basic::{resource, temp_dir}; +use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("exe-unit-start-terminate")?; + let temp_dir = dir.path(); + let image_repo = temp_dir.join("images"); + + let hash = generate_image(&image_repo, "image-big", 4096_usize, 10 * 1024); + log::info!("{}", hex::encode(&hash)); + start_http(ctx, image_repo) + .await + .expect("unable to start http servers"); + + let config = exe_unit_config( + temp_dir, + &resource!("agreement.json"), + cargo_binary("ya-mock-runtime")?, + ); + + let exe = exe_unit(config).await.unwrap(); + let mut finish = exe.send(FinishNotifier {}).await??; + ctx.register(ExeUnitHandle(exe.clone())); + + log::info!("Sending [deploy, start] batch for execution."); + + let batch_id = send_script( + exe.clone(), + None, + vec![ + ExeScriptCommand::Deploy { + net: vec![], + progress: Some(ProgressArgs { + update_interval: Some(DurationString::from_str("300ms").unwrap()), + update_step: None, + }), + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec![], + }, + ExeScriptCommand::Start { args: vec![] }, + ], + ) + .await + .unwrap(); + + exe.wait_for_batch(&batch_id).await.unwrap(); + exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); + + log::info!("Waiting for shutdown.."); + + finish.recv().await.unwrap(); + Ok(()) +} diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index 53474a3257..b21a87124f 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -29,6 +29,7 @@ async-compression = { version = "=0.3.7", features = [ "xz", ] } bytes = "1.0" +cargo_metadata = "0.18" crossterm = "0.26.1" env_logger = "0.7" futures = "0.3.4" @@ -45,6 +46,7 @@ serde = "1.0.104" sha2 = "0.8.1" sha3 = "0.8.2" tempdir = "0.3.7" +test-binary = { version = "3.0", git = "https://github.com/golemfactory/test-binary.git" } test-context = "0.1.4" thiserror = "1.0.11" tokio = { version = "1", features = ["fs", "io-util"] } diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs index bf0e1c892b..0be3a63cd2 100644 --- a/test-utils/test-framework/framework-basic/src/test_dirs.rs +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -1,7 +1,9 @@ use anyhow::{anyhow, bail}; use std::fs; use std::path::PathBuf; +use std::str::FromStr; use tempdir::TempDir; +use test_binary::TestBinary; pub mod macros { /// Creates temporary directory in cargo target directory. @@ -19,9 +21,6 @@ pub mod macros { #[macro_export] macro_rules! resource { ($name:literal) => { - // CARGO_MANIFEST_DIR is available in compile time only in binary modules, so we can't - // use it in this library. Thanks to using macro it will be resolved in final code not here - // and it will work. ya_framework_basic::test_dirs::resource_(env!("CARGO_MANIFEST_DIR"), $name) }; } @@ -56,8 +55,7 @@ pub fn extension() -> String { "".to_string() } -/// Returns absolute path to cargo project binary. -pub fn cargo_binary(bin_name: &str) -> anyhow::Result { +fn find_binary(bin_name: &str) -> anyhow::Result { let current = std::env::current_exe() .map_err(|e| anyhow!("Failed to get path to current binary. {e}"))? .parent() @@ -81,6 +79,21 @@ pub fn cargo_binary(bin_name: &str) -> anyhow::Result { Ok(bin_path) } +/// Returns path to test binary from workspace. +pub fn cargo_binary(bin_name: &str) -> anyhow::Result { + // Check if binary is already compiled. + if let Err(_) = find_binary(bin_name) { + TestBinary::from_workspace(&bin_name)? + .build() + .map_err(|e| anyhow!("Failed to compile binary: {e}"))? + .to_str() + .map(PathBuf::from_str) + .ok_or(anyhow!("Failed to convert path from OsString"))??; + }; + + find_binary(bin_name) +} + /// Returns resource from `resources` directory in tests. pub fn resource_(base_dir: &str, name: &str) -> PathBuf { PathBuf::from(base_dir) From 1a0458b7ae78ef81b9efa59ae42c02deb558da1a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 26 Jan 2024 17:05:00 +0100 Subject: [PATCH 025/125] Test progress report from streaming batch results --- Cargo.lock | 1 + exe-unit/components/mock-runtime/Cargo.toml | 1 + .../components/mock-runtime/src/testing.rs | 2 +- .../mock-runtime/src/testing/exe_unit_ext.rs | 123 +++++++++++++++--- exe-unit/src/handlers/rpc.rs | 1 + exe-unit/src/lib.rs | 8 +- .../tests/resources/agreement.template.json | 80 ++++++++++++ exe-unit/tests/test_exe_unit_basic.rs | 15 +-- exe-unit/tests/test_progress.rs | 114 +++++++++++----- .../framework-basic/src/test_dirs.rs | 23 +++- 10 files changed, 303 insertions(+), 65 deletions(-) create mode 100644 exe-unit/tests/resources/agreement.template.json diff --git a/Cargo.lock b/Cargo.lock index 4c8211108c..22148afcb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8813,6 +8813,7 @@ dependencies = [ "tokio", "tokio-util", "url", + "uuid 0.8.2", "ya-client-model", "ya-core-model", "ya-exe-unit", diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 6f36159b6e..426b232110 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -40,3 +40,4 @@ async-trait = "0.1.77" actix = { version = "0.13", default-features = false } hex = "0.4.3" rand = "0.8.5" +uuid = { version = "0.8.2", features = ["v4"] } diff --git a/exe-unit/components/mock-runtime/src/testing.rs b/exe-unit/components/mock-runtime/src/testing.rs index cb349f200a..7b2c833428 100644 --- a/exe-unit/components/mock-runtime/src/testing.rs +++ b/exe-unit/components/mock-runtime/src/testing.rs @@ -1,3 +1,3 @@ mod exe_unit_ext; -pub use exe_unit_ext::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; +pub use exe_unit_ext::{create_exe_unit, exe_unit_config, ExeUnitExt, ExeUnitHandle}; diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index 6fd31ac1ea..ad1e26e666 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -1,35 +1,68 @@ use actix::Addr; use anyhow::{anyhow, bail}; use std::path::Path; +use std::sync::Arc; use std::time::Duration; +use tokio::sync::broadcast; +use uuid::Uuid; -use ya_client_model::activity::ExeScriptCommand; +use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::{ExeScriptCommand, State, StatePair}; use ya_core_model::activity; -use ya_exe_unit::message::{GetBatchResults, Shutdown, ShutdownReason}; +use ya_exe_unit::message::{GetBatchResults, GetState, GetStateResponse, Shutdown, ShutdownReason}; use ya_exe_unit::runtime::process::RuntimeProcess; -use ya_exe_unit::{ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; -use ya_framework_basic::async_drop::AsyncDroppable; +use ya_exe_unit::{exe_unit, ExeUnit, ExeUnitConfig, FinishNotifier, RunArgs, SuperviseCli}; +use ya_framework_basic::async_drop::{AsyncDroppable, DroppableTestContext}; use ya_service_bus::RpcEnvelope; #[async_trait::async_trait] pub trait ExeUnitExt { async fn exec( &self, - activity_id: Option, + batch_id: Option, exe_script: Vec, ) -> anyhow::Result; + + async fn deploy(&self, progress: Option) -> anyhow::Result; + async fn start(&self, args: Vec) -> anyhow::Result; + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()>; + + /// Waits until ExeUnit will be ready to receive commands. + async fn await_init(&self) -> anyhow::Result<()>; } #[derive(Debug, Clone)] -pub struct ExeUnitHandle(pub Addr>); +pub struct ExeUnitHandle { + pub addr: Addr>, + pub config: Arc, +} + +impl ExeUnitHandle { + pub fn new( + addr: Addr>, + config: ExeUnitConfig, + ) -> anyhow::Result { + Ok(ExeUnitHandle { + addr, + config: Arc::new(config), + }) + } + + pub async fn finish_notifier(&self) -> anyhow::Result> { + Ok(self.addr.send(FinishNotifier {}).await??) + } +} #[async_trait::async_trait] impl AsyncDroppable for ExeUnitHandle { async fn async_drop(&self) { - let finish = self.0.send(FinishNotifier {}).await; - self.0.send(Shutdown(ShutdownReason::Finished)).await.ok(); - if let Ok(Ok(mut finish)) = finish { + let finish = self.finish_notifier().await; + self.addr + .send(Shutdown(ShutdownReason::Finished)) + .await + .ok(); + if let Ok(mut finish) = finish { finish.recv().await.ok(); } } @@ -54,38 +87,81 @@ pub fn exe_unit_config( }, sec_key: None, requestor_pub_key: None, - service_id: None, + //service_id: None, + service_id: Some(Uuid::new_v4().to_simple().to_string()), + //report_url: Some("/local/activity".to_string()), report_url: None, } } +pub async fn create_exe_unit( + config: ExeUnitConfig, + ctx: &mut DroppableTestContext, +) -> anyhow::Result { + let exe = exe_unit(config.clone()).await.unwrap(); + let handle = ExeUnitHandle::new(exe, config)?; + ctx.register(handle.clone()); + Ok(handle) +} + #[async_trait::async_trait] -impl ExeUnitExt for Addr> { +impl ExeUnitExt for ExeUnitHandle { async fn exec( &self, - activity_id: Option, + batch_id: Option, exe_script: Vec, ) -> anyhow::Result { log::debug!("Executing commands: {:?}", exe_script); - let batch_id = hex::encode(rand::random::<[u8; 16]>()); + let batch_id = if let Some(batch_id) = batch_id { + batch_id + } else { + hex::encode(rand::random::<[u8; 16]>()) + }; + let msg = activity::Exec { - activity_id: activity_id.unwrap_or_default(), + activity_id: self.config.service_id.clone().unwrap_or_default(), batch_id: batch_id.clone(), exe_script, timeout: None, }; - self.send(RpcEnvelope::with_caller(String::new(), msg)) + self.addr + .send(RpcEnvelope::with_caller(String::new(), msg)) .await .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))? .map_err(|e| anyhow!("Unable to execute exe script: {e:?}"))?; Ok(batch_id) } + async fn deploy(&self, progress: Option) -> anyhow::Result { + Ok(self + .exec( + None, + vec![ExeScriptCommand::Deploy { + net: vec![], + progress, + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec![], + }], + ) + .await + .unwrap()) + } + + async fn start(&self, args: Vec) -> anyhow::Result { + Ok(self + .exec(None, vec![ExeScriptCommand::Start { args }]) + .await + .unwrap()) + } + async fn wait_for_batch(&self, batch_id: &str) -> anyhow::Result<()> { let delay = Duration::from_secs_f32(0.5); loop { match self + .addr .send(GetBatchResults { batch_id: batch_id.to_string(), idx: None, @@ -104,4 +180,21 @@ impl ExeUnitExt for Addr> { tokio::time::sleep(delay).await; } } + + async fn await_init(&self) -> anyhow::Result<()> { + let delay = Duration::from_secs_f32(0.3); + loop { + match self.addr.send(GetState).await { + Ok(GetStateResponse(StatePair(State::Initialized, None))) => break, + Ok(GetStateResponse(StatePair(State::Terminated, _))) + | Ok(GetStateResponse(StatePair(_, Some(State::Terminated)))) + | Err(_) => { + log::error!("ExeUnit has terminated"); + bail!("ExeUnit has terminated"); + } + _ => tokio::time::sleep(delay).await, + } + } + Ok(()) + } } diff --git a/exe-unit/src/handlers/rpc.rs b/exe-unit/src/handlers/rpc.rs index 44ccd344ca..9788175683 100644 --- a/exe-unit/src/handlers/rpc.rs +++ b/exe-unit/src/handlers/rpc.rs @@ -22,6 +22,7 @@ impl Handler> for ExeUnit { type Result = as Message>::Result; fn handle(&mut self, msg: RpcEnvelope, ctx: &mut Self::Context) -> Self::Result { + log::debug!("Received Exec message: {:?}", msg.as_ref()); self.ctx.verify_activity_id(&msg.activity_id)?; let batch_id = msg.batch_id.clone(); diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 38608f5561..5e259fc843 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -205,12 +205,10 @@ pub async fn send_script( exe_script, timeout: None, }; - if let Err(e) = exe_unit + + exe_unit .send(RpcEnvelope::with_caller(String::new(), msg)) - .await - { - log::error!("Unable to execute exe script: {:?}", e); - } + .await??; Ok(batch_id) } diff --git a/exe-unit/tests/resources/agreement.template.json b/exe-unit/tests/resources/agreement.template.json new file mode 100644 index 0000000000..a3488f8354 --- /dev/null +++ b/exe-unit/tests/resources/agreement.template.json @@ -0,0 +1,80 @@ +{ + "agreementId": "0ec929f5acc8f98a47ab72d61a2c2f343d45d8438d3aa4ccdc84e717c219e185", + "proposedSignature": "NoSignature", + "state": "Pending", + "timestamp": "2022-05-22T10:41:42.564784259Z", + "validTo": "2022-05-22T11:41:42.562457Z", + + "offer": { + "properties": { + "golem.activity.caps.transfer.protocol": [ + "gftp", + "https", + "http" + ], + "golem.activity.caps.transfer.report-progress": true, + "golem.com.payment.debit-notes.accept-timeout?": 240, + "golem.com.payment.platform.erc20-goerli-tglm.address": "0x95369fc6fd02afeca110b9c32a21fb8ad899ee0a", + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": [ + 0.001388888888888889, + 0.0002777777777777778, + 0.0 + ], + "golem.com.scheme": "payu", + "golem.com.scheme.payu.debit-note.interval-sec?": 120, + "golem.com.scheme.payu.payment-timeout-sec?": 120, + "golem.com.usage.vector": [ + "golem.usage.cpu_sec", + "golem.usage.duration_sec" + ], + "golem.inf.cpu.architecture": "x86_64", + "golem.inf.cpu.brand": "Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz", + "golem.inf.cpu.capabilities": [], + "golem.inf.cpu.cores": 6, + "golem.inf.cpu.model": "Stepping 10 Family 6 Model 302", + "golem.inf.cpu.threads": 11, + "golem.inf.cpu.vendor": "GenuineIntel", + "golem.inf.mem.gib": 28.0, + "golem.inf.storage.gib": 10.188121795654297, + "golem.node.id.name": "mock-provider", + "golem.node.net.is-public": true, + "golem.runtime.capabilities": [], + "golem.runtime.name": "ya-mock-runtime", + "golem.runtime.version": "0.1.0", + "golem.srv.caps.multi-activity": true, + "golem.srv.caps.payload-manifest": true + }, + "constraints": "(&\n (golem.srv.comp.expiration>1705586871777)\n)", + "offerId": "afce49b1ea5b45db91bdd6e5481479f9-9095fca9dea0a91ce95cf994125b33cdd838fcc963a1106f2be9e4b5b65a52f0", + "providerId": "0x86a269498fb5270f20bdc6fdcf6039122b0d3b23", + "timestamp": "2022-05-22T10:41:42.564784259Z" + }, + + "demand": { + "constraints": "(&(golem.com.payment.platform.erc20-goerli-tglm.address=*)\n\t(golem.com.pricing.model=linear)\n\t(&(golem.inf.mem.gib>=0.5)\n\t(golem.inf.storage.gib>=2.0)\n\t(golem.inf.cpu.threads>=1)\n\t(golem.runtime.name=ya-mock-runtime)))", + "demandId": "773035fc685c46da8e61473ac2a2568e-3f3eb86d6ef9a01708d0f57d0b19cc69fd74422150c120e33cc1b5f4a1a12b96", + "properties": { + "golem": { + "com.payment": { + "chosen-platform": "erc20-goerli-tglm", + "debit-notes.accept-timeout?": 240, + "platform.erc20-goerli-tglm.address": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7" + }, + "node": {}, + "srv": { + "caps.multi-activity": true, + "comp": { + "expiration": 1653216996555, + "task_package": "${task-package}", + "vm": { + "package_format": "gvmkit-squash" + } + } + } + } + }, + "requestorId": "0xa5ad3f81e283983b8e9705b2e31d0c138bb2b1b7", + "timestamp": "2022-05-22T10:41:42.564784259Z" + } +} diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 7ca451ba9f..a0ae09d0b5 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -2,14 +2,14 @@ use test_context::test_context; use ya_client_model::activity::ExeScriptCommand; use ya_exe_unit::message::{Shutdown, ShutdownReason}; -use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; +use ya_exe_unit::send_script; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; -use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; +use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] @@ -32,15 +32,14 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow cargo_binary("ya-mock-runtime")?, ); - let exe = exe_unit(config).await.unwrap(); - let mut finish = exe.send(FinishNotifier {}).await??; - ctx.register(ExeUnitHandle(exe.clone())); + let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + let mut finish = exe.finish_notifier().await?; log::info!("Sending [deploy, start] batch for execution."); let batch_id = send_script( - exe.clone(), - None, + exe.addr.clone(), + config.service_id.clone(), vec![ ExeScriptCommand::Deploy { net: vec![], @@ -63,7 +62,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow exe.exec(None, vec![ExeScriptCommand::Terminate {}]) .await .unwrap(); - exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); + exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); log::info!("Waiting for shutdown.."); diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 4b1b0492bb..e72d3b4966 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -1,72 +1,116 @@ +use anyhow::Context; use duration_string::DurationString; +use futures::StreamExt; use std::str::FromStr; use test_context::test_context; +use url::Url; + use ya_client_model::activity::exe_script_command::ProgressArgs; -use ya_client_model::activity::ExeScriptCommand; +use ya_client_model::activity::RuntimeEventKind; +use ya_core_model::activity; use ya_exe_unit::message::{Shutdown, ShutdownReason}; -use ya_exe_unit::{exe_unit, send_script, FinishNotifier}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; use ya_framework_basic::server_external::start_http; -use ya_framework_basic::test_dirs::cargo_binary; +use ya_framework_basic::test_dirs::{cargo_binary, template}; use ya_framework_basic::{resource, temp_dir}; -use ya_mock_runtime::testing::{exe_unit_config, ExeUnitExt, ExeUnitHandle}; +use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; + +use ya_service_bus::typed as bus; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] -async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); +async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(false); - let dir = temp_dir!("exe-unit-start-terminate")?; + let dir = temp_dir!("progress_reporting")?; let temp_dir = dir.path(); let image_repo = temp_dir.join("images"); - let hash = generate_image(&image_repo, "image-big", 4096_usize, 10 * 1024); - log::info!("{}", hex::encode(&hash)); - start_http(ctx, image_repo) + let chunk_size = 4096_usize; + let chunk_count = 1024 * 1; + let file_size = (chunk_size * chunk_count) as u64; + + let hash = generate_image(&image_repo, "image-big", chunk_size, chunk_count); + let package = format!( + "hash://sha3:{}:http://127.0.0.1:8001/image-big", + hex::encode(&hash) + ); + start_http(ctx, image_repo.clone()) .await .expect("unable to start http servers"); + let gsb_url = Url::from_str(&format!("unix://{}/mock-yagna.sock", temp_dir.display())).unwrap(); + std::env::set_var("GSB_URL", gsb_url.to_string()); + ya_sb_router::bind_gsb_router(Some(gsb_url)) + .await + .context("binding service bus router")?; + let config = exe_unit_config( temp_dir, - &resource!("agreement.json"), + &template( + &resource!("agreement.template.json"), + temp_dir.join("agreement.json"), + &[("task-package", package)], + )?, cargo_binary("ya-mock-runtime")?, ); - let exe = exe_unit(config).await.unwrap(); - let mut finish = exe.send(FinishNotifier {}).await??; - ctx.register(ExeUnitHandle(exe.clone())); + let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + let mut finish = exe.finish_notifier().await?; + exe.await_init().await.unwrap(); log::info!("Sending [deploy, start] batch for execution."); - let batch_id = send_script( - exe.clone(), - None, - vec![ - ExeScriptCommand::Deploy { - net: vec![], - progress: Some(ProgressArgs { - update_interval: Some(DurationString::from_str("300ms").unwrap()), - update_step: None, - }), - env: Default::default(), - hosts: Default::default(), - hostname: None, - volumes: vec![], - }, - ExeScriptCommand::Start { args: vec![] }, - ], - ) - .await - .unwrap(); + let batch_id = exe + .deploy(Some(ProgressArgs { + update_interval: Some(DurationString::from_str("300ms").unwrap()), + update_step: None, + })) + .await + .unwrap(); + + let msg = activity::StreamExecBatchResults { + activity_id: config.service_id.unwrap(), + batch_id: batch_id.clone(), + }; + + // Note: Since we have already sent commands, we may loose a few events on the beginning. + // Our API has a problem here. We can't call `StreamExecBatchResults` before Exeunit knows + // `batch_id`. Even if we would generate id ourselves (possible in test, but not possible for Requestor), + // we still can't call this function too early. + let mut stream = bus::service(activity::exeunit::bus_id(&msg.activity_id)).call_streaming(msg); + + let mut last_progress = 0u64; + while let Some(Ok(Ok(item))) = stream.next().await { + if item.index == 0 { + match item.kind { + RuntimeEventKind::Finished { return_code, .. } => { + assert_eq!(return_code, 0); + break; + } + RuntimeEventKind::Progress(progress) => { + log::info!("Progress report: {:?}", progress); + + assert_eq!(progress.step, (0, 1)); + assert_eq!(progress.unit, Some("Bytes".to_string())); + assert_eq!(progress.progress.1.unwrap(), file_size); + assert!(progress.progress.0 >= last_progress); + + last_progress = progress.progress.0; + } + _ => (), + } + } + } exe.wait_for_batch(&batch_id).await.unwrap(); - exe.send(Shutdown(ShutdownReason::Finished)).await.ok(); log::info!("Waiting for shutdown.."); + exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); finish.recv().await.unwrap(); Ok(()) } diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs index 0be3a63cd2..c0039e6577 100644 --- a/test-utils/test-framework/framework-basic/src/test_dirs.rs +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -1,6 +1,6 @@ use anyhow::{anyhow, bail}; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; use tempdir::TempDir; use test_binary::TestBinary; @@ -101,3 +101,24 @@ pub fn resource_(base_dir: &str, name: &str) -> PathBuf { .join("resources") .join(name) } + +/// Generates resource from template by replacing occurrences of `${name}` pattern +/// using variables from `vars` dictionary. +/// Returns path to generated file, which is the same as `target` param, but makes it easier +/// to use this function in code. +pub fn template( + template: &Path, + target: impl AsRef, + vars: &[(&str, String)], +) -> anyhow::Result { + let mut template = fs::read_to_string(&template) + .map_err(|e| anyhow!("Loading template {} failed: {e}", template.display()))?; + for var in vars { + template = template.replace(&format!("${{{}}}", var.0), &var.1); + } + + let target = target.as_ref(); + fs::write(target, template) + .map_err(|e| anyhow!("Saving template {} failed: {e}", target.display()))?; + Ok(target.to_path_buf()) +} From 9a8042ef39b95dfbd871f80a294af6bf7b556cdc Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 26 Jan 2024 17:46:54 +0100 Subject: [PATCH 026/125] Handle GSB router setup in test utils --- Cargo.lock | 11 ++++++ exe-unit/components/mock-runtime/Cargo.toml | 2 ++ .../mock-runtime/src/testing/exe_unit_ext.rs | 34 +++++++++++++++++-- exe-unit/tests/test_progress.rs | 8 ----- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22148afcb1..ef4055fc60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4662,6 +4662,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "portpicker" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -8806,6 +8815,7 @@ dependencies = [ "futures 0.3.30", "hex", "log", + "portpicker", "rand 0.8.5", "serde", "serde_json", @@ -8819,6 +8829,7 @@ dependencies = [ "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.0", + "ya-sb-router", "ya-service-bus", ] diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 426b232110..dc65e9dee9 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -35,9 +35,11 @@ ya-framework-basic = { version = "0.1" } ya-client-model = "0.6" ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } ya-service-bus = "0.6" +ya-sb-router = "0.6" async-trait = "0.1.77" actix = { version = "0.13", default-features = false } hex = "0.4.3" +portpicker = "0.1.1" rand = "0.8.5" uuid = { version = "0.8.2", features = ["v4"] } diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index ad1e26e666..d8b26d0666 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -1,9 +1,12 @@ use actix::Addr; use anyhow::{anyhow, bail}; -use std::path::Path; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::Arc; use std::time::Duration; use tokio::sync::broadcast; +use url::Url; use uuid::Uuid; use ya_client_model::activity::exe_script_command::ProgressArgs; @@ -87,9 +90,7 @@ pub fn exe_unit_config( }, sec_key: None, requestor_pub_key: None, - //service_id: None, service_id: Some(Uuid::new_v4().to_simple().to_string()), - //report_url: Some("/local/activity".to_string()), report_url: None, } } @@ -98,6 +99,33 @@ pub async fn create_exe_unit( config: ExeUnitConfig, ctx: &mut DroppableTestContext, ) -> anyhow::Result { + if config.service_id.is_some() { + let gsb_url = match std::env::consts::FAMILY { + "unix" => Url::from_str(&format!( + "unix://{}/gsb.sock", + config.args.work_dir.display() + ))?, + _ => Url::from_str(&format!( + "tcp://127.0.0.1:{}", + portpicker::pick_unused_port().ok_or(anyhow!("No ports free"))? + ))?, + }; + + if gsb_url.scheme() == "unix" { + let dir = PathBuf::from_str(gsb_url.path())? + .parent() + .map(|path| path.to_path_buf()) + .ok_or(anyhow!("`gsb_url` unix socket has no parent directory."))?; + fs::create_dir_all(dir)?; + } + + // GSB takes url from this variable and we can't set it directly. + std::env::set_var("GSB_URL", gsb_url.to_string()); + ya_sb_router::bind_gsb_router(Some(gsb_url.clone())) + .await + .map_err(|e| anyhow!("Error binding service bus router to '{}': {e}", &gsb_url))?; + } + let exe = exe_unit(config.clone()).await.unwrap(); let handle = ExeUnitHandle::new(exe, config)?; ctx.register(handle.clone()); diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index e72d3b4966..5fd33edc73 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -1,9 +1,7 @@ -use anyhow::Context; use duration_string::DurationString; use futures::StreamExt; use std::str::FromStr; use test_context::test_context; -use url::Url; use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::RuntimeEventKind; @@ -42,12 +40,6 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu .await .expect("unable to start http servers"); - let gsb_url = Url::from_str(&format!("unix://{}/mock-yagna.sock", temp_dir.display())).unwrap(); - std::env::set_var("GSB_URL", gsb_url.to_string()); - ya_sb_router::bind_gsb_router(Some(gsb_url)) - .await - .context("binding service bus router")?; - let config = exe_unit_config( temp_dir, &template( From 83f62863dd585ebe59b5ead26512c906edf17813 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 26 Jan 2024 17:55:18 +0100 Subject: [PATCH 027/125] Hide shutdown function in tests --- .../mock-runtime/src/testing/exe_unit_ext.rs | 18 +++++++++++++----- exe-unit/tests/test_exe_unit_basic.rs | 7 +------ exe-unit/tests/test_progress.rs | 10 +++------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index d8b26d0666..bbded7e242 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -55,19 +55,27 @@ impl ExeUnitHandle { pub async fn finish_notifier(&self) -> anyhow::Result> { Ok(self.addr.send(FinishNotifier {}).await??) } -} -#[async_trait::async_trait] -impl AsyncDroppable for ExeUnitHandle { - async fn async_drop(&self) { + pub async fn shutdown(&self) -> anyhow::Result<()> { let finish = self.finish_notifier().await; + + log::info!("Waiting for shutdown.."); + self.addr .send(Shutdown(ShutdownReason::Finished)) .await .ok(); if let Ok(mut finish) = finish { - finish.recv().await.ok(); + finish.recv().await?; } + Ok(()) + } +} + +#[async_trait::async_trait] +impl AsyncDroppable for ExeUnitHandle { + async fn async_drop(&self) { + self.shutdown().await.ok(); } } diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index a0ae09d0b5..618fa20595 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -1,7 +1,6 @@ use test_context::test_context; use ya_client_model::activity::ExeScriptCommand; -use ya_exe_unit::message::{Shutdown, ShutdownReason}; use ya_exe_unit::send_script; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; @@ -33,7 +32,6 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow ); let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); - let mut finish = exe.finish_notifier().await?; log::info!("Sending [deploy, start] batch for execution."); @@ -62,10 +60,7 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow exe.exec(None, vec![ExeScriptCommand::Terminate {}]) .await .unwrap(); - exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); - log::info!("Waiting for shutdown.."); - - finish.recv().await.unwrap(); + exe.shutdown().await.unwrap(); Ok(()) } diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 5fd33edc73..4befc7c2c1 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -6,7 +6,6 @@ use test_context::test_context; use ya_client_model::activity::exe_script_command::ProgressArgs; use ya_client_model::activity::RuntimeEventKind; use ya_core_model::activity; -use ya_exe_unit::message::{Shutdown, ShutdownReason}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; @@ -51,7 +50,6 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu ); let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); - let mut finish = exe.finish_notifier().await?; exe.await_init().await.unwrap(); log::info!("Sending [deploy, start] batch for execution."); @@ -76,6 +74,7 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let mut stream = bus::service(activity::exeunit::bus_id(&msg.activity_id)).call_streaming(msg); let mut last_progress = 0u64; + let mut num_progresses = 0u64; while let Some(Ok(Ok(item))) = stream.next().await { if item.index == 0 { match item.kind { @@ -92,17 +91,14 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu assert!(progress.progress.0 >= last_progress); last_progress = progress.progress.0; + num_progresses += 1; } _ => (), } } } + assert!(num_progresses > 1); exe.wait_for_batch(&batch_id).await.unwrap(); - - log::info!("Waiting for shutdown.."); - - exe.addr.send(Shutdown(ShutdownReason::Finished)).await.ok(); - finish.recv().await.unwrap(); Ok(()) } From 39a790f3a58222669fe5bf696cfa404121f9ac3c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 15:02:34 +0100 Subject: [PATCH 028/125] Deploy - adding additional volumes (not included in vm image); Test transfer progress --- .../mock-runtime/src/testing/exe_unit_ext.rs | 2 + exe-unit/src/exe_unit.rs | 12 ++++ exe-unit/tests/test_progress.rs | 70 ++++++++++++++++--- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index bbded7e242..8559a20c27 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -107,6 +107,8 @@ pub async fn create_exe_unit( config: ExeUnitConfig, ctx: &mut DroppableTestContext, ) -> anyhow::Result { + // If activity id was provided, ExeUnit will bind endpoints on remote GSB. + // For this to work we need to setup gsb router. if config.service_id.is_some() { let gsb_url = match std::env::consts::FAMILY { "unix" => Url::from_str(&format!( diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index aa4a426038..10f97c967d 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -15,6 +15,7 @@ use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, use ya_core_model::activity; use ya_core_model::activity::local::Credentials; use ya_runtime_api::deploy; +use ya_runtime_api::deploy::ContainerVolume; use ya_service_bus::{actix_rpc, RpcEndpoint, RpcMessage}; use ya_transfer::transfer::{ AddVolumes, DeployImage, ForwardProgressToSink, TransferResource, TransferService, @@ -313,8 +314,19 @@ impl RuntimeRef { net, hosts, progress, + volumes, .. } => { + let volumes = volumes + .iter() + .enumerate() + .map(|(idx, vol)| ContainerVolume { + name: format!("vol-custom-{idx}"), + path: vol.to_string(), + }) + .collect::>(); + transfer_service.send(AddVolumes::new(volumes)).await??; + // TODO: We should pass `task_package` here not in `TransferService` initialization. let mut msg = DeployImage::default(); if let Some(args) = progress { diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 4befc7c2c1..2c23548083 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use test_context::test_context; use ya_client_model::activity::exe_script_command::ProgressArgs; -use ya_client_model::activity::RuntimeEventKind; +use ya_client_model::activity::{ExeScriptCommand, RuntimeEventKind, TransferArgs}; use ya_core_model::activity; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; @@ -16,13 +16,15 @@ use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; use ya_service_bus::typed as bus; +/// Test if progress reporting mechanisms work on gsb level +/// with full ExeUnit setup. #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { enable_logs(false); - let dir = temp_dir!("progress_reporting")?; + let dir = temp_dir!("progress-reporting")?; let temp_dir = dir.path(); let image_repo = temp_dir.join("images"); @@ -52,18 +54,67 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); exe.await_init().await.unwrap(); - log::info!("Sending [deploy, start] batch for execution."); + log::info!("Sending [deploy] batch for execution."); let batch_id = exe - .deploy(Some(ProgressArgs { - update_interval: Some(DurationString::from_str("300ms").unwrap()), - update_step: None, - })) + .exec( + None, + vec![ExeScriptCommand::Deploy { + net: vec![], + progress: Some(ProgressArgs { + update_interval: Some(DurationString::from_str("300ms").unwrap()), + update_step: None, + }), + env: Default::default(), + hosts: Default::default(), + hostname: None, + volumes: vec!["/input".to_owned()], + }], + ) .await .unwrap(); + validate_progress( + config.service_id.clone().unwrap(), + batch_id.clone(), + file_size, + ) + .await; + + exe.wait_for_batch(&batch_id).await.unwrap(); + exe.wait_for_batch(&exe.start(vec![]).await.unwrap()) + .await + .unwrap(); + + let batch_id = exe + .exec( + None, + vec![ExeScriptCommand::Transfer { + args: TransferArgs::default(), + progress: Some(ProgressArgs { + update_interval: Some(DurationString::from_str("100ms").unwrap()), + update_step: None, + }), + // Important: Use hashed transfer, because it is significantly slower in debug mode. + // Otherwise we won't get any progress message, because it is too fast. + from: format!( + "hash://sha3:{}:http://127.0.0.1:8001/image-big", + hex::encode(&hash) + ), + to: "container:/input/image-copy".to_string(), + }], + ) + .await + .unwrap(); + + validate_progress(config.service_id.unwrap(), batch_id.clone(), file_size).await; + exe.wait_for_batch(&batch_id).await.unwrap(); + Ok(()) +} + +async fn validate_progress(activity_id: String, batch_id: String, file_size: u64) { let msg = activity::StreamExecBatchResults { - activity_id: config.service_id.unwrap(), + activity_id: activity_id.clone(), batch_id: batch_id.clone(), }; @@ -98,7 +149,4 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu } } assert!(num_progresses > 1); - - exe.wait_for_batch(&batch_id).await.unwrap(); - Ok(()) } From 22c04f6db0451bbf1ade1a0274f04f181bc4a93c Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 16:21:55 +0100 Subject: [PATCH 029/125] Rename framework-test to system-test in exeunits --- Cargo.toml | 2 +- exe-unit/Cargo.toml | 2 +- exe-unit/components/transfer/Cargo.toml | 2 +- exe-unit/components/transfer/tests/test_deploy.rs | 4 ++-- exe-unit/components/transfer/tests/test_transfer_abort.rs | 2 +- exe-unit/components/transfer/tests/test_transfer_resume.rs | 2 +- exe-unit/components/transfer/tests/test_transfer_service.rs | 4 ++-- exe-unit/tests/test_exe_unit_basic.rs | 2 +- exe-unit/tests/test_progress.rs | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 99643039c9..ca1444b979 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] erc20next-driver = ['ya-erc20next-driver'] tos = [] -framework-test = ['ya-exe-unit/framework-test'] +framework-test = ['ya-exe-unit/system-test'] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index 5dbed9afc2..55a66d937c 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -24,7 +24,7 @@ sgx = [ 'ya-transfer/sgx', ] packet-trace-enable = ["ya-packet-trace/enable"] -framework-test = ["ya-transfer/framework-test"] +system-test = ["ya-transfer/system-test"] [target.'cfg(target_family = "unix")'.dependencies] nix = "0.22.0" diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index e9e67ddae0..468b6dd483 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -59,7 +59,7 @@ sgx = [ 'ya-core-model/sgx', 'reqwest/trust-dns', ] -framework-test = [] +system-test = [] [dependencies.zip] version = "0.5.6" diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 2f84fb4d24..19e15078d2 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -18,7 +18,7 @@ use ya_transfer::transfer::{AbortTransfers, DeployImage, TransferService, Transf /// When re-deploying image, `TransferService` should uses partially downloaded image. /// Hash computations should be correct in both cases. -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -78,7 +78,7 @@ async fn test_deploy_image_restart(ctx: &mut DroppableTestContext) -> anyhow::Re Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/components/transfer/tests/test_transfer_abort.rs b/exe-unit/components/transfer/tests/test_transfer_abort.rs index 3b8e7e7ec6..12d2dc8f92 100644 --- a/exe-unit/components/transfer/tests/test_transfer_abort.rs +++ b/exe-unit/components/transfer/tests/test_transfer_abort.rs @@ -50,7 +50,7 @@ async fn interrupted_transfer( Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_abort(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/components/transfer/tests/test_transfer_resume.rs b/exe-unit/components/transfer/tests/test_transfer_resume.rs index 03920246b6..bf62391943 100644 --- a/exe-unit/components/transfer/tests/test_transfer_resume.rs +++ b/exe-unit/components/transfer/tests/test_transfer_resume.rs @@ -128,7 +128,7 @@ async fn transfer( transfer_with_args(addr, from, to, TransferArgs::default()).await } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_resume(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/components/transfer/tests/test_transfer_service.rs b/exe-unit/components/transfer/tests/test_transfer_service.rs index 500b21ecf8..4c4217bfec 100644 --- a/exe-unit/components/transfer/tests/test_transfer_service.rs +++ b/exe-unit/components/transfer/tests/test_transfer_service.rs @@ -38,7 +38,7 @@ async fn transfer_with_args( Ok(()) } -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -151,7 +151,7 @@ async fn test_transfer_scenarios(ctx: &mut DroppableTestContext) -> anyhow::Resu } #[ignore] -//#[cfg_attr(not(feature = "framework-test"), ignore)] +//#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_transfer_archived(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 618fa20595..5d97f6bfe9 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -10,7 +10,7 @@ use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 2c23548083..d6c2146988 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -18,11 +18,11 @@ use ya_service_bus::typed as bus; /// Test if progress reporting mechanisms work on gsb level /// with full ExeUnit setup. -#[cfg_attr(not(feature = "framework-test"), ignore)] +#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(false); + enable_logs(true); let dir = temp_dir!("progress-reporting")?; let temp_dir = dir.path(); From 0fbd17126d8ad1e12f44c3ec867e0187415c94cf Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 17:59:19 +0100 Subject: [PATCH 030/125] Replace basic exeunit test with helper deploy, start commands --- exe-unit/tests/test_exe_unit_basic.rs | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 5d97f6bfe9..5c354e4b6c 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -1,7 +1,6 @@ use test_context::test_context; use ya_client_model::activity::ExeScriptCommand; -use ya_exe_unit::send_script; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::file::generate_image; use ya_framework_basic::log::enable_logs; @@ -32,28 +31,16 @@ async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow ); let exe = create_exe_unit(config.clone(), ctx).await.unwrap(); + exe.await_init().await.unwrap(); log::info!("Sending [deploy, start] batch for execution."); - let batch_id = send_script( - exe.addr.clone(), - config.service_id.clone(), - vec![ - ExeScriptCommand::Deploy { - net: vec![], - progress: None, - env: Default::default(), - hosts: Default::default(), - hostname: None, - volumes: vec![], - }, - ExeScriptCommand::Start { args: vec![] }, - ], - ) - .await - .unwrap(); - - exe.wait_for_batch(&batch_id).await.unwrap(); + exe.wait_for_batch(&exe.deploy(None).await.unwrap()) + .await + .unwrap(); + exe.wait_for_batch(&exe.start(vec![]).await.unwrap()) + .await + .unwrap(); log::info!("Sending shutdown request."); From 148362f44acdc79b45c78472ff7afa18e92d9f3d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 18:09:24 +0100 Subject: [PATCH 031/125] Fix clippy --- exe-unit/components/transfer/src/progress.rs | 4 ++-- test-utils/test-framework/framework-basic/src/file.rs | 2 +- test-utils/test-framework/framework-basic/src/test_dirs.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 289ed049b2..b8df121ef6 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -57,7 +57,7 @@ impl ProgressReporter { .unwrap_or(Duration::from_secs(1)); let _update_step = self.config.update_step; - self.inner.lock().unwrap().as_mut().map(|inner| { + if let Some(inner) = self.inner.lock().unwrap().as_mut() { inner.last.progress = (progress, size); if inner.last_send + update_interval <= Instant::now() { inner.last_send = Instant::now(); @@ -69,7 +69,7 @@ impl ProgressReporter { }) .ok(); } - }); + } } pub fn report_message(&self, message: String) { diff --git a/test-utils/test-framework/framework-basic/src/file.rs b/test-utils/test-framework/framework-basic/src/file.rs index 89a83ecc32..b2e1c0b9bf 100644 --- a/test-utils/test-framework/framework-basic/src/file.rs +++ b/test-utils/test-framework/framework-basic/src/file.rs @@ -60,7 +60,7 @@ fn generate_file_with_hash_( chunk_count: usize, mut gen: impl ContentGenerator, ) -> HashOutput { - fs::create_dir_all(&path).ok(); + fs::create_dir_all(path).ok(); let path = path.join(name); log::debug!( diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs index c0039e6577..374c697b2a 100644 --- a/test-utils/test-framework/framework-basic/src/test_dirs.rs +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -82,8 +82,8 @@ fn find_binary(bin_name: &str) -> anyhow::Result { /// Returns path to test binary from workspace. pub fn cargo_binary(bin_name: &str) -> anyhow::Result { // Check if binary is already compiled. - if let Err(_) = find_binary(bin_name) { - TestBinary::from_workspace(&bin_name)? + if find_binary(bin_name).is_err() { + TestBinary::from_workspace(bin_name)? .build() .map_err(|e| anyhow!("Failed to compile binary: {e}"))? .to_str() @@ -111,7 +111,7 @@ pub fn template( target: impl AsRef, vars: &[(&str, String)], ) -> anyhow::Result { - let mut template = fs::read_to_string(&template) + let mut template = fs::read_to_string(template) .map_err(|e| anyhow!("Loading template {} failed: {e}", template.display()))?; for var in vars { template = template.replace(&format!("${{{}}}", var.0), &var.1); From 15eeb5ae49eb77910225fdf434b34342f77e7be3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 18:20:16 +0100 Subject: [PATCH 032/125] Fix clipy mut sgx --- exe-unit/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 5e259fc843..7d9ac9e7db 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -308,7 +308,9 @@ pub struct ExeUnitConfig { pub requestor_pub_key: Option, } -pub async fn exe_unit(config: ExeUnitConfig) -> anyhow::Result>> { +// Mut is necessary in case of sgx compilation :((((( +#[allow(unused_mut)] +pub async fn exe_unit(mut config: ExeUnitConfig) -> anyhow::Result>> { let args = config.args; if !args.agreement.exists() { bail!( From bc8c02be3c46ecefb4e9e2cf3c76e1ce30a9dfe4 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 18:34:34 +0100 Subject: [PATCH 033/125] Allow large enum variant --- exe-unit/src/message.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/exe-unit/src/message.rs b/exe-unit/src/message.rs index 8c19e9c3ee..d105cc392d 100644 --- a/exe-unit/src/message.rs +++ b/exe-unit/src/message.rs @@ -125,6 +125,7 @@ impl ExecuteCommand { } } +#[allow(clippy::large_enum_variant)] #[derive(Clone, Debug)] pub enum RuntimeEvent { Process(activity::RuntimeEvent), From 7a6613dab6c00fce4d651fe5674ede70daf91723 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 19:33:00 +0100 Subject: [PATCH 034/125] Run system tests for ya-transfer and ya-exe-unit --- .github/workflows/system-test.yml | 2 +- exe-unit/components/mock-runtime/src/lib.rs | 4 ++-- exe-unit/components/transfer/src/progress.rs | 2 +- tests/readme.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/system-test.yml b/.github/workflows/system-test.yml index bf10fc47a9..50898e7003 100644 --- a/.github/workflows/system-test.yml +++ b/.github/workflows/system-test.yml @@ -73,4 +73,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --test '*' --features framework-test + args: --test '*' -p yagna -p ya-exe-unit -p ya-transfer --features framework-test diff --git a/exe-unit/components/mock-runtime/src/lib.rs b/exe-unit/components/mock-runtime/src/lib.rs index c818b73da4..ee9ed1bab4 100644 --- a/exe-unit/components/mock-runtime/src/lib.rs +++ b/exe-unit/components/mock-runtime/src/lib.rs @@ -60,12 +60,12 @@ impl RuntimeService for RuntimeMock { // client // holds last received status -#[derive(Clone)] +#[derive(Clone, Default)] pub struct EventMock(Arc>); impl EventMock { pub fn new() -> Self { - Self(Arc::new(Mutex::new(Default::default()))) + Self::default() } pub fn get_last_status(&self) -> ProcessStatus { diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index b8df121ef6..332c5d6d3b 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -255,7 +255,7 @@ mod tests { Duration::from_millis(525) - update }; - assert!(diff <= Duration::from_millis(20)); + assert!(diff <= Duration::from_millis(40)); // `ProgressReporter` should ignore 10 messages in each loop. assert_eq!(event.progress.0, counter * 10); diff --git a/tests/readme.md b/tests/readme.md index 84919fcc7d..02f4742e73 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -11,7 +11,7 @@ To run all tests including framework tests and unit tests (but without market te `cargo test --workspace --features framework-test` To run only framework tests use command: -`cargo test --test '*' --features framework-test` +`cargo test --test '*' -p yagna -p ya-exe-unit -p ya-transfer --features framework-test` ## Creating tests From 50ca348a6a9f08e0c3526e1e5167c17d732ebf8e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 19:46:14 +0100 Subject: [PATCH 035/125] Fix ya-transfer features of dependencies --- exe-unit/components/transfer/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 468b6dd483..97ecf9a6f9 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] ya-client-model = "0.6" -ya-core-model = { version = "^0.9" } +ya-core-model = { version = "0.9", features = ["activity"] } ya-service-bus = "0.6.1" ya-utils-path = { version ="0.1", path = "../../../utils/path" } ya-utils-futures = { version = "0.3", path = "../../../utils/futures" } @@ -42,7 +42,7 @@ tokio-util = { version = "0.7", features = ["io"] } url = "2.1.1" walkdir = "2.3.1" async-trait = "0.1.74" -tokio-stream = "0.1.14" +tokio-stream = { version = "0.1.14", features = ["sync"] } [target.'cfg(target_family = "unix")'.dependencies] awc = { version = "3", features = ["openssl"] } From d7f3df0732c1a2ed020cc8632f17c379eb05b13a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 29 Jan 2024 19:52:42 +0100 Subject: [PATCH 036/125] New clippy fixes --- exe-unit/tests/test_progress.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index f27a950669..dd9f039128 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -29,13 +29,13 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let image_repo = temp_dir.join("images"); let chunk_size = 4096_usize; - let chunk_count = 1024 * 1; + let chunk_count = 1024 * 2; let file_size = (chunk_size * chunk_count) as u64; let hash = generate_image(&image_repo, "image-big", chunk_size, chunk_count); let package = format!( "hash://sha3:{}:http://127.0.0.1:8001/image-big", - hex::encode(&hash) + hex::encode(hash) ); start_http(ctx, image_repo.clone()) .await @@ -99,7 +99,7 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu // Otherwise we won't get any progress message, because it is too fast. from: format!( "hash://sha3:{}:http://127.0.0.1:8001/image-big", - hex::encode(&hash) + hex::encode(hash) ), to: "container:/input/image-copy".to_string(), }], From 63eb34bf23f61d65363e2ced89b194be85d6b964 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jan 2024 15:33:31 +0100 Subject: [PATCH 037/125] Extend time periods in progress tests --- exe-unit/components/transfer/src/progress.rs | 14 +++++++------- exe-unit/tests/test_progress.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 332c5d6d3b..7c9d1ba2b9 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -216,7 +216,7 @@ mod tests { Some(ProgressConfig { progress: tx, progress_args: ProgressArgs { - update_interval: Some("500ms".parse::().unwrap()), + update_interval: Some("1s".parse::().unwrap()), update_step: None, }, }), @@ -228,11 +228,11 @@ mod tests { let mut before = Instant::now(); tokio::task::spawn_local(async move { for step in 0..2 { - tokio::time::sleep(Duration::from_millis(25)).await; + tokio::time::sleep(Duration::from_millis(50)).await; for i in 0..=size { report.report_progress(i, Some(size)); - tokio::time::sleep(Duration::from_millis(50)).await; + tokio::time::sleep(Duration::from_millis(100)).await; } if step == 0 { report.next_step(); @@ -249,13 +249,13 @@ mod tests { counter += 1; let update = Instant::now().duration_since(before); before = Instant::now(); - let diff = if update > Duration::from_millis(525) { - update - Duration::from_millis(525) + let diff = if update > Duration::from_millis(1050) { + update - Duration::from_millis(1050) } else { - Duration::from_millis(525) - update + Duration::from_millis(1050) - update }; - assert!(diff <= Duration::from_millis(40)); + assert!(diff <= Duration::from_millis(60)); // `ProgressReporter` should ignore 10 messages in each loop. assert_eq!(event.progress.0, counter * 10); diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index dd9f039128..7dc6b1dc39 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -22,7 +22,7 @@ use ya_service_bus::typed as bus; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(false); + enable_logs(true); let dir = temp_dir!("progress-reporting")?; let temp_dir = dir.path(); From 80277a41ff6957f8fb53301828e041b4fd4e352e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jan 2024 15:42:40 +0100 Subject: [PATCH 038/125] Fix ExeUnit binary compilation with packet-trace flag --- exe-unit/components/transfer/tests/test_deploy.rs | 2 +- exe-unit/src/bin.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/exe-unit/components/transfer/tests/test_deploy.rs b/exe-unit/components/transfer/tests/test_deploy.rs index 19e15078d2..81345ab00a 100644 --- a/exe-unit/components/transfer/tests/test_deploy.rs +++ b/exe-unit/components/transfer/tests/test_deploy.rs @@ -97,7 +97,7 @@ async fn test_deploy_progress(ctx: &mut DroppableTestContext) -> anyhow::Result< } let chunk_size = 4096_usize; - let chunk_count = 1024 * 1; + let chunk_count = 1024; let file_size = (chunk_size * chunk_count) as u64; let hash = generate_random_file_with_hash(temp_dir, "rnd", chunk_size, chunk_count); diff --git a/exe-unit/src/bin.rs b/exe-unit/src/bin.rs index 4dc9e0b7bc..95fb3e5822 100644 --- a/exe-unit/src/bin.rs +++ b/exe-unit/src/bin.rs @@ -27,8 +27,11 @@ async fn main() { }; dotenv::dotenv().ok(); + #[cfg(feature = "packet-trace-enable")] - init_packet_trace()?; + if let Err(error) = init_packet_trace() { + log::warn!("Initializing packet tracing failed: {error:?}"); + } let cli: Cli = Cli::from_args(); From ab7b65f6f4c3dbdcb2d9dd2e0437b0fb91210445 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 1 Feb 2024 17:38:29 +0100 Subject: [PATCH 039/125] Better error message from gftp --- core/gftp/src/rpc.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/gftp/src/rpc.rs b/core/gftp/src/rpc.rs index afbccf377f..948816d45c 100644 --- a/core/gftp/src/rpc.rs +++ b/core/gftp/src/rpc.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use std::fmt::Display; use std::io::Write; use std::path::PathBuf; use structopt::StructOpt; @@ -24,9 +25,9 @@ pub enum JsonRpcError { ServerError = -32000, } -impl ToString for JsonRpcError { - fn to_string(&self) -> String { - format!("{:?}", self) +impl Display for JsonRpcError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", format!("{:?}", self)) } } @@ -89,8 +90,8 @@ impl RpcMessage { Self::response(id, RpcResult::Files(items)) } - pub fn error + ToString>(id: Option<&RpcId>, err: E) -> Self { - let message = err.to_string(); + pub fn error + std::fmt::Display>(id: Option<&RpcId>, err: E) -> Self { + let message = format!("{err:#}"); RpcMessage { jsonrpc: JSON_RPC_VERSION.to_string(), id: id.cloned(), From 26d7d53b7d6bba696aeb7bb84595d46959c5d43b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 1 Feb 2024 17:48:41 +0100 Subject: [PATCH 040/125] Fix clippy --- core/gftp/src/rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/gftp/src/rpc.rs b/core/gftp/src/rpc.rs index 948816d45c..2c1b4f9f12 100644 --- a/core/gftp/src/rpc.rs +++ b/core/gftp/src/rpc.rs @@ -27,7 +27,7 @@ pub enum JsonRpcError { impl Display for JsonRpcError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", format!("{:?}", self)) + write!(f, "{self:?}") } } From 26c0fca201de8cb18020706829052a3439a4c22e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 5 Feb 2024 18:05:40 +0100 Subject: [PATCH 041/125] Increase progress test duration to fix windows tests --- exe-unit/components/transfer/src/progress.rs | 26 ++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/exe-unit/components/transfer/src/progress.rs b/exe-unit/components/transfer/src/progress.rs index 7a9ffb4b4e..d7e6cf155c 100644 --- a/exe-unit/components/transfer/src/progress.rs +++ b/exe-unit/components/transfer/src/progress.rs @@ -210,13 +210,25 @@ mod tests { #[actix_rt::test] async fn test_progress_reporter_interval() { + // Note: This test is time dependent and you can expect it to fail on very slow machines. + // If this happens, you could scale intervals by increasing `loop_interval`. Rather you + // shouldn't touch relations between these variables, if you don't know what are you doing. + let loop_interval = 200u64; + let update_interval = 10 * loop_interval; + let offset = loop_interval / 2; + let margin = loop_interval * 8 / 10; + let mut report = ProgressReporter::default(); let (tx, mut rx) = tokio::sync::broadcast::channel(10); report.register_reporter( Some(ProgressConfig { progress: tx, progress_args: ProgressArgs { - update_interval: Some("1s".parse::().unwrap()), + update_interval: Some( + format!("{update_interval}ms") + .parse::() + .unwrap(), + ), update_step: None, }, }), @@ -228,11 +240,11 @@ mod tests { let mut before = Instant::now(); tokio::task::spawn_local(async move { for step in 0..2 { - tokio::time::sleep(Duration::from_millis(50)).await; + tokio::time::sleep(Duration::from_millis(offset)).await; for i in 0..=size { report.report_progress(i, Some(size)); - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(loop_interval)).await; } if step == 0 { report.next_step(); @@ -249,13 +261,13 @@ mod tests { counter += 1; let update = Instant::now().duration_since(before); before = Instant::now(); - let diff = if update > Duration::from_millis(1050) { - update - Duration::from_millis(1050) + let diff = if update > Duration::from_millis(update_interval + offset) { + update - Duration::from_millis(update_interval + offset) } else { - Duration::from_millis(1050) - update + Duration::from_millis(update_interval + offset) - update }; - assert!(diff <= Duration::from_millis(80)); + assert!(diff <= Duration::from_millis(margin)); // `ProgressReporter` should ignore 10 messages in each loop. assert_eq!(event.progress.0, counter * 10); From 36f659a6ce30d67e95acb366cb0846b38362bd85 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Feb 2024 16:23:31 +0100 Subject: [PATCH 042/125] Deploy larger file in test so we will get at least 1 progress event --- exe-unit/tests/test_progress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index 7dc6b1dc39..a815134b39 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -29,7 +29,7 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu let image_repo = temp_dir.join("images"); let chunk_size = 4096_usize; - let chunk_count = 1024 * 2; + let chunk_count = 1024 * 20; let file_size = (chunk_size * chunk_count) as u64; let hash = generate_image(&image_repo, "image-big", chunk_size, chunk_count); From fac0b8eb3683f21865bbaef09d0839f3dd32936d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Feb 2024 17:18:14 +0100 Subject: [PATCH 043/125] Add Command progress report capability to docs --- docs/provider/capabilities.md | 21 ++++++++++++++------- docs/provider/exe-unit/command-progress.md | 2 ++ 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 docs/provider/exe-unit/command-progress.md diff --git a/docs/provider/capabilities.md b/docs/provider/capabilities.md index e5ddaf928e..7e618ab5ee 100644 --- a/docs/provider/capabilities.md +++ b/docs/provider/capabilities.md @@ -2,10 +2,17 @@ ## Protocol -| Capability | Yagna package version | Backwards-compatible? | Description | -|-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use (which?) property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | -| Restart Proposal Negotiations | 0.7.0 | Yes | Agent is allowed to restart negotiations, by sending `Counter Proposal`, after he rejected Proposal at some point. Counter-party will receive regular `ProposalEvent` in this case. Only Agent rejecting Proposal has initiative in restarting negotiations, rejected Agent can only wait for this to happen. To indicate, that Proposal rejection isn't final and negotiations can be restarted later, Agent can set `golem.proposal.rejection.is-final` (bool) field in `Reason` structure. If this value is set to false, Agent can free any state related to this negotiation. The same field can be set in `Reason` sent in `Reject Agreement` operation. Requestor can send new counter Proposal after some period of time or propose the same Agreement for the second time. (No change to specification) | -| manifest-support | | | TODO | -| inet | | | TODO | -| start-entrypoint | | | TODO | +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------| +| Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use `golem.srv.caps.multi-activity` property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | golem.srv.caps.multi-activity | +| Restart Proposal Negotiations | 0.7.0 | Yes | Agent is allowed to restart negotiations, by sending `Counter Proposal`, after he rejected Proposal at some point. Counter-party will receive regular `ProposalEvent` in this case. Only Agent rejecting Proposal has initiative in restarting negotiations, rejected Agent can only wait for this to happen. To indicate, that Proposal rejection isn't final and negotiations can be restarted later, Agent can set `golem.proposal.rejection.is-final` (bool) field in `Reason` structure. If this value is set to false, Agent can free any state related to this negotiation. The same field can be set in `Reason` sent in `Reject Agreement` operation. Requestor can send new counter Proposal after some period of time or propose the same Agreement for the second time. (No change to specification) | | + + +## ExeUnit + +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------|----------------------------------------------| +| manifest-support | | | TODO | | +| inet | | | TODO | | +| start-entrypoint | | | TODO | | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | golem.activity.caps.transfer.report-progress | diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md new file mode 100644 index 0000000000..6850256211 --- /dev/null +++ b/docs/provider/exe-unit/command-progress.md @@ -0,0 +1,2 @@ +# Command progress reporting + From 9a11470ee6db42636c952000f88469590519abb6 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 14:55:18 +0100 Subject: [PATCH 044/125] Update capabilities for deploy --- docs/provider/capabilities.md | 14 ++++++++------ docs/provider/exe-unit/command-progress.md | 2 -- exe-unit/src/exe_unit.rs | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) delete mode 100644 docs/provider/exe-unit/command-progress.md diff --git a/docs/provider/capabilities.md b/docs/provider/capabilities.md index 7e618ab5ee..e60d8f00d1 100644 --- a/docs/provider/capabilities.md +++ b/docs/provider/capabilities.md @@ -2,6 +2,7 @@ ## Protocol + | Capability | Yagna package version | Backwards-compatible? | Description | Property | |-------------------------------|-----------------------|-----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------| | Multi-Activity Agreement | 0.5.0 | Yes | Negotiate ability to create multiple activities under single Agreement. Use `golem.srv.caps.multi-activity` property in Demand/Offer to indicate node's support for Multi-Activity. If counterparty does not support Multi-Activity, the node falls back to single Activity per Agreement behaviour. | golem.srv.caps.multi-activity | @@ -10,9 +11,10 @@ ## ExeUnit -| Capability | Yagna package version | Backwards-compatible? | Description | Property | -|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------|----------------------------------------------| -| manifest-support | | | TODO | | -| inet | | | TODO | | -| start-entrypoint | | | TODO | | -| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | golem.activity.caps.transfer.report-progress | + +| Capability | Yagna package version | Backwards-compatible? | Description | Property | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------| +| manifest-support | | | TODO | | +| inet | | | TODO | | +| start-entrypoint | | | TODO | | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | `golem.activity.caps.transfer.report-progress` `golem.activity.caps.deploy.report-progress` | diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md deleted file mode 100644 index 6850256211..0000000000 --- a/docs/provider/exe-unit/command-progress.md +++ /dev/null @@ -1,2 +0,0 @@ -# Command progress reporting - diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index 10f97c967d..086d33fc43 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -86,6 +86,7 @@ impl ExeUnit { "golem.com.usage.vector": MetricsService::usage_vector(), "golem.activity.caps.transfer.protocol": TransferService::schemes(), "golem.activity.caps.transfer.report-progress": true, + "golem.activity.caps.deploy.report-progress": true, })); Ok(supervisor_template.patch(runtime_template)) From 41b5317a00d1375c18b2a67d489688fe943316b3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 17:04:15 +0100 Subject: [PATCH 045/125] Document Exeunit progress reporting behavior --- docs/provider/exe-unit/command-progress.md | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docs/provider/exe-unit/command-progress.md diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md new file mode 100644 index 0000000000..99a77a147d --- /dev/null +++ b/docs/provider/exe-unit/command-progress.md @@ -0,0 +1,42 @@ +# Command progress reporting + +ExeUnit behaves according to specification defined [here](https://golemfactory.github.io/golem-architecture/specs/command-progress.html) +and support progress reporting for commands: `deploy` and `transfer`. + +This document aims to describe implementation details not covered by specification. + +## Specification + + +| Name | Description | +|-----------------------------|----------------------| +| Minimum ExeUnit version | {TODO} | +| Minimum Runtime API version | Always compatible | +| Minimum yagna version | {TODO} | +| Minimum provider version | Always compatible | +| Supported commands | `deploy`, `transfer` | + +## Runtime event + +### Steps + +`Deploy` and `transfer` command consist of only single step. + +### Progress + +- Progress is reported as `Bytes`. Fields is never a `None`. +- Size of file is always checked and put as second element of tuple. +- Initially `Size` element of tuple is set to `None` and if progress with `message` field is sent + than it can be received by Requestor agent + +### Message + +Two messages are currently possible: +- `Deployed image from cache` +- `Retry in {}s because of error: {err}` - indicates error during transfer, which will result in retry. + +When sending message, the rest of `CommandProgress` structure fields will be set to latest values. + +## Requestor Example + +PoC implementation using yapapi: https://github.com/golemfactory/yapapi/pull/1153 From edc2b042efd9ebd96c8b232d0def05c01d1c9a2d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 17:40:29 +0100 Subject: [PATCH 046/125] Describe ProgressArgs behavior --- docs/provider/exe-unit/command-progress.md | 9 ++++++++- docs/yagna/capabilities.md | 7 ++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md index 99a77a147d..2a14634f0c 100644 --- a/docs/provider/exe-unit/command-progress.md +++ b/docs/provider/exe-unit/command-progress.md @@ -16,7 +16,14 @@ This document aims to describe implementation details not covered by specificati | Minimum provider version | Always compatible | | Supported commands | `deploy`, `transfer` | -## Runtime event + +## [ProgressArgs](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#/model-ProgressArgs) + +ExeUnit supports only `update-interval`. If value is set to `1s` default will be used. + +`update-step` is not implemented. + +## [Runtime event](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#model-RuntimeEventKindProgress) ### Steps diff --git a/docs/yagna/capabilities.md b/docs/yagna/capabilities.md index 2844de7103..fd867bb1ec 100644 --- a/docs/yagna/capabilities.md +++ b/docs/yagna/capabilities.md @@ -6,6 +6,7 @@ Capabilities requiring Provider agent support are listed [here](../provider/capa ## Yagna API -| Capability | Yagna package version | Backwards-compatible? | Description | -|-------------|-----------------------|-----------------------|----------------------------------------------------------------------------| -| Cors Policy | 0.12.0 | Yes | Yagna is able to respond with Cors headers. [Spec](./capabilities/cors.md) | +| Capability | Yagna package version | Backwards-compatible? | Description | +|----------------------------|-----------------------|-----------------------|---------------------------------------------------------------------------------------------------| +| Cors Policy | 0.12.0 | Yes | Yagna is able to respond with Cors headers. [Spec](./capabilities/cors.md) | +| Command progress reporting | (not released) | Yes | ExeUnit can report progress of executed commands. [Specification](./exe-unit/command-progress.md) | \ No newline at end of file From a0aac51a004e8c310645501d1cd5db0bad75be1a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Feb 2024 17:42:49 +0100 Subject: [PATCH 047/125] Fix documentation --- docs/provider/exe-unit/command-progress.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provider/exe-unit/command-progress.md b/docs/provider/exe-unit/command-progress.md index 2a14634f0c..6fe412c985 100644 --- a/docs/provider/exe-unit/command-progress.md +++ b/docs/provider/exe-unit/command-progress.md @@ -19,7 +19,7 @@ This document aims to describe implementation details not covered by specificati ## [ProgressArgs](https://golemfactory.github.io/ya-client/index.html?urls.primaryName=Activity%20API#/model-ProgressArgs) -ExeUnit supports only `update-interval`. If value is set to `1s` default will be used. +ExeUnit supports only `update-interval`. If value is not set, `1s` default will be used. `update-step` is not implemented. From 9bfdd8984ffa02abf1e2fe34e1fa8a6f3dc97331 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 14 Feb 2024 13:56:27 +0100 Subject: [PATCH 048/125] Fix deb-build action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0199295814..f75c268bab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -269,7 +269,7 @@ jobs: VERSION=${TAGV_NAME#v} echo "::set-output name=tagv::${TAG_NAME}" echo "::set-output name=version::${VERSION}" - - uses: golemfactory/build-deb-action@main + - uses: golemfactory/build-deb-action@latest id: deb with: debVersion: ${{ steps.version.outputs.version }} From e39661c1db14dcb5cba20f2d09e8b8187af8d75e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 14 Feb 2024 14:39:48 +0100 Subject: [PATCH 049/125] Reference build-deb-action@v5 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f75c268bab..07508b6e79 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -269,7 +269,7 @@ jobs: VERSION=${TAGV_NAME#v} echo "::set-output name=tagv::${TAG_NAME}" echo "::set-output name=version::${VERSION}" - - uses: golemfactory/build-deb-action@latest + - uses: golemfactory/build-deb-action@v5 id: deb with: debVersion: ${{ steps.version.outputs.version }} From 7096e036b9614f4e5124d75d7ca2f35a234355b8 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 23 Feb 2024 14:17:39 +0100 Subject: [PATCH 050/125] Try to build documents structure --- docs/readme.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/readme.md b/docs/readme.md index 3de0b8ae24..7407a538c0 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,4 +1,23 @@ # Developer documentation -[Provider](./provider/architecture.md) -[Payment Driver](./../core/payment-driver/erc20/Readme.md) \ No newline at end of file +- [Architecture](https://golemfactory.github.io/golem-architecture/) +- [REST API specification](https://golemfactory.github.io/ya-client/) +- Developer guides + - [Installation](./provider/overview.md#installation) + - [Logging guidelines](./logging-guidelines.md) +- Implementation documentation + - Overview + - [Provider](./provider/architecture.md) + - [ExeUnit](./provider/exe-unit/exe-units.md) + - Yagna + - Identity + - Market + - Activity + - Payments + - [Payment Driver](./../core/payment-driver/erc20/Readme.md) + - Net + - GSB + - VPN + - Runtimes + - ya-runtime-vm + - ya-runtime-wasm \ No newline at end of file From 140ff6e621d36e593364a8260dd0dbd980730abb Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 25 Jul 2024 21:45:15 +0200 Subject: [PATCH 051/125] Example test using payment mock --- Cargo.lock | 13 +- Cargo.toml | 2 +- core/market/Cargo.toml | 2 +- core/market/src/testing.rs | 2 - core/market/src/testing/mock_node.rs | 14 ++- core/payment/Cargo.toml | 6 + core/payment/tests/mock_identity.rs | 35 ++++++ core/payment/tests/mock_payment.rs | 114 ++++++++++++++++++ core/payment/tests/test_happy_path.rs | 28 +++++ .../test-framework/framework-basic/Cargo.toml | 6 +- .../test-framework/framework-basic/src/lib.rs | 1 + .../framework-basic/src/mocks.rs | 2 + .../framework-basic/src/mocks}/bcast.rs | 0 .../src/mocks}/bcast/singleton.rs | 0 .../framework-basic/src/mocks/net.rs | 0 15 files changed, 211 insertions(+), 14 deletions(-) create mode 100644 core/payment/tests/mock_identity.rs create mode 100644 core/payment/tests/mock_payment.rs create mode 100644 core/payment/tests/test_happy_path.rs create mode 100644 test-utils/test-framework/framework-basic/src/mocks.rs rename {core/market/src/testing => test-utils/test-framework/framework-basic/src/mocks}/bcast.rs (100%) rename {core/market/src/testing => test-utils/test-framework/framework-basic/src/mocks}/bcast/singleton.rs (100%) rename core/market/src/testing/mock_net.rs => test-utils/test-framework/framework-basic/src/mocks/net.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index a728aefdbc..d7fcae39dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1827,15 +1827,15 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.17" +version = "0.99.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "convert_case 0.4.0", "proc-macro2", "quote", "rustc_version", - "syn 1.0.109", + "syn 2.0.60", ] [[package]] @@ -9037,6 +9037,7 @@ dependencies = [ "tokio-util", "url", "walkdir", + "ya-client", "ya-client-model", "ya-core-model", "ya-service-bus", @@ -9265,6 +9266,7 @@ dependencies = [ "ya-client", "ya-core-model", "ya-diesel-utils", + "ya-framework-basic", "ya-market-resolver", "ya-net", "ya-persistence", @@ -9390,6 +9392,7 @@ dependencies = [ "base64 0.12.3", "bigdecimal 0.2.2", "chrono", + "derive_more", "diesel", "diesel_migrations", "dotenv", @@ -9411,8 +9414,10 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", "structopt", "strum 0.24.1", + "test-context", "thiserror", "tokio", "tracing", @@ -9424,6 +9429,8 @@ dependencies = [ "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", + "ya-framework-basic", + "ya-identity", "ya-metrics", "ya-net", "ya-persistence", diff --git a/Cargo.toml b/Cargo.toml index 90a06e8e9e..7633d67f93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ static-openssl = ["openssl/vendored", "openssl-probe"] dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] tos = [] -framework-test = ['ya-exe-unit/framework-test'] +framework-test = ['ya-exe-unit/framework-test', 'ya-payment/framework-test'] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index c33661927f..e19e1acef9 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -7,13 +7,13 @@ edition = "2018" [features] test-suite = [] -bcast-singleton = [] [dependencies] ya-agreement-utils = { workspace = true } ya-client = "0.8" ya-core-model = { version = "^0.9", features = ["market", "net"] } ya-diesel-utils = { version = "0.1" } +ya-framework-basic = "0.1" ya-market-resolver = "0.2" ya-net = "0.3" ya-persistence = "0.3" diff --git a/core/market/src/testing.rs b/core/market/src/testing.rs index c1f61d0e14..e6f44271ad 100644 --- a/core/market/src/testing.rs +++ b/core/market/src/testing.rs @@ -10,12 +10,10 @@ pub use super::negotiation::{error::*, ApprovalStatus}; pub use super::protocol::*; pub mod agreement_utils; -pub mod bcast; pub mod dao; pub mod events_helper; pub mod mock_agreement; pub mod mock_identity; -pub mod mock_net; pub mod mock_node; pub mod mock_offer; pub mod proposal_util; diff --git a/core/market/src/testing/mock_node.rs b/core/market/src/testing/mock_node.rs index 3768b4c3ef..4901d1f3f3 100644 --- a/core/market/src/testing/mock_node.rs +++ b/core/market/src/testing/mock_node.rs @@ -15,12 +15,6 @@ use ya_service_api_web::middleware::{auth::dummy::DummyAuth, Identity}; use crate::MarketService; -#[cfg(feature = "bcast-singleton")] -use super::bcast::singleton::BCastService; -use super::bcast::BCast; -#[cfg(not(feature = "bcast-singleton"))] -use super::bcast::BCastService; -use super::mock_net::{gsb_prefixes, MockNet}; use super::negotiation::{provider, requestor}; use super::{store::SubscriptionStore, Matcher}; use crate::config::{Config, DiscoveryConfig}; @@ -38,6 +32,14 @@ use crate::protocol::negotiation::messages::*; use crate::testing::mock_identity::MockIdentity; use crate::testing::mock_node::default::*; +#[cfg(feature = "bcast-singleton")] +use ya_framework_basic::bcast::singleton::BCastService; +use ya_framework_basic::mocks::net::{gsb_prefixes, MockNet}; + +use ya_framework_basic::mocks::bcast::BCast; +#[cfg(not(feature = "bcast-singleton"))] +use ya_framework_basic::mocks::bcast::BCastService; + /// Instantiates market test nodes inside one process. pub struct MarketsNetwork { nodes: Vec, diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index 60be62a540..73cbf535c7 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [features] default = [] +framework-test = [] [dependencies] ya-agreement-utils = { workspace = true } @@ -30,6 +31,7 @@ anyhow = "1.0" base64 = "0.12" bigdecimal = "0.2" chrono = { version = "0.4", features = ["serde"] } +derive_more = "0.99.18" diesel = { version = "1.4", features = [ "sqlite", "r2d2", @@ -67,11 +69,15 @@ ya-client = "0.8" ya-dummy-driver = "0.3" ya-erc20-driver = "0.4" ya-net = { version = "0.3", features = ["service"] } +ya-identity = "0.3" ya-sb-router = { workspace = true } +ya-framework-basic = { version = "0.1" } actix-rt = "2.7" rand = "0.8" ethsign = "0.8" +serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"] } +test-context = "0.1.4" [lints] workspace = true diff --git a/core/payment/tests/mock_identity.rs b/core/payment/tests/mock_identity.rs new file mode 100644 index 0000000000..cd16c44432 --- /dev/null +++ b/core/payment/tests/mock_identity.rs @@ -0,0 +1,35 @@ +#![allow(dead_code)] + +use anyhow::anyhow; + +use ya_identity::service::Identity; +use ya_persistence::executor::DbExecutor; + +#[derive(Clone)] +pub struct MockIdentity { + name: String, + db: DbExecutor, +} + +impl MockIdentity { + pub fn new(name: &str) -> Self { + let db = Self::create_db(&format!("{name}.identity.db")).unwrap(); + + MockIdentity { + name: name.to_string(), + db, + } + } + + fn create_db(name: &str) -> anyhow::Result { + let db = DbExecutor::in_memory(name) + .map_err(|e| anyhow!("Failed to create in memory db [{name:?}]. Error: {e}"))?; + Ok(db) + } + + pub async fn bind_gsb(&self) -> anyhow::Result<()> { + log::info!("MockIdentity - binding GSB"); + Identity::gsb(&self.db).await?; + Ok(()) + } +} diff --git a/core/payment/tests/mock_payment.rs b/core/payment/tests/mock_payment.rs new file mode 100644 index 0000000000..42ef71b94d --- /dev/null +++ b/core/payment/tests/mock_payment.rs @@ -0,0 +1,114 @@ +#![allow(dead_code)] + +use actix_web::web::Data; +use actix_web::{middleware, App, HttpServer}; +use anyhow::anyhow; +use std::sync::Arc; + +use ya_core_model::driver::{driver_bus_id, Fund}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_payment::api::web_scope; +use ya_payment::migrations; +use ya_payment::processor::PaymentProcessor; +use ya_payment::service::BindOptions; +use ya_persistence::executor::DbExecutor; +use ya_service_bus::typed as bus; + +use ya_dummy_driver as dummy; +use ya_erc20_driver as erc20; + +#[derive(Clone, Debug, derive_more::Display)] +pub enum Driver { + #[display(fmt = "dummy")] + Dummy, + #[display(fmt = "erc20")] + Erc20, +} + +impl Driver { + pub fn gsb_name(&self) -> String { + match self { + Driver::Dummy => dummy::DRIVER_NAME.to_string(), + Driver::Erc20 => erc20::DRIVER_NAME.to_string(), + } + } +} + +#[derive(Clone)] +pub struct MockPayment { + name: String, + db: DbExecutor, + processor: Arc, +} + +impl MockPayment { + pub fn new(name: &str) -> Self { + let db = Self::create_db(&format!("{name}.identity.db")).unwrap(); + let processor = Arc::new(PaymentProcessor::new(db.clone())); + + MockPayment { + name: name.to_string(), + db, + processor, + } + } + + fn create_db(name: &str) -> anyhow::Result { + let db = DbExecutor::in_memory(name) + .map_err(|e| anyhow!("Failed to create in memory db [{name:?}]. Error: {e}"))?; + db.apply_migration(migrations::run_with_output)?; + Ok(db) + } + + pub async fn bind_gsb(&self) -> anyhow::Result<()> { + log::info!("MockPayment - binding GSB"); + + ya_payment::service::bind_service( + &self.db, + self.processor.clone(), + BindOptions::default().run_sync_job(false), + ); + Ok(()) + } + + pub async fn start_server( + &self, + ctx: &mut DroppableTestContext, + address: &str, + ) -> anyhow::Result<()> { + let db = self.db.clone(); + let srv = HttpServer::new(move || { + App::new() + .wrap(middleware::Logger::default()) + .app_data(Data::new(db.clone())) + .service(web_scope(&db)) + }) + .bind(address)? + .run(); + + ctx.register(srv.handle()); + tokio::task::spawn_local(async move { anyhow::Ok(srv.await?) }); + + Ok(()) + } + + pub async fn start_dummy_driver() -> anyhow::Result<()> { + dummy::PaymentDriverService::gsb(&()).await?; + Ok(()) + } + + pub async fn start_erc20_driver() -> anyhow::Result<()> { + Ok(()) + } + + pub async fn fund_account(&self, driver: Driver, address: &str) -> anyhow::Result<()> { + bus::service(driver_bus_id(driver.gsb_name())) + .call(Fund::new( + address.to_string(), + Some("holesky".to_string()), + None, + )) + .await??; + Ok(()) + } +} diff --git a/core/payment/tests/test_happy_path.rs b/core/payment/tests/test_happy_path.rs new file mode 100644 index 0000000000..f23f406df7 --- /dev/null +++ b/core/payment/tests/test_happy_path.rs @@ -0,0 +1,28 @@ +use test_context::test_context; + +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::temp_dir; + +use mock_payment::MockPayment; + +mod mock_identity; +mod mock_payment; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_payments_happy_path(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let _dir = temp_dir!("test_payments_happy_path")?; + + ya_sb_router::bind_gsb_router(None).await?; + log::debug!("bind_gsb_router()"); + + let payment = MockPayment::new("payments-1"); + payment.bind_gsb().await?; + payment.start_server(ctx, "127.0.0.1:8000").await?; + + Ok(()) +} diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index a4a06580f6..b5dbe8dfe9 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -5,11 +5,15 @@ description = "Yagna test framework basic utilities" authors = ["Golem Factory "] edition = "2018" +[features] +bcast-singleton = [] + [dependencies] ya-client-model = "0.6" +ya-client = "0.8" ya-core-model = { version = "^0.9" } ya-service-bus = { workspace = true } -ya-utils-path = { version ="0.1", path = "../../../utils/path" } +ya-utils-path = { version = "0.1", path = "../../../utils/path" } ya-utils-futures.workspace = true gftp = { workspace = true } diff --git a/test-utils/test-framework/framework-basic/src/lib.rs b/test-utils/test-framework/framework-basic/src/lib.rs index 46cf07ab6b..9e5077a628 100644 --- a/test-utils/test-framework/framework-basic/src/lib.rs +++ b/test-utils/test-framework/framework-basic/src/lib.rs @@ -2,6 +2,7 @@ pub mod async_drop; pub mod file; pub mod hash; pub mod log; +pub mod mocks; pub mod server_external; pub mod temp; pub mod template; diff --git a/test-utils/test-framework/framework-basic/src/mocks.rs b/test-utils/test-framework/framework-basic/src/mocks.rs new file mode 100644 index 0000000000..ea422ac3e1 --- /dev/null +++ b/test-utils/test-framework/framework-basic/src/mocks.rs @@ -0,0 +1,2 @@ +pub mod bcast; +pub mod net; diff --git a/core/market/src/testing/bcast.rs b/test-utils/test-framework/framework-basic/src/mocks/bcast.rs similarity index 100% rename from core/market/src/testing/bcast.rs rename to test-utils/test-framework/framework-basic/src/mocks/bcast.rs diff --git a/core/market/src/testing/bcast/singleton.rs b/test-utils/test-framework/framework-basic/src/mocks/bcast/singleton.rs similarity index 100% rename from core/market/src/testing/bcast/singleton.rs rename to test-utils/test-framework/framework-basic/src/mocks/bcast/singleton.rs diff --git a/core/market/src/testing/mock_net.rs b/test-utils/test-framework/framework-basic/src/mocks/net.rs similarity index 100% rename from core/market/src/testing/mock_net.rs rename to test-utils/test-framework/framework-basic/src/mocks/net.rs From 413a3edb23d6aaa9d78fffb9f17cdb7ff73a5b52 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jul 2024 14:54:01 +0200 Subject: [PATCH 052/125] Create MockNode abstraction to simplify initialization --- Cargo.lock | 11 ++ core/payment/Cargo.toml | 2 + .../{mock_identity.rs => mocks/identity.rs} | 2 +- core/payment/tests/mocks/mod.rs | 3 + core/payment/tests/mocks/node.rs | 107 ++++++++++++++++++ .../{mock_payment.rs => mocks/payment.rs} | 4 +- core/payment/tests/test_happy_path.rs | 28 ----- .../tests/tutorial_how_to_use_module_tests.rs | 28 +++++ 8 files changed, 154 insertions(+), 31 deletions(-) rename core/payment/tests/{mock_identity.rs => mocks/identity.rs} (92%) create mode 100644 core/payment/tests/mocks/mod.rs create mode 100644 core/payment/tests/mocks/node.rs rename core/payment/tests/{mock_payment.rs => mocks/payment.rs} (95%) delete mode 100644 core/payment/tests/test_happy_path.rs create mode 100644 core/payment/tests/tutorial_how_to_use_module_tests.rs diff --git a/Cargo.lock b/Cargo.lock index d7fcae39dd..910e33a7bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5065,6 +5065,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +[[package]] +name = "portpicker" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be97d76faf1bfab666e1375477b23fde79eccf0276e9b63b92a39d676a889ba9" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -9409,6 +9418,7 @@ dependencies = [ "metrics 0.12.1", "num-bigint 0.3.3", "open", + "portpicker", "problem_details", "r2d2", "rand 0.8.5", @@ -9422,6 +9432,7 @@ dependencies = [ "tokio", "tracing", "uint 0.7.1", + "url", "uuid 0.8.2", "ya-agreement-utils", "ya-client", diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index 73cbf535c7..ef6280235e 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -76,8 +76,10 @@ ya-framework-basic = { version = "0.1" } actix-rt = "2.7" rand = "0.8" ethsign = "0.8" +portpicker = "0.1" serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"] } test-context = "0.1.4" +url = "2.5" [lints] workspace = true diff --git a/core/payment/tests/mock_identity.rs b/core/payment/tests/mocks/identity.rs similarity index 92% rename from core/payment/tests/mock_identity.rs rename to core/payment/tests/mocks/identity.rs index cd16c44432..71922ebf4c 100644 --- a/core/payment/tests/mock_identity.rs +++ b/core/payment/tests/mocks/identity.rs @@ -28,7 +28,7 @@ impl MockIdentity { } pub async fn bind_gsb(&self) -> anyhow::Result<()> { - log::info!("MockIdentity - binding GSB"); + log::info!("MockIdentity ({}) - binding GSB", self.name); Identity::gsb(&self.db).await?; Ok(()) } diff --git a/core/payment/tests/mocks/mod.rs b/core/payment/tests/mocks/mod.rs new file mode 100644 index 0000000000..de12f6ba25 --- /dev/null +++ b/core/payment/tests/mocks/mod.rs @@ -0,0 +1,3 @@ +pub mod identity; +pub mod node; +pub mod payment; diff --git a/core/payment/tests/mocks/node.rs b/core/payment/tests/mocks/node.rs new file mode 100644 index 0000000000..d67f05f1a4 --- /dev/null +++ b/core/payment/tests/mocks/node.rs @@ -0,0 +1,107 @@ +#![allow(dead_code)] + +use anyhow::anyhow; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use url::Url; + +use super::identity::MockIdentity; +use super::payment::MockPayment; + +/// Represents Node abstraction in tests. +/// Provides functionality to instantiate selected modules and make tests setup easier. +/// +/// TODO: Currently setup with multiple Nodes with GSB bound modules is impossible, because +/// most yagna modules bind to fixed GSB addresses and have dependencies on other modules, +/// using fixed addresses. This should be improved in the future. +#[derive(Clone)] +pub struct MockNode { + name: String, + testdir: PathBuf, + pub identity: Option, + pub payment: Option, +} + +impl MockNode { + pub fn new(name: &str, testdir: &Path) -> Self { + let testdir = testdir.join(name); + MockNode { + name: name.to_string(), + testdir, + identity: None, + payment: None, + } + } + + pub fn with_identity(mut self) -> Self { + self.identity = Some(MockIdentity::new(&self.name)); + self + } + + pub fn with_payment(mut self) -> Self { + self.payment = Some(MockPayment::new(&self.name)); + self + } + + pub fn get_identity(&self) -> anyhow::Result { + self.identity + .clone() + .ok_or_else(|| anyhow!("Identity ({}) is not initialized", self.name)) + } + + pub fn get_payment(&self) -> anyhow::Result { + self.payment + .clone() + .ok_or_else(|| anyhow!("Payment ({}) is not initialized", self.name)) + } + + pub async fn bind_gsb(&self) -> anyhow::Result<()> { + self.bind_gsb_router().await?; + + if let Some(identity) = &self.identity { + identity.bind_gsb().await?; + } + + if let Some(payment) = &self.payment { + payment.bind_gsb().await?; + } + + Ok(()) + } + + pub async fn bind_gsb_router(&self) -> anyhow::Result<()> { + let gsb_url = self.gsb_router_address()?; + + log::info!( + "MockeNode ({}) - binding GSB router at: {gsb_url}", + self.name + ); + + ya_sb_router::bind_gsb_router(Some(gsb_url.clone())) + .await + .map_err(|e| anyhow!("Error binding service bus router to '{}': {e}", &gsb_url))?; + Ok(()) + } + + fn gsb_router_address(&self) -> anyhow::Result { + let gsb_url = match std::env::consts::FAMILY { + // It would be better to create socket in self.testdir, but it's not possible, because + // unix socket path length is limited to SUN_LEN (108 bytes). + "unix" => Url::from_str(&format!("unix:///tmp/{}/gsb.sock", self.name))?, + _ => Url::from_str(&format!( + "tcp://127.0.0.1:{}", + portpicker::pick_unused_port().ok_or(anyhow!("No ports free"))? + ))?, + }; + if gsb_url.scheme() == "unix" { + let dir = PathBuf::from_str(gsb_url.path())? + .parent() + .map(|path| path.to_path_buf()) + .ok_or(anyhow!("`gsb_url` unix socket has no parent directory."))?; + fs::create_dir_all(dir)?; + } + + Ok(gsb_url) + } +} diff --git a/core/payment/tests/mock_payment.rs b/core/payment/tests/mocks/payment.rs similarity index 95% rename from core/payment/tests/mock_payment.rs rename to core/payment/tests/mocks/payment.rs index 42ef71b94d..e7cabe471e 100644 --- a/core/payment/tests/mock_payment.rs +++ b/core/payment/tests/mocks/payment.rs @@ -43,7 +43,7 @@ pub struct MockPayment { impl MockPayment { pub fn new(name: &str) -> Self { - let db = Self::create_db(&format!("{name}.identity.db")).unwrap(); + let db = Self::create_db(&format!("{name}.payment.db")).unwrap(); let processor = Arc::new(PaymentProcessor::new(db.clone())); MockPayment { @@ -61,7 +61,7 @@ impl MockPayment { } pub async fn bind_gsb(&self) -> anyhow::Result<()> { - log::info!("MockPayment - binding GSB"); + log::info!("MockPayment ({}) - binding GSB", self.name); ya_payment::service::bind_service( &self.db, diff --git a/core/payment/tests/test_happy_path.rs b/core/payment/tests/test_happy_path.rs deleted file mode 100644 index f23f406df7..0000000000 --- a/core/payment/tests/test_happy_path.rs +++ /dev/null @@ -1,28 +0,0 @@ -use test_context::test_context; - -use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::log::enable_logs; -use ya_framework_basic::temp_dir; - -use mock_payment::MockPayment; - -mod mock_identity; -mod mock_payment; - -#[cfg_attr(not(feature = "framework-test"), ignore)] -#[test_context(DroppableTestContext)] -#[serial_test::serial] -async fn test_payments_happy_path(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); - - let _dir = temp_dir!("test_payments_happy_path")?; - - ya_sb_router::bind_gsb_router(None).await?; - log::debug!("bind_gsb_router()"); - - let payment = MockPayment::new("payments-1"); - payment.bind_gsb().await?; - payment.start_server(ctx, "127.0.0.1:8000").await?; - - Ok(()) -} diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs new file mode 100644 index 0000000000..97ba3fc3ca --- /dev/null +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -0,0 +1,28 @@ +use test_context::test_context; + +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::temp_dir; + +use mocks::node::MockNode; + +mod mocks; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("tutorial_how_to_use_module_tests")?; + + let node = MockNode::new("node-1", &dir.path()) + .with_identity() + .with_payment(); + node.bind_gsb().await?; + node.get_payment()? + .start_server(ctx, "127.0.0.1:8000") + .await?; + + Ok(()) +} From f9eed0bff92b5a665ba155a7389e002fc50ca8f7 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jul 2024 17:31:20 +0200 Subject: [PATCH 053/125] Run rest server --- core/payment/tests/mocks/node.rs | 52 +++++++++++++++++++ core/payment/tests/mocks/payment.rs | 23 +------- .../tests/tutorial_how_to_use_module_tests.rs | 4 +- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/core/payment/tests/mocks/node.rs b/core/payment/tests/mocks/node.rs index d67f05f1a4..b7611aea83 100644 --- a/core/payment/tests/mocks/node.rs +++ b/core/payment/tests/mocks/node.rs @@ -1,10 +1,15 @@ #![allow(dead_code)] +use actix_web::{middleware, App, HttpServer, Scope}; use anyhow::anyhow; use std::fs; use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::time::Duration; use url::Url; +use ya_client::payment::PaymentApi; +use ya_client::web::WebClient; +use ya_framework_basic::async_drop::DroppableTestContext; use super::identity::MockIdentity; use super::payment::MockPayment; @@ -19,6 +24,9 @@ use super::payment::MockPayment; pub struct MockNode { name: String, testdir: PathBuf, + + rest_url: String, + pub identity: Option, pub payment: Option, } @@ -29,6 +37,7 @@ impl MockNode { MockNode { name: name.to_string(), testdir, + rest_url: Self::generate_rest_url(), identity: None, payment: None, } @@ -56,6 +65,8 @@ impl MockNode { .ok_or_else(|| anyhow!("Payment ({}) is not initialized", self.name)) } + /// Binds GSB router and all initialized modules to GSB. + /// If you want to bind only chosen modules, you should bind them manually. pub async fn bind_gsb(&self) -> anyhow::Result<()> { self.bind_gsb_router().await?; @@ -70,6 +81,42 @@ impl MockNode { Ok(()) } + pub fn rest_payments(&self) -> anyhow::Result { + let provider: PaymentApi = WebClient::builder() + .timeout(Duration::from_secs(600 * 60)) + .api_url(Url::parse(&self.rest_url)?) + .build() + .interface()?; + Ok(provider) + } + + pub async fn start_server(&self, ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + log::info!( + "MockeNode ({}) - Starting server: {}", + self.name, + self.rest_url + ); + + let payments = self.payment.clone(); + + let srv = HttpServer::new(move || { + App::new().wrap(middleware::Logger::default()).service( + payments + .clone() + .map(|payment| payment.bind_rest()) + .unwrap_or_else(|| Scope::new("")), + ) + }) + .bind(self.rest_url.to_string()) + .map_err(|e| anyhow!("Running actix server failed: {e}"))? + .run(); + + ctx.register(srv.handle()); + tokio::task::spawn_local(async move { anyhow::Ok(srv.await?) }); + + Ok(()) + } + pub async fn bind_gsb_router(&self) -> anyhow::Result<()> { let gsb_url = self.gsb_router_address()?; @@ -104,4 +151,9 @@ impl MockNode { Ok(gsb_url) } + + fn generate_rest_url() -> String { + let port = portpicker::pick_unused_port().expect("No ports free"); + format!("127.0.0.1:{}", port) + } } diff --git a/core/payment/tests/mocks/payment.rs b/core/payment/tests/mocks/payment.rs index e7cabe471e..35a338b16e 100644 --- a/core/payment/tests/mocks/payment.rs +++ b/core/payment/tests/mocks/payment.rs @@ -1,12 +1,9 @@ #![allow(dead_code)] -use actix_web::web::Data; -use actix_web::{middleware, App, HttpServer}; use anyhow::anyhow; use std::sync::Arc; use ya_core_model::driver::{driver_bus_id, Fund}; -use ya_framework_basic::async_drop::DroppableTestContext; use ya_payment::api::web_scope; use ya_payment::migrations; use ya_payment::processor::PaymentProcessor; @@ -71,25 +68,9 @@ impl MockPayment { Ok(()) } - pub async fn start_server( - &self, - ctx: &mut DroppableTestContext, - address: &str, - ) -> anyhow::Result<()> { + pub fn bind_rest(&self) -> actix_web::Scope { let db = self.db.clone(); - let srv = HttpServer::new(move || { - App::new() - .wrap(middleware::Logger::default()) - .app_data(Data::new(db.clone())) - .service(web_scope(&db)) - }) - .bind(address)? - .run(); - - ctx.register(srv.handle()); - tokio::task::spawn_local(async move { anyhow::Ok(srv.await?) }); - - Ok(()) + web_scope(&db) } pub async fn start_dummy_driver() -> anyhow::Result<()> { diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 97ba3fc3ca..6fe69d7439 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -20,9 +20,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any .with_identity() .with_payment(); node.bind_gsb().await?; - node.get_payment()? - .start_server(ctx, "127.0.0.1:8000") - .await?; + node.start_server(ctx).await?; Ok(()) } From 5d500ecf061ad186a8cca67009bb73500b667a53 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jul 2024 19:26:22 +0200 Subject: [PATCH 054/125] Test environment for payments: generating identity and appkey --- core/payment/tests/mocks/identity.rs | 71 ++++++++++++++++++- core/payment/tests/mocks/node.rs | 34 +++++---- .../tests/tutorial_how_to_use_module_tests.rs | 21 ++++++ utils/cli/src/cmd.rs | 1 + 4 files changed, 114 insertions(+), 13 deletions(-) diff --git a/core/payment/tests/mocks/identity.rs b/core/payment/tests/mocks/identity.rs index 71922ebf4c..8089bb3655 100644 --- a/core/payment/tests/mocks/identity.rs +++ b/core/payment/tests/mocks/identity.rs @@ -1,9 +1,15 @@ #![allow(dead_code)] -use anyhow::anyhow; +use anyhow::{anyhow, bail}; +use serde::de::DeserializeOwned; +use ya_client_model::NodeId; +use ya_core_model::appkey::AppKey; +use ya_core_model::identity::IdentityInfo; +use ya_identity::cli::{AppKeyCommand, IdentityCommand}; use ya_identity::service::Identity; use ya_persistence::executor::DbExecutor; +use ya_service_api::{CliCtx, CommandOutput}; #[derive(Clone)] pub struct MockIdentity { @@ -32,4 +38,67 @@ impl MockIdentity { Identity::gsb(&self.db).await?; Ok(()) } + + pub async fn create_identity_key(&self, name: &str) -> anyhow::Result { + let identity: IdentityInfo = self + .create_identity(name) + .await + .map_err(|e| anyhow!("Creating Identity: {e}"))?; + let appkey = self + .create_appkey(name, identity.node_id) + .await + .map_err(|e| anyhow!("Creating AppKey: {e}"))?; + + Ok(appkey) + } + + pub async fn create_identity(&self, name: &str) -> anyhow::Result { + let ctx = CliCtx::default(); + let command = IdentityCommand::Create { + no_password: true, + alias: Some(name.to_string()), + password: None, + from_keystore: None, + from_private_key: None, + }; + + Ok(parse_output_result::( + command.run_command(&ctx).await?, + )?) + } + pub async fn create_appkey(&self, name: &str, id: NodeId) -> anyhow::Result { + let ctx = CliCtx::default(); + let command = AppKeyCommand::Create { + name: name.to_string(), + role: "manager".to_string(), + id: Some(id.to_string()), + allow_origins: vec![], + }; + let _key = command.run_command(&ctx).await?; + + let output = AppKeyCommand::Show { + name: name.to_string(), + } + .run_command(&ctx) + .await?; + + Ok(parse_output::(output)?) + } +} + +fn parse_output_result(output: CommandOutput) -> anyhow::Result { + Ok(match output { + CommandOutput::Object(json) => serde_json::from_value::>(json) + .map_err(|e| anyhow!("Error parsing command response: {e}"))? + .map_err(|e| anyhow!("Command failed: {e}"))?, + _ => bail!("Unexpected output: {output:?}"), + }) +} + +fn parse_output(output: CommandOutput) -> anyhow::Result { + Ok(match output { + CommandOutput::Object(json) => serde_json::from_value::(json) + .map_err(|e| anyhow!("Error parsing command response: {e}"))?, + _ => bail!("Unexpected output: {output:?}"), + }) } diff --git a/core/payment/tests/mocks/node.rs b/core/payment/tests/mocks/node.rs index b7611aea83..058779dd1e 100644 --- a/core/payment/tests/mocks/node.rs +++ b/core/payment/tests/mocks/node.rs @@ -10,6 +10,9 @@ use url::Url; use ya_client::payment::PaymentApi; use ya_client::web::WebClient; use ya_framework_basic::async_drop::DroppableTestContext; +use ya_service_api_web::middleware::auth; +use ya_service_api_web::middleware::cors::{AppKeyCors, CorsConfig}; +use ya_service_api_web::rest_api_host_port; use super::identity::MockIdentity; use super::payment::MockPayment; @@ -25,7 +28,7 @@ pub struct MockNode { name: String, testdir: PathBuf, - rest_url: String, + rest_url: Url, pub identity: Option, pub payment: Option, @@ -81,10 +84,11 @@ impl MockNode { Ok(()) } - pub fn rest_payments(&self) -> anyhow::Result { + pub fn rest_payments(&self, token: &str) -> anyhow::Result { let provider: PaymentApi = WebClient::builder() + .auth_token(token) .timeout(Duration::from_secs(600 * 60)) - .api_url(Url::parse(&self.rest_url)?) + .api_url(self.rest_url.clone()) .build() .interface()?; Ok(provider) @@ -99,15 +103,21 @@ impl MockNode { let payments = self.payment.clone(); + let cors = AppKeyCors::new(&CorsConfig::default()).await?; + let srv = HttpServer::new(move || { - App::new().wrap(middleware::Logger::default()).service( - payments - .clone() - .map(|payment| payment.bind_rest()) - .unwrap_or_else(|| Scope::new("")), - ) + App::new() + .wrap(middleware::Logger::default()) + .wrap(auth::Auth::new(cors.cache())) + .wrap(cors.cors()) + .service( + payments + .clone() + .map(|payment| payment.bind_rest()) + .unwrap_or_else(|| Scope::new("")), + ) }) - .bind(self.rest_url.to_string()) + .bind(rest_api_host_port(self.rest_url.clone())) .map_err(|e| anyhow!("Running actix server failed: {e}"))? .run(); @@ -152,8 +162,8 @@ impl MockNode { Ok(gsb_url) } - fn generate_rest_url() -> String { + fn generate_rest_url() -> Url { let port = portpicker::pick_unused_port().expect("No ports free"); - format!("127.0.0.1:{}", port) + Url::parse(&format!("http://127.0.0.1:{}", port)).expect("Failed to parse generated URL") } } diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 6fe69d7439..0e70afa002 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -1,4 +1,7 @@ +use bigdecimal::BigDecimal; use test_context::test_context; +use ya_client_model::payment::allocation::PaymentPlatformEnum; +use ya_client_model::payment::NewAllocation; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; @@ -22,5 +25,23 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any node.bind_gsb().await?; node.start_server(ctx).await?; + let appkey = node.get_identity()?.create_identity_key("test").await?; + let api = node.rest_payments(&appkey.key)?; + + let _allocation = api + .create_allocation(&NewAllocation { + address: None, // Use default address (i.e. identity) + payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName( + "erc20-holesky-tglm".to_string(), + )), + total_amount: BigDecimal::from(10u64), + timeout: None, + make_deposit: false, + deposit: None, + extend_timeout: None, + }) + .await + .unwrap(); + Ok(()) } diff --git a/utils/cli/src/cmd.rs b/utils/cli/src/cmd.rs index 038c5c4673..4eec5de406 100644 --- a/utils/cli/src/cmd.rs +++ b/utils/cli/src/cmd.rs @@ -1,6 +1,7 @@ use anyhow::Result; use serde::Serialize; +#[derive(Debug)] pub enum CommandOutput { NoOutput, Object(serde_json::Value), From d267cd1a98e42cab1aa33ea21fbf3134304e9ca1 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 30 Jul 2024 20:02:07 +0200 Subject: [PATCH 055/125] Test environment for payments: first rest call to payment modules works --- core/payment/tests/mocks/node.rs | 7 ++++++- core/payment/tests/mocks/payment.rs | 14 +++++++++++--- .../tests/tutorial_how_to_use_module_tests.rs | 5 +++++ .../test-framework/framework-basic/src/log.rs | 4 +++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/core/payment/tests/mocks/node.rs b/core/payment/tests/mocks/node.rs index 058779dd1e..f21b986926 100644 --- a/core/payment/tests/mocks/node.rs +++ b/core/payment/tests/mocks/node.rs @@ -37,6 +37,8 @@ pub struct MockNode { impl MockNode { pub fn new(name: &str, testdir: &Path) -> Self { let testdir = testdir.join(name); + fs::create_dir_all(&testdir).expect("Failed to create test directory"); + MockNode { name: name.to_string(), testdir, @@ -52,7 +54,7 @@ impl MockNode { } pub fn with_payment(mut self) -> Self { - self.payment = Some(MockPayment::new(&self.name)); + self.payment = Some(MockPayment::new(&self.name, &self.testdir)); self } @@ -135,6 +137,9 @@ impl MockNode { self.name ); + // GSB RemoteRouter takes url from this variable and we can't set it directly. + std::env::set_var("GSB_URL", gsb_url.to_string()); + ya_sb_router::bind_gsb_router(Some(gsb_url.clone())) .await .map_err(|e| anyhow!("Error binding service bus router to '{}': {e}", &gsb_url))?; diff --git a/core/payment/tests/mocks/payment.rs b/core/payment/tests/mocks/payment.rs index 35a338b16e..06a904cfe6 100644 --- a/core/payment/tests/mocks/payment.rs +++ b/core/payment/tests/mocks/payment.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] use anyhow::anyhow; +use std::path::{Path, PathBuf}; use std::sync::Arc; use ya_core_model::driver::{driver_bus_id, Fund}; @@ -34,17 +35,20 @@ impl Driver { #[derive(Clone)] pub struct MockPayment { name: String, + testdir: PathBuf, + db: DbExecutor, processor: Arc, } impl MockPayment { - pub fn new(name: &str) -> Self { + pub fn new(name: &str, testdir: &Path) -> Self { let db = Self::create_db(&format!("{name}.payment.db")).unwrap(); let processor = Arc::new(PaymentProcessor::new(db.clone())); MockPayment { name: name.to_string(), + testdir: testdir.to_path_buf(), db, processor, } @@ -65,6 +69,9 @@ impl MockPayment { self.processor.clone(), BindOptions::default().run_sync_job(false), ); + + self.start_dummy_driver().await?; + self.start_erc20_driver().await?; Ok(()) } @@ -73,12 +80,13 @@ impl MockPayment { web_scope(&db) } - pub async fn start_dummy_driver() -> anyhow::Result<()> { + pub async fn start_dummy_driver(&self) -> anyhow::Result<()> { dummy::PaymentDriverService::gsb(&()).await?; Ok(()) } - pub async fn start_erc20_driver() -> anyhow::Result<()> { + pub async fn start_erc20_driver(&self) -> anyhow::Result<()> { + erc20::PaymentDriverService::gsb(self.testdir.clone()).await?; Ok(()) } diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 0e70afa002..c108c4c7b7 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -7,6 +7,7 @@ use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::temp_dir; +use crate::mocks::payment::Driver; use mocks::node::MockNode; mod mocks; @@ -28,6 +29,10 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any let appkey = node.get_identity()?.create_identity_key("test").await?; let api = node.rest_payments(&appkey.key)?; + node.get_payment()? + .fund_account(Driver::Erc20, &appkey.identity.to_string()) + .await?; + let _allocation = api .create_allocation(&NewAllocation { address: None, // Use default address (i.e. identity) diff --git a/test-utils/test-framework/framework-basic/src/log.rs b/test-utils/test-framework/framework-basic/src/log.rs index 410bbedfe2..4335f6c067 100644 --- a/test-utils/test-framework/framework-basic/src/log.rs +++ b/test-utils/test-framework/framework-basic/src/log.rs @@ -4,7 +4,9 @@ use std::env; pub fn enable_logs(enable: bool) { env::set_var( "RUST_LOG", - env::var("RUST_LOG").unwrap_or_else(|_| "debug".into()), + env::var("RUST_LOG").unwrap_or_else(|_| { + "info,web3=warn,sqlx_core=warn,hyper=warn,erc20_payment_lib=info,trust_dns_proto=warn,erc20_rpc_pool=info,trust_dns_resolver=warn,ya_erc20_driver=info".into() + }), ); if enable { env_logger::builder() From a6d1f14d9d0db9ae059d7142a5b4bd4445d7827d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 31 Jul 2024 16:01:32 +0200 Subject: [PATCH 056/125] Payments test environment: Implement fake market to respond on GSB --- Cargo.lock | 1 + core/payment/Cargo.toml | 1 + core/payment/tests/mocks/identity.rs | 4 +- core/payment/tests/mocks/market.rs | 243 ++++++++++++++++++ core/payment/tests/mocks/mod.rs | 1 + core/payment/tests/mocks/node.rs | 32 ++- core/payment/tests/test_release_allocation.rs | 157 +++++++++++ .../tests/tutorial_how_to_use_module_tests.rs | 16 +- 8 files changed, 450 insertions(+), 5 deletions(-) create mode 100644 core/payment/tests/mocks/market.rs create mode 100644 core/payment/tests/test_release_allocation.rs diff --git a/Cargo.lock b/Cargo.lock index 910e33a7bd..08829b787b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9442,6 +9442,7 @@ dependencies = [ "ya-erc20-driver", "ya-framework-basic", "ya-identity", + "ya-market", "ya-metrics", "ya-net", "ya-persistence", diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index ef6280235e..e9bbda1235 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -70,6 +70,7 @@ ya-dummy-driver = "0.3" ya-erc20-driver = "0.4" ya-net = { version = "0.3", features = ["service"] } ya-identity = "0.3" +ya-market = "0.4" ya-sb-router = { workspace = true } ya-framework-basic = { version = "0.1" } diff --git a/core/payment/tests/mocks/identity.rs b/core/payment/tests/mocks/identity.rs index 8089bb3655..8981edc188 100644 --- a/core/payment/tests/mocks/identity.rs +++ b/core/payment/tests/mocks/identity.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] use anyhow::{anyhow, bail}; +use futures::FutureExt; use serde::de::DeserializeOwned; use ya_client_model::NodeId; use ya_core_model::appkey::AppKey; @@ -63,7 +64,7 @@ impl MockIdentity { }; Ok(parse_output_result::( - command.run_command(&ctx).await?, + command.run_command(&ctx).boxed_local().await?, )?) } pub async fn create_appkey(&self, name: &str, id: NodeId) -> anyhow::Result { @@ -80,6 +81,7 @@ impl MockIdentity { name: name.to_string(), } .run_command(&ctx) + .boxed_local() .await?; Ok(parse_output::(output)?) diff --git a/core/payment/tests/mocks/market.rs b/core/payment/tests/mocks/market.rs new file mode 100644 index 0000000000..ee65a2be8a --- /dev/null +++ b/core/payment/tests/mocks/market.rs @@ -0,0 +1,243 @@ +#![allow(dead_code)] + +use chrono::{Duration, Utc}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::Arc; +use tokio::sync::RwLock; + +use ya_agreement_utils::agreement::expand; +use ya_agreement_utils::{OfferTemplate, ProposalView}; +use ya_client_model::market::agreement; +use ya_client_model::market::proposal; +use ya_client_model::market::{Agreement, AgreementListEntry, Demand, Offer, Role}; +use ya_client_model::NodeId; +use ya_core_model::market; +use ya_market::testing::{AgreementId, Owner, ProposalId, SubscriptionId}; +use ya_service_bus::typed as bus; + +/// Market that doesn't wrap real Market module, but simulates it's +/// behavior by providing GSB bindings for crucial messages. +#[derive(Clone)] +pub struct FakeMarket { + name: String, + testdir: PathBuf, + + inner: Arc>, +} + +pub struct FakeMarketInner { + agreements: HashMap, +} + +impl FakeMarket { + pub fn new(name: &str, testdir: &PathBuf) -> Self { + FakeMarket { + name: name.to_string(), + testdir: testdir.clone(), + inner: Arc::new(RwLock::new(FakeMarketInner { + agreements: HashMap::new(), + })), + } + } + + pub async fn bind_gsb(&self) -> anyhow::Result<()> { + log::info!("FakeMarket ({}) - binding GSB", self.name); + + let self_ = self.clone(); + bus::bind_with_caller( + market::BUS_ID, + move |sender: String, msg: market::GetAgreement| { + let self_ = self_.clone(); + async move { self_.get_agreement_handler(sender, msg).await } + }, + ); + let self_ = self.clone(); + bus::bind_with_caller( + market::BUS_ID, + move |sender: String, msg: market::ListAgreements| { + let self_ = self_.clone(); + async move { self_.list_agreements_handler(sender, msg).await } + }, + ); + Ok(()) + } + + async fn get_agreement_handler( + &self, + _sender_id: String, + msg: market::GetAgreement, + ) -> Result { + let owner = match msg.role { + Role::Provider => Owner::Provider, + Role::Requestor => Owner::Requestor, + }; + + let agreement_id = AgreementId::from_client(&msg.agreement_id, owner) + .map_err(|e| market::RpcMessageError::Market(e.to_string()))?; + + Ok(self + .get_agreement(agreement_id.clone()) + .await + .ok_or_else(|| { + market::RpcMessageError::NotFound(format!("Agreement id: {agreement_id}")) + })?) + } + + async fn list_agreements_handler( + &self, + _sender_id: String, + msg: market::ListAgreements, + ) -> Result, market::RpcMessageError> { + let lock = self.inner.read().await; + let agreements = lock + .agreements + .iter() + .filter(|(_, agreement)| { + msg.app_session_id.is_none() || agreement.app_session_id == msg.app_session_id + }) + .filter(|(_, agreement)| msg.state.is_none() || agreement.state == msg.state.unwrap()) + .filter(|(_, agreement)| { + msg.before_date.is_none() || agreement.timestamp < msg.before_date.unwrap() + }) + .filter(|(_, agreement)| { + msg.after_date.is_none() || agreement.timestamp > msg.after_date.unwrap() + }) + .map(|(id, agreement)| AgreementListEntry { + id: agreement.agreement_id.clone(), + timestamp: agreement.timestamp.clone(), + approved_date: agreement.approved_date.clone(), + role: match id.owner() { + Owner::Provider => Role::Provider, + Owner::Requestor => Role::Requestor, + }, + }) + .collect(); + + Ok(agreements) + } + + pub async fn get_agreement(&self, agreement_id: AgreementId) -> Option { + self.inner + .read() + .await + .agreements + .get(&agreement_id) + .cloned() + } + + pub async fn add_agreement(&self, agreement: Agreement) { + let provider_id = + AgreementId::from_client(&agreement.agreement_id, Owner::Provider).unwrap(); + let requestor_id = + AgreementId::from_client(&agreement.agreement_id, Owner::Requestor).unwrap(); + + let mut lock = self.inner.write().await; + lock.agreements.insert(provider_id, agreement.clone()); + lock.agreements.insert(requestor_id, agreement); + } + + pub fn create_fake_agreement( + requestor_id: NodeId, + provider_id: NodeId, + ) -> anyhow::Result { + let offer = Self::create_default_offer(provider_id)?; + let demand = Self::create_default_demand(requestor_id)?; + + Ok(Self::agreement_from(offer, demand)?) + } + + pub fn agreement_from(offer: ProposalView, demand: ProposalView) -> anyhow::Result { + let timestamp = Utc::now(); + let agreement_id = ProposalId::generate_id( + &SubscriptionId::from_str(&offer.id)?, + &SubscriptionId::from_str(&demand.id)?, + ×tamp.naive_utc(), + Owner::Requestor, + ); + Ok(Agreement { + agreement_id: agreement_id.into_client(), + demand: Demand { + properties: demand.content.properties, + constraints: demand.content.constraints, + demand_id: demand.id, + requestor_id: demand.issuer, + timestamp: demand.timestamp, + }, + offer: Offer { + properties: offer.content.properties, + constraints: offer.content.constraints, + offer_id: offer.id, + provider_id: offer.issuer, + timestamp: offer.timestamp, + }, + valid_to: timestamp + Duration::hours(2), + approved_date: None, + state: agreement::State::Approved, + timestamp, + app_session_id: None, + proposed_signature: None, + approved_signature: None, + committed_signature: None, + }) + } + + pub fn create_default_offer(provider_id: NodeId) -> anyhow::Result { + let template = OfferTemplate { + properties: expand(serde_json::from_str(r#"{ "any": "thing" }"#).unwrap()), + constraints: "()".to_string(), + }; + Self::create_demand(provider_id, template) + } + + pub fn create_offer( + provider_id: NodeId, + content: OfferTemplate, + ) -> anyhow::Result { + let offer = ProposalView { + id: "".to_string(), + content: content.flatten(), + issuer: provider_id, + state: proposal::State::Accepted, + timestamp: Utc::now(), + }; + + let id = subscription_id_from(&offer)?.to_string(); + Ok(ProposalView { id, ..offer }) + } + + pub fn create_default_demand(requestor_id: NodeId) -> anyhow::Result { + let template = OfferTemplate { + properties: expand(serde_json::from_str(r#"{ "any": "thing" }"#).unwrap()), + constraints: "()".to_string(), + }; + Self::create_demand(requestor_id, template) + } + pub fn create_demand( + requestor_id: NodeId, + content: OfferTemplate, + ) -> anyhow::Result { + let demand = ProposalView { + id: "".to_string(), + content: content.flatten(), + issuer: requestor_id, + state: proposal::State::Accepted, + timestamp: Utc::now(), + }; + + let id = subscription_id_from(&demand)?.to_string(); + Ok(ProposalView { id, ..demand }) + } +} + +fn subscription_id_from(template: &ProposalView) -> anyhow::Result { + let id = SubscriptionId::generate_id( + &serde_json::to_string_pretty(&template.content.properties)?, + &serde_json::to_string_pretty(&template.content.constraints)?, + &template.issuer, + &template.timestamp.naive_utc(), + &(template.timestamp + Duration::hours(2)).naive_utc(), + ); + Ok(id) +} diff --git a/core/payment/tests/mocks/mod.rs b/core/payment/tests/mocks/mod.rs index de12f6ba25..cffc8e639f 100644 --- a/core/payment/tests/mocks/mod.rs +++ b/core/payment/tests/mocks/mod.rs @@ -1,3 +1,4 @@ pub mod identity; +pub mod market; pub mod node; pub mod payment; diff --git a/core/payment/tests/mocks/node.rs b/core/payment/tests/mocks/node.rs index f21b986926..b5e0cbfaf3 100644 --- a/core/payment/tests/mocks/node.rs +++ b/core/payment/tests/mocks/node.rs @@ -1,5 +1,6 @@ #![allow(dead_code)] +use crate::mocks::market::FakeMarket; use actix_web::{middleware, App, HttpServer, Scope}; use anyhow::anyhow; use std::fs; @@ -32,6 +33,7 @@ pub struct MockNode { pub identity: Option, pub payment: Option, + pub market: Option, } impl MockNode { @@ -45,19 +47,28 @@ impl MockNode { rest_url: Self::generate_rest_url(), identity: None, payment: None, + market: None, } } + /// Use full wrapped Identity module for this node. pub fn with_identity(mut self) -> Self { self.identity = Some(MockIdentity::new(&self.name)); self } + /// Use full wrapped Payment module for this node. pub fn with_payment(mut self) -> Self { self.payment = Some(MockPayment::new(&self.name, &self.testdir)); self } + /// Use fake Market module for this node. + pub fn with_fake_market(mut self) -> Self { + self.market = Some(FakeMarket::new(&self.name, &self.testdir)); + self + } + pub fn get_identity(&self) -> anyhow::Result { self.identity .clone() @@ -70,6 +81,12 @@ impl MockNode { .ok_or_else(|| anyhow!("Payment ({}) is not initialized", self.name)) } + pub fn get_market(&self) -> anyhow::Result { + self.market + .clone() + .ok_or_else(|| anyhow!("Market ({}) is not initialized", self.name)) + } + /// Binds GSB router and all initialized modules to GSB. /// If you want to bind only chosen modules, you should bind them manually. pub async fn bind_gsb(&self) -> anyhow::Result<()> { @@ -83,9 +100,17 @@ impl MockNode { payment.bind_gsb().await?; } + if let Some(market) = &self.market { + market.bind_gsb().await?; + } Ok(()) } + /// Query REST API client for payment module. + /// + /// You need to provider access token, which can be generated together with identity + /// using `MockIdentity::create_identity_key` function. + /// Token is not validated. Incorrect token can be useful in some testing scenarios. pub fn rest_payments(&self, token: &str) -> anyhow::Result { let provider: PaymentApi = WebClient::builder() .auth_token(token) @@ -96,6 +121,11 @@ impl MockNode { Ok(provider) } + /// Start actix server with all requested modules and some additional middlewares, that are + /// normally used by yagna. + /// You can make REST API requests using client created with `rest_payments` function. + /// + /// Server will be automatically stopped when `ctx` is dropped, which will happen after test will exit. pub async fn start_server(&self, ctx: &mut DroppableTestContext) -> anyhow::Result<()> { log::info!( "MockeNode ({}) - Starting server: {}", @@ -137,7 +167,7 @@ impl MockNode { self.name ); - // GSB RemoteRouter takes url from this variable and we can't set it directly. + // GSB RemoteRouter takes url from this variable, and we can't set it directly. std::env::set_var("GSB_URL", gsb_url.to_string()); ya_sb_router::bind_gsb_router(Some(gsb_url.clone())) diff --git a/core/payment/tests/test_release_allocation.rs b/core/payment/tests/test_release_allocation.rs new file mode 100644 index 0000000000..699248220e --- /dev/null +++ b/core/payment/tests/test_release_allocation.rs @@ -0,0 +1,157 @@ +use bigdecimal::BigDecimal; +use chrono::Utc; +use test_context::test_context; + +use crate::mocks::market::FakeMarket; +use ya_client_model::payment::allocation::PaymentPlatformEnum; +use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::temp_dir; + +use crate::mocks::node::MockNode; +use crate::mocks::payment::Driver; + +mod mocks; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_release_allocation")?; + + let node = MockNode::new("node-1", &dir.path()) + .with_identity() + .with_payment() + .with_fake_market(); + node.bind_gsb().await?; + node.start_server(ctx).await?; + + let requestor_appkey = node + .get_identity()? + .create_identity_key("requestor") + .await?; + let provider_appkey = node.get_identity()?.create_identity_key("provider").await?; + + let provider = node.rest_payments(&provider_appkey.key)?; + let requestor = node.rest_payments(&requestor_appkey.key)?; + + node.get_payment()? + .fund_account(Driver::Erc20, &requestor_appkey.identity.to_string()) + .await?; + + let payment_platform = + PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); + + log::info!("Creating allocation..."); + let allocation = requestor + .create_allocation(&NewAllocation { + address: Some(requestor_appkey.identity.to_string()), + payment_platform: Some(payment_platform.clone()), + total_amount: BigDecimal::from(10u64), + timeout: None, + make_deposit: false, + deposit: None, + extend_timeout: None, + }) + .await?; + log::info!("Allocation created."); + + log::info!("Verifying allocation..."); + let allocations = requestor.get_allocations::(None, None).await?; + assert_eq!(allocations.len(), 1); + assert_eq!(allocations[0], allocation); + let allocation1 = requestor.get_allocation(&allocation.allocation_id).await?; + assert_eq!(allocation1, allocation); + log::info!("Done."); + + log::info!("Releasing allocation..."); + requestor + .release_allocation(&allocation.allocation_id) + .await?; + log::info!("Allocation released."); + + log::info!("Verifying allocation removal..."); + let allocations = requestor.get_allocations::(None, None).await?; + assert_eq!(allocations.len(), 0); + let result = requestor.get_allocation(&allocation.allocation_id).await; + assert!(result.is_err()); + log::info!("Done. (Verifying allocation removal)"); + + let agreement = + FakeMarket::create_fake_agreement(requestor_appkey.identity, provider_appkey.identity) + .unwrap(); + node.get_market()?.add_agreement(agreement.clone()).await; + + log::info!("Issuing invoice..."); + let invoice = provider + .issue_invoice(&NewInvoice { + agreement_id: agreement.agreement_id.clone(), + activity_ids: None, + amount: BigDecimal::from(1u64), + payment_due_date: Utc::now(), + }) + .await?; + log::info!("Invoice issued."); + + log::info!("Sending invoice..."); + provider.send_invoice(&invoice.invoice_id).await?; + log::info!("Invoice sent."); + + log::info!("Attempting to accept invoice..."); + let result = requestor + .accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id, + }, + ) + .await; + assert!(result.is_err()); + log::info!("Failed to accept invoice (as expected)."); + + log::info!("Creating another allocation..."); + let allocation = requestor + .create_allocation(&NewAllocation { + address: Some(requestor_appkey.identity.to_string()), + payment_platform: Some(payment_platform.clone()), + total_amount: BigDecimal::from(10u64), + timeout: None, + make_deposit: false, + deposit: None, + extend_timeout: None, + }) + .await?; + log::info!("Allocation created."); + + log::info!("Accepting invoice..."); + requestor + .accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id.clone(), + }, + ) + .await?; + log::info!("Invoice accepted."); + + log::info!("Releasing allocation..."); + requestor + .release_allocation(&allocation.allocation_id) + .await?; + log::info!("Allocation released."); + + log::info!("Verifying allocation removal..."); + let allocations = requestor.get_allocations::(None, None).await?; + assert_eq!(allocations.len(), 0); + let result = requestor.get_allocation(&allocation.allocation_id).await; + assert!(result.is_err()); + log::info!("Done."); + + log::info!(" 👍🏻 Example completed successfully ❤️"); + Ok(()) +} diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index c108c4c7b7..a972800b77 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -7,8 +7,8 @@ use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::temp_dir; +use crate::mocks::node::MockNode; use crate::mocks::payment::Driver; -use mocks::node::MockNode; mod mocks; @@ -20,19 +20,29 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any let dir = temp_dir!("tutorial_how_to_use_module_tests")?; + // Create MockNode which is container for all Golem modules and represents + // single node in tests. let node = MockNode::new("node-1", &dir.path()) + // Request wrappers around real Golem modules. .with_identity() - .with_payment(); + .with_payment() + // Mock market module with very basic implementation, which will allow to manually + // create fake Agreements without need for Offers broadcasting and negotiation process. + .with_fake_market(); + + // Bind GSB and start server like yagna node would do in full setup. + // Those functions will bind only modules chosen for MockNode. node.bind_gsb().await?; node.start_server(ctx).await?; let appkey = node.get_identity()?.create_identity_key("test").await?; - let api = node.rest_payments(&appkey.key)?; node.get_payment()? .fund_account(Driver::Erc20, &appkey.identity.to_string()) .await?; + let api = node.rest_payments(&appkey.key)?; + let _allocation = api .create_allocation(&NewAllocation { address: None, // Use default address (i.e. identity) From b507085e358c841a62873d167fc0bbd2328daa55 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 31 Jul 2024 18:34:12 +0200 Subject: [PATCH 057/125] Payment test environment: Implement MockNet to redirect payment messages --- Cargo.lock | 1 + core/net/src/hybrid/mod.rs | 4 + core/net/src/hybrid/service.rs | 4 +- core/payment/tests/mocks/identity.rs | 19 ++- core/payment/tests/mocks/market.rs | 9 +- core/payment/tests/mocks/node.rs | 8 +- core/payment/tests/test_release_allocation.rs | 20 ++- .../tests/tutorial_how_to_use_module_tests.rs | 33 +++-- .../test-framework/framework-basic/Cargo.toml | 2 + .../test-framework/framework-basic/src/log.rs | 29 +++-- .../framework-basic/src/mocks/net.rs | 121 +++++++++--------- 11 files changed, 159 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08829b787b..803fb07eca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9049,6 +9049,7 @@ dependencies = [ "ya-client", "ya-client-model", "ya-core-model", + "ya-net", "ya-service-bus", "ya-utils-futures", "ya-utils-path", diff --git a/core/net/src/hybrid/mod.rs b/core/net/src/hybrid/mod.rs index 80adb99290..6c90139229 100644 --- a/core/net/src/hybrid/mod.rs +++ b/core/net/src/hybrid/mod.rs @@ -8,3 +8,7 @@ mod service; pub use api::*; pub use rest_api::web_scope; pub use service::{send_bcast_new_neighbour, start_network, Net}; + +pub mod testing { + pub use crate::hybrid::service::{parse_from_to_addr, parse_net_to_addr}; +} diff --git a/core/net/src/hybrid/service.rs b/core/net/src/hybrid/service.rs index 0a57415815..93316abbf8 100644 --- a/core/net/src/hybrid/service.rs +++ b/core/net/src/hybrid/service.rs @@ -1147,7 +1147,7 @@ fn handler_reply_err( }); } -fn parse_net_to_addr(addr: &str) -> anyhow::Result<(NodeId, String)> { +pub fn parse_net_to_addr(addr: &str) -> anyhow::Result<(NodeId, String)> { const ADDR_CONST: usize = 6; let mut it = addr.split('/').fuse().skip(1).peekable(); @@ -1165,7 +1165,7 @@ fn parse_net_to_addr(addr: &str) -> anyhow::Result<(NodeId, String)> { Ok((to_id, format!("{}{}", prefix, addr))) } -fn parse_from_to_addr(addr: &str) -> anyhow::Result<(NodeId, NodeId, String)> { +pub fn parse_from_to_addr(addr: &str) -> anyhow::Result<(NodeId, NodeId, String)> { const ADDR_CONST: usize = 10; let mut it = addr.split('/').fuse().skip(1).peekable(); diff --git a/core/payment/tests/mocks/identity.rs b/core/payment/tests/mocks/identity.rs index 8981edc188..7363539cbb 100644 --- a/core/payment/tests/mocks/identity.rs +++ b/core/payment/tests/mocks/identity.rs @@ -6,6 +6,7 @@ use serde::de::DeserializeOwned; use ya_client_model::NodeId; use ya_core_model::appkey::AppKey; use ya_core_model::identity::IdentityInfo; +use ya_framework_basic::mocks::net::MockNet; use ya_identity::cli::{AppKeyCommand, IdentityCommand}; use ya_identity::service::Identity; @@ -14,15 +15,17 @@ use ya_service_api::{CliCtx, CommandOutput}; #[derive(Clone)] pub struct MockIdentity { + net: MockNet, name: String, db: DbExecutor, } impl MockIdentity { - pub fn new(name: &str) -> Self { + pub fn new(net: MockNet, name: &str) -> Self { let db = Self::create_db(&format!("{name}.identity.db")).unwrap(); MockIdentity { + net, name: name.to_string(), db, } @@ -50,9 +53,23 @@ impl MockIdentity { .await .map_err(|e| anyhow!("Creating AppKey: {e}"))?; + self.register_identity_in_net(identity.node_id); Ok(appkey) } + fn register_identity_in_net(&self, id: NodeId) { + // This line is temporary, until we will be able to rebind all modules to non-fixed prefix. + // Currently, all modules must be bound under `/local/{module}` and `/public/{module}`. + // Not doing so would break most of them. + // For example Payment module uses fixed prefix to call market and identity modules. + // When we will work around this problem, we will be able to instantiate many nodes in tests. + self.net.register_node(&id, "/public"); + + // Should be instead in the future: + // self.net + // .register_node(&id, &format!("/{}/public/{id}", self.name)); + } + pub async fn create_identity(&self, name: &str) -> anyhow::Result { let ctx = CliCtx::default(); let command = IdentityCommand::Create { diff --git a/core/payment/tests/mocks/market.rs b/core/payment/tests/mocks/market.rs index ee65a2be8a..87bb1bea4e 100644 --- a/core/payment/tests/mocks/market.rs +++ b/core/payment/tests/mocks/market.rs @@ -1,6 +1,7 @@ #![allow(dead_code)] use chrono::{Duration, Utc}; +use serde_json::json; use std::collections::HashMap; use std::path::PathBuf; use std::str::FromStr; @@ -208,8 +209,14 @@ impl FakeMarket { } pub fn create_default_demand(requestor_id: NodeId) -> anyhow::Result { + let basic_props = json!({ + "golem.com.payment.platform.erc20-holesky-tglm.address": requestor_id.to_string(), + "golem.com.payment.protocol.version": 3, + "golem.com.payment.chosen-platform": "erc20-holesky-tglm", + }); + let template = OfferTemplate { - properties: expand(serde_json::from_str(r#"{ "any": "thing" }"#).unwrap()), + properties: expand(basic_props), constraints: "()".to_string(), }; Self::create_demand(requestor_id, template) diff --git a/core/payment/tests/mocks/node.rs b/core/payment/tests/mocks/node.rs index b5e0cbfaf3..3c238b256b 100644 --- a/core/payment/tests/mocks/node.rs +++ b/core/payment/tests/mocks/node.rs @@ -11,6 +11,7 @@ use url::Url; use ya_client::payment::PaymentApi; use ya_client::web::WebClient; use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::mocks::net::MockNet; use ya_service_api_web::middleware::auth; use ya_service_api_web::middleware::cors::{AppKeyCors, CorsConfig}; use ya_service_api_web::rest_api_host_port; @@ -26,6 +27,8 @@ use super::payment::MockPayment; /// using fixed addresses. This should be improved in the future. #[derive(Clone)] pub struct MockNode { + net: MockNet, + name: String, testdir: PathBuf, @@ -37,11 +40,12 @@ pub struct MockNode { } impl MockNode { - pub fn new(name: &str, testdir: &Path) -> Self { + pub fn new(net: MockNet, name: &str, testdir: &Path) -> Self { let testdir = testdir.join(name); fs::create_dir_all(&testdir).expect("Failed to create test directory"); MockNode { + net, name: name.to_string(), testdir, rest_url: Self::generate_rest_url(), @@ -53,7 +57,7 @@ impl MockNode { /// Use full wrapped Identity module for this node. pub fn with_identity(mut self) -> Self { - self.identity = Some(MockIdentity::new(&self.name)); + self.identity = Some(MockIdentity::new(self.net.clone(), &self.name)); self } diff --git a/core/payment/tests/test_release_allocation.rs b/core/payment/tests/test_release_allocation.rs index 699248220e..938b1047f4 100644 --- a/core/payment/tests/test_release_allocation.rs +++ b/core/payment/tests/test_release_allocation.rs @@ -7,6 +7,7 @@ use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; +use ya_framework_basic::mocks::net::MockNet; use ya_framework_basic::temp_dir; use crate::mocks::node::MockNode; @@ -22,7 +23,10 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu let dir = temp_dir!("test_release_allocation")?; - let node = MockNode::new("node-1", &dir.path()) + let net = MockNet::default(); + net.bind_gsb(); + + let node = MockNode::new(net, "node-1", &dir.path()) .with_identity() .with_payment() .with_fake_market(); @@ -59,6 +63,15 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu .await?; log::info!("Allocation created."); + log::info!("Creating fake Agreement..."); + + let agreement = + FakeMarket::create_fake_agreement(requestor_appkey.identity, provider_appkey.identity) + .unwrap(); + node.get_market()?.add_agreement(agreement.clone()).await; + + log::info!("Fake Agreement created: {}", agreement.agreement_id); + log::info!("Verifying allocation..."); let allocations = requestor.get_allocations::(None, None).await?; assert_eq!(allocations.len(), 1); @@ -80,11 +93,6 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu assert!(result.is_err()); log::info!("Done. (Verifying allocation removal)"); - let agreement = - FakeMarket::create_fake_agreement(requestor_appkey.identity, provider_appkey.identity) - .unwrap(); - node.get_market()?.add_agreement(agreement.clone()).await; - log::info!("Issuing invoice..."); let invoice = provider .issue_invoice(&NewInvoice { diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index a972800b77..7bd254ec1b 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -5,8 +5,10 @@ use ya_client_model::payment::NewAllocation; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; +use ya_framework_basic::mocks::net::MockNet; use ya_framework_basic::temp_dir; +use crate::mocks::market::FakeMarket; use crate::mocks::node::MockNode; use crate::mocks::payment::Driver; @@ -20,10 +22,17 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any let dir = temp_dir!("tutorial_how_to_use_module_tests")?; + // MockNet routes traffic between MockNodes. + // Currently instantiating many MockNodes is not possible, but MockNet is necessary even + // for communication on the same node, because messages directed to external GSB addresses `/net/0x437544...` + // when NodeId belongs to local Node, need to be routed back. + let net = MockNet::default(); + net.bind_gsb(); + // Create MockNode which is container for all Golem modules and represents // single node in tests. - let node = MockNode::new("node-1", &dir.path()) - // Request wrappers around real Golem modules. + let node = MockNode::new(net, "node-1", &dir.path()) + // Request instantiating wrappers around real Golem modules. .with_identity() .with_payment() // Mock market module with very basic implementation, which will allow to manually @@ -35,20 +44,24 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any node.bind_gsb().await?; node.start_server(ctx).await?; - let appkey = node.get_identity()?.create_identity_key("test").await?; + // Creating identities is essential to use REST API and create Agreements and Payments. + // Provider and Requestor should use separate identity. + let identity = node.get_identity()?; + let appkey_req = identity.create_identity_key("requestor").await?; + let appkey_prov = identity.create_identity_key("provider").await?; + let payment_platform = + PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); node.get_payment()? - .fund_account(Driver::Erc20, &appkey.identity.to_string()) + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) .await?; - let api = node.rest_payments(&appkey.key)?; + let api = node.rest_payments(&appkey_req.key)?; let _allocation = api .create_allocation(&NewAllocation { address: None, // Use default address (i.e. identity) - payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName( - "erc20-holesky-tglm".to_string(), - )), + payment_platform: Some(payment_platform.clone()), total_amount: BigDecimal::from(10u64), timeout: None, make_deposit: false, @@ -58,5 +71,9 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any .await .unwrap(); + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node.get_market()?.add_agreement(agreement.clone()).await; + Ok(()) } diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index b5dbe8dfe9..6ad09f8c21 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -17,6 +17,8 @@ ya-utils-path = { version = "0.1", path = "../../../utils/path" } ya-utils-futures.workspace = true gftp = { workspace = true } +ya-net = "0.3" + actix = "0.13" actix-files = "0.6" actix-http = "3" diff --git a/test-utils/test-framework/framework-basic/src/log.rs b/test-utils/test-framework/framework-basic/src/log.rs index 4335f6c067..9d85d40bf1 100644 --- a/test-utils/test-framework/framework-basic/src/log.rs +++ b/test-utils/test-framework/framework-basic/src/log.rs @@ -1,17 +1,24 @@ -use env_logger::TimestampPrecision; +use env_logger::{Env, TimestampPrecision}; use std::env; pub fn enable_logs(enable: bool) { - env::set_var( - "RUST_LOG", - env::var("RUST_LOG").unwrap_or_else(|_| { - "info,web3=warn,sqlx_core=warn,hyper=warn,erc20_payment_lib=info,trust_dns_proto=warn,erc20_rpc_pool=info,trust_dns_resolver=warn,ya_erc20_driver=info".into() - }), - ); if enable { - env_logger::builder() - .format_timestamp(Some(TimestampPrecision::Millis)) - .try_init() - .ok(); + if let Ok(_env) = env::var("RUST_LOG") { + env_logger::try_init_from_env(Env::default()).ok(); + } else { + env_logger::builder() + .filter_level(log::LevelFilter::Info) + .filter(Some("web3"), log::LevelFilter::Warn) + .filter(Some("sqlx_core"), log::LevelFilter::Warn) + .filter(Some("hyper"), log::LevelFilter::Warn) + .filter(Some("erc20_payment_lib"), log::LevelFilter::Info) + .filter(Some("trust_dns_proto"), log::LevelFilter::Warn) + .filter(Some("erc20_rpc_pool"), log::LevelFilter::Info) + .filter(Some("trust_dns_resolver"), log::LevelFilter::Warn) + .filter(Some("ya_erc20_driver"), log::LevelFilter::Info) + .format_timestamp(Some(TimestampPrecision::Millis)) + .try_init() + .ok(); + } } } diff --git a/test-utils/test-framework/framework-basic/src/mocks/net.rs b/test-utils/test-framework/framework-basic/src/mocks/net.rs index 99fdb1acb6..0f3ab089e7 100644 --- a/test-utils/test-framework/framework-basic/src/mocks/net.rs +++ b/test-utils/test-framework/framework-basic/src/mocks/net.rs @@ -1,12 +1,12 @@ use anyhow::Result; use std::collections::HashMap; use std::rc::Rc; -use std::str::FromStr; use std::sync::{Arc, Mutex}; use ya_client::model::NodeId; use ya_core_model::net; use ya_core_model::net::{local as local_net, local::SendBroadcastMessage}; +use ya_net::hybrid::testing::{parse_from_to_addr, parse_net_to_addr}; use ya_service_bus::{serialization, typed as bus, untyped as local_bus, Error, RpcMessage}; #[cfg(feature = "bcast-singleton")] @@ -47,16 +47,11 @@ impl MockNet { } pub fn register_node(&self, node_id: &NodeId, prefix: &str) { - // Only two first components - let mut iter = prefix.split('/').fuse(); - let prefix = match (iter.next(), iter.next(), iter.next()) { - (Some(""), Some(test_name), Some(name)) => format!("/{}/{}", test_name, name), - _ => panic!("[MockNet] Can't register prefix {}", prefix), - }; + log::info!("[MockNet] Registering node {node_id} at prefix: {prefix}"); let mut inner = self.inner.lock().unwrap(); - if inner.nodes.insert(*node_id, prefix).is_some() { - panic!("[MockNet] Node [{}] already existed.", &node_id); + if inner.nodes.insert(*node_id, prefix.to_string()).is_some() { + panic!("[MockNet] Node [{}] already existed.", node_id); } } @@ -69,31 +64,19 @@ impl MockNet { .ok_or_else(|| anyhow::anyhow!("node not registered: {}", node_id)) } - async fn translate_address(&self, address: String) -> Result<(NodeId, String)> { - let (from_node, to_addr) = match parse_from_addr(&address) { - Ok(v) => v, - Err(e) => Err(Error::GsbBadRequest(e.to_string()))?, - }; - - let mut iter = to_addr.split('/').fuse(); - let dst_id = match (iter.next(), iter.next(), iter.next()) { - (Some(""), Some("net"), Some(dst_id)) => dst_id, - _ => panic!("[MockNet] Invalid destination address {}", to_addr), - }; + fn translate_to(&self, id: NodeId, addr: &str) -> Result { + let prefix = self.node_prefix(id)?; + let net_prefix = format!("/net/{}", id); + Ok(addr.replacen(&net_prefix, &prefix, 1)) + } - let dest_node_id = NodeId::from_str(dst_id)?; + fn node_prefix(&self, id: NodeId) -> Result { let inner = self.inner.lock().unwrap(); - let local_prefix = inner.nodes.get(&dest_node_id); - - if let Some(local_prefix) = local_prefix { - let net_prefix = format!("/net/{}", dst_id); - Ok((from_node, to_addr.replacen(&net_prefix, local_prefix, 1))) - } else { - Err(Error::GsbFailure(format!( - "[MockNet] Can't find destination address for endpoint [{}].", - &address - )))? - } + Ok(inner + .nodes + .get(&id) + .cloned() + .ok_or_else(|| anyhow::anyhow!("node not registered: {}", id))?) } pub fn node_by_prefix(&self, address: &str) -> Option { @@ -180,56 +163,74 @@ impl MockNetInner { (), ); + Self::bind_local_bus(MockNet::default(), FROM_BUS_ID, from_address_resolver); + Self::bind_local_bus(MockNet::default(), FROM_UDP_BUS_ID, from_address_resolver); + Self::bind_local_bus( + MockNet::default(), + FROM_TRANSFER_BUS_ID, + from_address_resolver, + ); + + Self::bind_local_bus(MockNet::default(), net::BUS_ID, net_address_resolver); + Self::bind_local_bus(MockNet::default(), net::BUS_ID_UDP, net_address_resolver); + Self::bind_local_bus( + MockNet::default(), + net::BUS_ID_TRANSFER, + net_address_resolver, + ); + } + + fn bind_local_bus(net: MockNet, address: &'static str, resolver: F) + where + F: Fn(&str, &str) -> anyhow::Result<(String, NodeId, String)> + 'static, + { + let resolver = Arc::new(resolver); + local_bus::subscribe( - FROM_BUS_ID, + address, move |caller: &str, addr: &str, msg: &[u8]| { - let mock_net = MockNet::default(); + let mock_net = net.clone(); let data = Vec::from(msg); let caller = caller.to_string(); let addr = addr.to_string(); + let resolver_ = resolver.clone(); async move { - let (from, local_addr) = mock_net - .translate_address(addr) - .await + log::info!("[MockNet] Received message from [{caller}], on address [{addr}]."); + + let (from, to, address) = resolver_(&caller, &addr) + .map_err(|e| Error::GsbBadRequest(e.to_string()))?; + let translated = mock_net + .translate_to(to, &address) .map_err(|e| Error::GsbBadRequest(e.to_string()))?; - log::debug!( - "[MockNet] Sending message from [{}], to address [{}].", - &caller, - &local_addr + log::info!( + "[MockNet] Sending message from [{from}], to: [{to}], address [{translated}]." ); - local_bus::send(&local_addr, &from.to_string(), &data).await + local_bus::send(&translated, &from.to_string(), &data).await } }, + // TODO: Implement stream handler (), ); } } -// Copied from core/net/api.rs -pub(crate) fn parse_from_addr(from_addr: &str) -> Result<(NodeId, String)> { - let mut it = from_addr.split('/').fuse(); - if let (Some(""), Some("from"), Some(from_node_id), Some("to"), Some(to_node_id)) = - (it.next(), it.next(), it.next(), it.next(), it.next()) - { - to_node_id.parse::()?; - let prefix = 10 + from_node_id.len(); - let service_id = &from_addr[prefix..]; - if it.next().is_some() { - return Ok((from_node_id.parse()?, net_service(service_id))); - } - } - anyhow::bail!("invalid net-from destination: {}", from_addr) +fn from_address_resolver(_caller: &str, addr: &str) -> anyhow::Result<(String, NodeId, String)> { + let (from, to, addr) = + parse_from_to_addr(addr).map_err(|e| anyhow::anyhow!("invalid address: {}", e))?; + Ok((from.to_string(), to, addr)) } -// Copied from core/net/api.rs -#[inline] -pub(crate) fn net_service(service: impl ToString) -> String { - format!("{}/{}", net::BUS_ID, service.to_string()) +fn net_address_resolver(caller: &str, addr: &str) -> anyhow::Result<(String, NodeId, String)> { + let (to, addr) = + parse_net_to_addr(addr).map_err(|e| anyhow::anyhow!("invalid address: {}", e))?; + Ok((caller.to_string(), to, addr)) } pub(crate) const FROM_BUS_ID: &str = "/from"; +pub(crate) const FROM_UDP_BUS_ID: &str = "/udp/from"; +pub(crate) const FROM_TRANSFER_BUS_ID: &str = "/transfer/from"; pub fn gsb_prefixes(test_name: &str, name: &str) -> (String, String) { let public_gsb_prefix = format!("/{}/{}/market", test_name, name); From 3e020be05019187b16a4a8d6e5f8ca797f37edbe Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 31 Jul 2024 18:39:09 +0200 Subject: [PATCH 058/125] After merge test fixes --- core/payment/tests/mocks/payment.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/payment/tests/mocks/payment.rs b/core/payment/tests/mocks/payment.rs index 06a904cfe6..46194c7331 100644 --- a/core/payment/tests/mocks/payment.rs +++ b/core/payment/tests/mocks/payment.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use ya_core_model::driver::{driver_bus_id, Fund}; use ya_payment::api::web_scope; +use ya_payment::config::Config; use ya_payment::migrations; use ya_payment::processor::PaymentProcessor; use ya_payment::service::BindOptions; @@ -68,6 +69,7 @@ impl MockPayment { &self.db, self.processor.clone(), BindOptions::default().run_sync_job(false), + Arc::new(Config::from_env()?), ); self.start_dummy_driver().await?; From 755cc6c7fcd7396047dbe7971eb945301182516d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 31 Jul 2024 18:45:39 +0200 Subject: [PATCH 059/125] Enable all tests behind feature flag on CI system tests --- .github/workflows/system-test.yml | 2 +- test-utils/test-framework/framework-basic/src/mocks/net.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/system-test.yml b/.github/workflows/system-test.yml index fe5968ee1b..6f9a1523ff 100644 --- a/.github/workflows/system-test.yml +++ b/.github/workflows/system-test.yml @@ -71,4 +71,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --test '*' --features framework-test + args: --test '*' -p yagna -p ya-exe-unit -p ya-transfer -p ya-payment --features framework-test diff --git a/test-utils/test-framework/framework-basic/src/mocks/net.rs b/test-utils/test-framework/framework-basic/src/mocks/net.rs index 0f3ab089e7..4e00f46d4a 100644 --- a/test-utils/test-framework/framework-basic/src/mocks/net.rs +++ b/test-utils/test-framework/framework-basic/src/mocks/net.rs @@ -72,11 +72,11 @@ impl MockNet { fn node_prefix(&self, id: NodeId) -> Result { let inner = self.inner.lock().unwrap(); - Ok(inner + inner .nodes .get(&id) .cloned() - .ok_or_else(|| anyhow::anyhow!("node not registered: {}", id))?) + .ok_or_else(|| anyhow::anyhow!("Node not registered: {id}")) } pub fn node_by_prefix(&self, address: &str) -> Option { From 3335b1e0c9e257e7780593239cf107c09cbc3886 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 31 Jul 2024 18:55:44 +0200 Subject: [PATCH 060/125] Clippy fixes --- core/payment/tests/mocks/identity.rs | 6 ++---- core/payment/tests/mocks/market.rs | 9 ++++----- core/payment/tests/test_release_allocation.rs | 2 +- core/payment/tests/tutorial_how_to_use_module_tests.rs | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/core/payment/tests/mocks/identity.rs b/core/payment/tests/mocks/identity.rs index 7363539cbb..2f971a8b9d 100644 --- a/core/payment/tests/mocks/identity.rs +++ b/core/payment/tests/mocks/identity.rs @@ -80,9 +80,7 @@ impl MockIdentity { from_private_key: None, }; - Ok(parse_output_result::( - command.run_command(&ctx).boxed_local().await?, - )?) + parse_output_result::(command.run_command(&ctx).boxed_local().await?) } pub async fn create_appkey(&self, name: &str, id: NodeId) -> anyhow::Result { let ctx = CliCtx::default(); @@ -101,7 +99,7 @@ impl MockIdentity { .boxed_local() .await?; - Ok(parse_output::(output)?) + parse_output::(output) } } diff --git a/core/payment/tests/mocks/market.rs b/core/payment/tests/mocks/market.rs index 87bb1bea4e..e456195091 100644 --- a/core/payment/tests/mocks/market.rs +++ b/core/payment/tests/mocks/market.rs @@ -78,12 +78,11 @@ impl FakeMarket { let agreement_id = AgreementId::from_client(&msg.agreement_id, owner) .map_err(|e| market::RpcMessageError::Market(e.to_string()))?; - Ok(self - .get_agreement(agreement_id.clone()) + self.get_agreement(agreement_id.clone()) .await .ok_or_else(|| { market::RpcMessageError::NotFound(format!("Agreement id: {agreement_id}")) - })?) + }) } async fn list_agreements_handler( @@ -108,7 +107,7 @@ impl FakeMarket { .map(|(id, agreement)| AgreementListEntry { id: agreement.agreement_id.clone(), timestamp: agreement.timestamp.clone(), - approved_date: agreement.approved_date.clone(), + approved_date: agreement.approved_date, role: match id.owner() { Owner::Provider => Role::Provider, Owner::Requestor => Role::Requestor, @@ -146,7 +145,7 @@ impl FakeMarket { let offer = Self::create_default_offer(provider_id)?; let demand = Self::create_default_demand(requestor_id)?; - Ok(Self::agreement_from(offer, demand)?) + Self::agreement_from(offer, demand) } pub fn agreement_from(offer: ProposalView, demand: ProposalView) -> anyhow::Result { diff --git a/core/payment/tests/test_release_allocation.rs b/core/payment/tests/test_release_allocation.rs index 938b1047f4..7313d217b7 100644 --- a/core/payment/tests/test_release_allocation.rs +++ b/core/payment/tests/test_release_allocation.rs @@ -26,7 +26,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu let net = MockNet::default(); net.bind_gsb(); - let node = MockNode::new(net, "node-1", &dir.path()) + let node = MockNode::new(net, "node-1", dir.path()) .with_identity() .with_payment() .with_fake_market(); diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 7bd254ec1b..b703366567 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -31,7 +31,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Create MockNode which is container for all Golem modules and represents // single node in tests. - let node = MockNode::new(net, "node-1", &dir.path()) + let node = MockNode::new(net, "node-1", dir.path()) // Request instantiating wrappers around real Golem modules. .with_identity() .with_payment() From a2ba51b9062e22f9cb50c747a446b6c224d028d0 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 31 Jul 2024 19:23:30 +0200 Subject: [PATCH 061/125] Payment test environment: use pre-generated wallet address for all tests --- core/payment/tests/mocks/identity.rs | 41 ++++++++++++ .../tests/resources/ci-requestor-1.key.priv | 1 + core/payment/tests/test_release_allocation.rs | 4 +- .../tests/tutorial_how_to_use_module_tests.rs | 16 +++-- .../test-framework/framework-basic/src/lib.rs | 2 +- .../framework-basic/src/temp.rs | 23 ------- .../framework-basic/src/test_dirs.rs | 63 +++++++++++++++++++ 7 files changed, 120 insertions(+), 30 deletions(-) create mode 100644 core/payment/tests/resources/ci-requestor-1.key.priv delete mode 100644 test-utils/test-framework/framework-basic/src/temp.rs create mode 100644 test-utils/test-framework/framework-basic/src/test_dirs.rs diff --git a/core/payment/tests/mocks/identity.rs b/core/payment/tests/mocks/identity.rs index 2f971a8b9d..d6e64e5ba8 100644 --- a/core/payment/tests/mocks/identity.rs +++ b/core/payment/tests/mocks/identity.rs @@ -3,6 +3,8 @@ use anyhow::{anyhow, bail}; use futures::FutureExt; use serde::de::DeserializeOwned; +use std::fs; +use std::path::Path; use ya_client_model::NodeId; use ya_core_model::appkey::AppKey; use ya_core_model::identity::IdentityInfo; @@ -57,6 +59,27 @@ impl MockIdentity { Ok(appkey) } + pub async fn create_from_private_key(&self, path: &Path) -> anyhow::Result { + let name = path + .file_name() + .ok_or(anyhow!("Invalid private key path: {}", path.display()))? + .to_string_lossy() + .to_string(); + + let key: String = fs::read_to_string(path)?; + let identity: IdentityInfo = self + .load_identity(&name, key) + .await + .map_err(|e| anyhow!("Creating Identity: {e}"))?; + let appkey = self + .create_appkey(&name, identity.node_id) + .await + .map_err(|e| anyhow!("Creating AppKey: {e}"))?; + + self.register_identity_in_net(identity.node_id); + Ok(appkey) + } + fn register_identity_in_net(&self, id: NodeId) { // This line is temporary, until we will be able to rebind all modules to non-fixed prefix. // Currently, all modules must be bound under `/local/{module}` and `/public/{module}`. @@ -82,6 +105,24 @@ impl MockIdentity { parse_output_result::(command.run_command(&ctx).boxed_local().await?) } + + pub async fn load_identity( + &self, + name: &str, + private_key: String, + ) -> anyhow::Result { + let ctx = CliCtx::default(); + let command = IdentityCommand::Create { + no_password: true, + alias: Some(name.to_string()), + password: None, + from_keystore: None, + from_private_key: Some(private_key), + }; + + parse_output_result::(command.run_command(&ctx).boxed_local().await?) + } + pub async fn create_appkey(&self, name: &str, id: NodeId) -> anyhow::Result { let ctx = CliCtx::default(); let command = AppKeyCommand::Create { diff --git a/core/payment/tests/resources/ci-requestor-1.key.priv b/core/payment/tests/resources/ci-requestor-1.key.priv new file mode 100644 index 0000000000..b1f87929a6 --- /dev/null +++ b/core/payment/tests/resources/ci-requestor-1.key.priv @@ -0,0 +1 @@ +c61ba358059175970504bc0fe1a48918e9ca5a38207cb1d11a4eb6f268f12764 \ No newline at end of file diff --git a/core/payment/tests/test_release_allocation.rs b/core/payment/tests/test_release_allocation.rs index 7313d217b7..25991692b5 100644 --- a/core/payment/tests/test_release_allocation.rs +++ b/core/payment/tests/test_release_allocation.rs @@ -8,7 +8,7 @@ use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::mocks::net::MockNet; -use ya_framework_basic::temp_dir; +use ya_framework_basic::{resource, temp_dir}; use crate::mocks::node::MockNode; use crate::mocks::payment::Driver; @@ -35,7 +35,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu let requestor_appkey = node .get_identity()? - .create_identity_key("requestor") + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) .await?; let provider_appkey = node.get_identity()?.create_identity_key("provider").await?; diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index b703366567..07330b54e3 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -6,7 +6,7 @@ use ya_client_model::payment::NewAllocation; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::mocks::net::MockNet; -use ya_framework_basic::temp_dir; +use ya_framework_basic::{resource, temp_dir}; use crate::mocks::market::FakeMarket; use crate::mocks::node::MockNode; @@ -47,17 +47,25 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Creating identities is essential to use REST API and create Agreements and Payments. // Provider and Requestor should use separate identity. let identity = node.get_identity()?; - let appkey_req = identity.create_identity_key("requestor").await?; + // Requestor identity is created from pre-existing private key. Provider will use newly created identity. + // Using the same identity exposes our private key, but these are testnet money anyway. + // By doing this we can speed up tests significantly, because we don't have to wait for + // wallet founding, which is rather long operation. + let appkey_req = identity + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; let appkey_prov = identity.create_identity_key("provider").await?; - let payment_platform = - PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); + // Fund Requestor account. In most case we already have funds on this wallet, + // so this will be no-op. node.get_payment()? .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) .await?; let api = node.rest_payments(&appkey_req.key)?; + let payment_platform = + PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); let _allocation = api .create_allocation(&NewAllocation { address: None, // Use default address (i.e. identity) diff --git a/test-utils/test-framework/framework-basic/src/lib.rs b/test-utils/test-framework/framework-basic/src/lib.rs index 9e5077a628..f53c894395 100644 --- a/test-utils/test-framework/framework-basic/src/lib.rs +++ b/test-utils/test-framework/framework-basic/src/lib.rs @@ -4,5 +4,5 @@ pub mod hash; pub mod log; pub mod mocks; pub mod server_external; -pub mod temp; pub mod template; +pub mod test_dirs; diff --git a/test-utils/test-framework/framework-basic/src/temp.rs b/test-utils/test-framework/framework-basic/src/temp.rs deleted file mode 100644 index aea081f7ef..0000000000 --- a/test-utils/test-framework/framework-basic/src/temp.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::fs; -use tempdir::TempDir; - -pub mod macros { - #[macro_export] - macro_rules! temp_dir { - ($prefix:literal) => { - // CARGO_TARGET_TMPDIR is available in compile time only in binary modules, so we can't - // use it in this library. Thanks to using macro it will be resolved in final code not here - // and it will work. - ya_framework_basic::temp::temp_dir_(env!("CARGO_TARGET_TMPDIR"), $prefix) - }; - } -} - -pub fn temp_dir_(base_dir: &str, prefix: &str) -> anyhow::Result { - fs::create_dir_all(base_dir)?; - let dir = TempDir::new_in(base_dir, prefix)?; - let temp_dir = dir.path(); - fs::create_dir_all(temp_dir)?; - - Ok(dir) -} diff --git a/test-utils/test-framework/framework-basic/src/test_dirs.rs b/test-utils/test-framework/framework-basic/src/test_dirs.rs new file mode 100644 index 0000000000..bbe35b4643 --- /dev/null +++ b/test-utils/test-framework/framework-basic/src/test_dirs.rs @@ -0,0 +1,63 @@ +use anyhow::anyhow; +use std::fs; +use std::path::{Path, PathBuf}; +use tempdir::TempDir; + +pub mod macros { + /// Creates temporary directory in cargo target directory. + #[macro_export] + macro_rules! temp_dir { + ($prefix:literal) => { + // CARGO_TARGET_TMPDIR is available in compile time only in binary modules, so we can't + // use it in this library. Thanks to using macro it will be resolved in final code not here + // and it will work. + ya_framework_basic::test_dirs::temp_dir_(env!("CARGO_TARGET_TMPDIR"), $prefix) + }; + } + + /// Returns resource from `resources` directory in tests folder. + #[macro_export] + macro_rules! resource { + ($name:literal) => { + ya_framework_basic::test_dirs::resource_(env!("CARGO_MANIFEST_DIR"), $name) + }; + } +} + +pub fn temp_dir_(base_dir: &str, prefix: &str) -> anyhow::Result { + fs::create_dir_all(base_dir)?; + let dir = TempDir::new_in(base_dir, prefix)?; + let temp_dir = dir.path(); + fs::create_dir_all(temp_dir)?; + + Ok(dir) +} + +/// Returns resource from `resources` directory in tests. +pub fn resource_(base_dir: &str, name: &str) -> PathBuf { + PathBuf::from(base_dir) + .join("tests") + .join("resources") + .join(name) +} + +/// Generates resource from template by replacing occurrences of `${name}` pattern +/// using variables from `vars` dictionary. +/// Returns path to generated file, which is the same as `target` param, but makes it easier +/// to use this function in code. +pub fn template( + template: &Path, + target: impl AsRef, + vars: &[(&str, String)], +) -> anyhow::Result { + let mut template = fs::read_to_string(template) + .map_err(|e| anyhow!("Loading template {} failed: {e}", template.display()))?; + for var in vars { + template = template.replace(&format!("${{{}}}", var.0), &var.1); + } + + let target = target.as_ref(); + fs::write(target, template) + .map_err(|e| anyhow!("Saving template {} failed: {e}", target.display()))?; + Ok(target.to_path_buf()) +} From af9e36f548ec12f15f03b8ca262c5809d32ddac3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 31 Jul 2024 19:25:41 +0200 Subject: [PATCH 062/125] Add comments about using fixed wallet; Clippy fixes --- core/payment/tests/mocks/market.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/payment/tests/mocks/market.rs b/core/payment/tests/mocks/market.rs index e456195091..374d762291 100644 --- a/core/payment/tests/mocks/market.rs +++ b/core/payment/tests/mocks/market.rs @@ -3,7 +3,7 @@ use chrono::{Duration, Utc}; use serde_json::json; use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; @@ -33,10 +33,10 @@ pub struct FakeMarketInner { } impl FakeMarket { - pub fn new(name: &str, testdir: &PathBuf) -> Self { + pub fn new(name: &str, testdir: &Path) -> Self { FakeMarket { name: name.to_string(), - testdir: testdir.clone(), + testdir: testdir.to_path_buf(), inner: Arc::new(RwLock::new(FakeMarketInner { agreements: HashMap::new(), })), @@ -106,7 +106,7 @@ impl FakeMarket { }) .map(|(id, agreement)| AgreementListEntry { id: agreement.agreement_id.clone(), - timestamp: agreement.timestamp.clone(), + timestamp: agreement.timestamp, approved_date: agreement.approved_date, role: match id.owner() { Owner::Provider => Role::Provider, From cade23db37ca5d5d5f453b84fb90525a587507d9 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 1 Aug 2024 19:15:00 +0200 Subject: [PATCH 063/125] Fix market test suite after MockNet changes --- core/market/src/testing/mock_node.rs | 22 ++++++---- core/market/tests/test_agreement.rs | 4 +- .../framework-basic/src/mocks/net.rs | 42 +++++++++++-------- 3 files changed, 41 insertions(+), 27 deletions(-) diff --git a/core/market/src/testing/mock_node.rs b/core/market/src/testing/mock_node.rs index 4901d1f3f3..00daf288d4 100644 --- a/core/market/src/testing/mock_node.rs +++ b/core/market/src/testing/mock_node.rs @@ -34,7 +34,7 @@ use crate::testing::mock_node::default::*; #[cfg(feature = "bcast-singleton")] use ya_framework_basic::bcast::singleton::BCastService; -use ya_framework_basic::mocks::net::{gsb_prefixes, MockNet}; +use ya_framework_basic::mocks::net::{gsb_market_prefixes, gsb_prefixes, MockNet}; use ya_framework_basic::mocks::bcast::BCast; #[cfg(not(feature = "bcast-singleton"))] @@ -80,31 +80,32 @@ pub enum MockNodeKind { impl MockNodeKind { pub async fn bind_gsb(&self, test_name: &str, name: &str) -> Result { - let (public, local) = gsb_prefixes(test_name, name); + let (gsb_public, gsb_local) = gsb_prefixes(test_name, name); + let (market_public, market_local) = gsb_market_prefixes(&gsb_public, &gsb_local); match self { MockNodeKind::Market(market) => { - market.bind_gsb(&public, &local).await?; + market.bind_gsb(&market_public, &market_local).await?; market.matcher.discovery.bind_gsb_broadcast().await?; } MockNodeKind::Matcher { matcher, .. } => { - matcher.bind_gsb(&public, &local).await?; + matcher.bind_gsb(&market_public, &market_local).await?; matcher.discovery.bind_gsb_broadcast().await?; } MockNodeKind::Discovery(discovery) => { - discovery.bind_gsb(&public, &local).await?; + discovery.bind_gsb(&market_public, &market_local).await?; discovery.bind_gsb_broadcast().await?; } MockNodeKind::Negotiation { provider, requestor, } => { - provider.bind_gsb(&public, &local).await?; - requestor.bind_gsb(&public, &local).await?; + provider.bind_gsb(&market_public, &market_local).await?; + requestor.bind_gsb(&market_public, &market_local).await?; } } - Ok(public) + Ok(gsb_public) } } @@ -498,6 +499,11 @@ impl MarketsNetwork { pub fn node_gsb_prefixes(&self, node_name: &str) -> (String, String) { gsb_prefixes(&self.test_name, node_name) } + + pub fn market_gsb_prefixes(&self, node_name: &str) -> (String, String) { + let (gsb_public, gsb_local) = gsb_prefixes(&self.test_name, node_name); + gsb_market_prefixes(&gsb_public, &gsb_local) + } } fn test_data_dir() -> PathBuf { diff --git a/core/market/tests/test_agreement.rs b/core/market/tests/test_agreement.rs index b73a3b5cc4..4e958d26ca 100644 --- a/core/market/tests/test_agreement.rs +++ b/core/market/tests/test_agreement.rs @@ -54,7 +54,7 @@ async fn test_gsb_get_agreement() { .await .unwrap(); - let agreement = bus::service(network.node_gsb_prefixes(REQ_NAME).0) + let agreement = bus::service(network.market_gsb_prefixes(REQ_NAME).0) .send(market::GetAgreement { agreement_id: agreement_id.into_client(), role: Role::Requestor, @@ -102,7 +102,7 @@ async fn test_gsb_list_agreements() { .await .unwrap(); - let agreements = bus::service(network.node_gsb_prefixes(REQ_NAME).0) + let agreements = bus::service(network.market_gsb_prefixes(REQ_NAME).0) .send(market::ListAgreements::default()) .await .unwrap() diff --git a/test-utils/test-framework/framework-basic/src/mocks/net.rs b/test-utils/test-framework/framework-basic/src/mocks/net.rs index 4e00f46d4a..719a3f6855 100644 --- a/test-utils/test-framework/framework-basic/src/mocks/net.rs +++ b/test-utils/test-framework/framework-basic/src/mocks/net.rs @@ -23,7 +23,7 @@ pub struct MockNet { #[derive(Default)] struct MockNetInner { - /// Maps NodeIds to gsb prefixes of market nodes. + /// Maps NodeIds to gsb prefixes of other nodes. pub nodes: HashMap, } @@ -35,7 +35,7 @@ lazy_static::lazy_static! { impl Default for MockNet { fn default() -> Self { - log::debug!("getting singleton MockNet"); + log::debug!("Getting singleton MockNet"); (*NET).clone() } } @@ -129,29 +129,31 @@ impl MockNetInner { let topic = stub.topic; let endpoints = bcast.resolve(&caller, &topic); - log::debug!("BCasting on {} to {:?} from {}", topic, endpoints, caller); + log::debug!("BCasting on {topic} to {endpoints:?} from {caller}"); for endpoint in endpoints { - let addr = format!("{}/{}", endpoint, bcast_service_id); + let addr = format!("{endpoint}/{bcast_service_id}"); - let node_id = match mock_net.node_by_prefix(&addr) { + // Normal net would have additional step: Broadcast message would be sent to other node first on /net/{node_id}. + // Net would receive message, check topic and translate it to local addresses interested in this topic. + // Here for simplicity we are skipping those additional steps and directly sending to all endpoints waiting for broadcast. + // + // But since all broadcast handlers are bound on `/local` and all addresses registered in net are on `/public`, + // we must replace `local` -> `public` to find NodeId of receiver. + let addr_local = addr.replacen("local", "public", 1); + + let node_id = match mock_net.node_by_prefix(&addr_local) { Some(node_id) => node_id, None => { log::debug!( - "Not broadcasting on topic {} to {}. Node not found on list. \ - Probably networking was disabled for this Node.", - topic, - addr + "Not broadcasting on topic {topic} to {addr}. Node not found on list. \ + Probably networking was disabled for this Node." ); continue; } }; - log::debug!( - "BCasting on {} to address: {}, node: [{}]", - topic, - addr, - node_id - ); + log::debug!("BCasting on {topic} to address: {addr}, node: [{node_id}]"); + let caller = caller.clone(); let msg = msg.clone(); tokio::task::spawn_local(async move { @@ -233,7 +235,13 @@ pub(crate) const FROM_UDP_BUS_ID: &str = "/udp/from"; pub(crate) const FROM_TRANSFER_BUS_ID: &str = "/transfer/from"; pub fn gsb_prefixes(test_name: &str, name: &str) -> (String, String) { - let public_gsb_prefix = format!("/{}/{}/market", test_name, name); - let local_gsb_prefix = format!("/{}/{}/market", test_name, name); + let public_gsb_prefix = format!("/{}/{}/public", test_name, name); + let local_gsb_prefix = format!("/{}/{}/local", test_name, name); + (public_gsb_prefix, local_gsb_prefix) +} + +pub fn gsb_market_prefixes(public: &str, local: &str) -> (String, String) { + let public_gsb_prefix = format!("{}/market", public); + let local_gsb_prefix = format!("{}/market", local); (public_gsb_prefix, local_gsb_prefix) } From 6d6dbf156603ce43e65ca52c99a3c38487f9db03 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 2 Aug 2024 12:10:54 +0200 Subject: [PATCH 064/125] Move mocks to separate library --- Cargo.lock | 51 +++++++++++++++++-- Cargo.toml | 6 +-- core/payment/Cargo.toml | 1 + core/payment/tests/test_release_allocation.rs | 8 ++- .../tests/tutorial_how_to_use_module_tests.rs | 9 ++-- .../test-framework/framework-mocks/Cargo.toml | 51 +++++++++++++++++++ .../framework-mocks/src}/identity.rs | 0 .../test-framework/framework-mocks/src/lib.rs | 0 .../framework-mocks/src}/market.rs | 0 .../framework-mocks/src}/node.rs | 7 +-- .../framework-mocks/src}/payment.rs | 0 11 files changed, 113 insertions(+), 20 deletions(-) create mode 100644 test-utils/test-framework/framework-mocks/Cargo.toml rename {core/payment/tests/mocks => test-utils/test-framework/framework-mocks/src}/identity.rs (100%) rename core/payment/tests/mocks/mod.rs => test-utils/test-framework/framework-mocks/src/lib.rs (100%) rename {core/payment/tests/mocks => test-utils/test-framework/framework-mocks/src}/market.rs (100%) rename {core/payment/tests/mocks => test-utils/test-framework/framework-mocks/src}/node.rs (98%) rename {core/payment/tests/mocks => test-utils/test-framework/framework-mocks/src}/payment.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 803fb07eca..c51cf6fae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3861,11 +3861,11 @@ checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.5.2", + "spin 0.9.8", ] [[package]] @@ -9065,6 +9065,50 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ya-framework-mocks" +version = "0.1.0" +dependencies = [ + "actix-http", + "actix-rt", + "actix-service", + "actix-web", + "anyhow", + "async-trait", + "chrono", + "derive_more", + "ethsign", + "futures 0.3.30", + "lazy_static", + "log", + "portpicker", + "rand 0.8.5", + "regex", + "serde", + "serde_json", + "test-context", + "tokio", + "url", + "uuid 1.8.0", + "ya-agreement-utils", + "ya-client", + "ya-client-model", + "ya-core-model", + "ya-dummy-driver", + "ya-erc20-driver", + "ya-framework-basic", + "ya-identity", + "ya-market", + "ya-net", + "ya-payment", + "ya-persistence", + "ya-sb-router", + "ya-service-api", + "ya-service-api-web", + "ya-service-bus", + "ya-utils-path", +] + [[package]] name = "ya-gsb-api" version = "0.1.0" @@ -9442,6 +9486,7 @@ dependencies = [ "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", + "ya-framework-mocks", "ya-identity", "ya-market", "ya-metrics", diff --git a/Cargo.toml b/Cargo.toml index 7633d67f93..b5b3c6f713 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -230,6 +230,7 @@ members = [ "test-utils/test-framework", "test-utils/test-framework/framework-macro", "test-utils/test-framework/framework-basic", + "test-utils/test-framework/framework-mocks", ] [workspace.dependencies] @@ -239,10 +240,6 @@ members = [ derive_more = "0.99.11" erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "594d8a93b820b177ec267ccec5de61d8a9a45203" } erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "594d8a93b820b177ec267ccec5de61d8a9a45203" } -#erc20_payment_lib = { path = "../../payments/erc20_payment_lib/crates/erc20_payment_lib" } -#erc20_processor = { path = "../../payments/erc20_payment_lib" } -#erc20_payment_lib = { version = "=0.4.1" } -#erc20_processor = { version = "=0.4.1" } gftp = { version = "0.4.0", path = "core/gftp" } hex = "0.4.3" libsqlite3-sys = { version = "0.26.0", features = ["bundled"] } @@ -330,6 +327,7 @@ ya-manifest-test-utils = { path = "utils/manifest-utils/test-utils" } ya-test-framework = { path = "test-utils/test-framework" } ya-framework-macro = { path = "test-utils/test-framework/framework-macro" } ya-framework-basic = { path = "test-utils/test-framework/framework-basic" } +ya-framework-mocks = { path = "test-utils/test-framework/framework-mocks" } ethereum-tx-sign = { git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195" } graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407" } diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index e9bbda1235..69c4900b8d 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -73,6 +73,7 @@ ya-identity = "0.3" ya-market = "0.4" ya-sb-router = { workspace = true } ya-framework-basic = { version = "0.1" } +ya-framework-mocks = { version = "0.1" } actix-rt = "2.7" rand = "0.8" diff --git a/core/payment/tests/test_release_allocation.rs b/core/payment/tests/test_release_allocation.rs index 25991692b5..1e2e9764d8 100644 --- a/core/payment/tests/test_release_allocation.rs +++ b/core/payment/tests/test_release_allocation.rs @@ -2,7 +2,6 @@ use bigdecimal::BigDecimal; use chrono::Utc; use test_context::test_context; -use crate::mocks::market::FakeMarket; use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; @@ -10,10 +9,9 @@ use ya_framework_basic::log::enable_logs; use ya_framework_basic::mocks::net::MockNet; use ya_framework_basic::{resource, temp_dir}; -use crate::mocks::node::MockNode; -use crate::mocks::payment::Driver; - -mod mocks; +use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::Driver; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 07330b54e3..fc1956225f 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -1,5 +1,6 @@ use bigdecimal::BigDecimal; use test_context::test_context; + use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::NewAllocation; @@ -8,11 +9,9 @@ use ya_framework_basic::log::enable_logs; use ya_framework_basic::mocks::net::MockNet; use ya_framework_basic::{resource, temp_dir}; -use crate::mocks::market::FakeMarket; -use crate::mocks::node::MockNode; -use crate::mocks::payment::Driver; - -mod mocks; +use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::Driver; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml new file mode 100644 index 0000000000..d1429a4ef0 --- /dev/null +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "ya-framework-mocks" +version = "0.1.0" +edition = "2021" + +[features] +bcast-singleton = [] + + +[dependencies] +ya-agreement-utils = { workspace = true } +ya-client-model = "0.6" +ya-client = "0.8" +ya-core-model = { version = "^0.9" } +ya-dummy-driver = "0.3" +ya-erc20-driver = "0.4" +ya-identity = "0.3" +ya-market = "0.4" +ya-net = { version = "0.3", features = ["service"] } +ya-payment = "0.3" +ya-persistence = "0.3" +ya-service-api = "0.1" +ya-service-api-web = "0.2" +ya-service-bus = { workspace = true } +ya-sb-router = { workspace = true } +ya-utils-path = { version = "0.1", path = "../../../utils/path" } + +ya-framework-basic = { version = "0.1" } + +actix-rt = "2.7" +actix-web = "4" +actix-http = "3" +actix-service = "2" +anyhow = "1.0" +async-trait = "0.1.80" +chrono = "0.4" +derive_more = { workspace = true } +ethsign = "0.8" +futures = "0.3" +lazy_static = "1.5" +log = "0.4" +portpicker = "0.1" +rand = "0.8" +regex = "1.5" +serde = "1.0" +serde_json = "1.0" +test-context = "0.1.4" +tokio = "1" +uuid = "1" +url = "2.5" + diff --git a/core/payment/tests/mocks/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs similarity index 100% rename from core/payment/tests/mocks/identity.rs rename to test-utils/test-framework/framework-mocks/src/identity.rs diff --git a/core/payment/tests/mocks/mod.rs b/test-utils/test-framework/framework-mocks/src/lib.rs similarity index 100% rename from core/payment/tests/mocks/mod.rs rename to test-utils/test-framework/framework-mocks/src/lib.rs diff --git a/core/payment/tests/mocks/market.rs b/test-utils/test-framework/framework-mocks/src/market.rs similarity index 100% rename from core/payment/tests/mocks/market.rs rename to test-utils/test-framework/framework-mocks/src/market.rs diff --git a/core/payment/tests/mocks/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs similarity index 98% rename from core/payment/tests/mocks/node.rs rename to test-utils/test-framework/framework-mocks/src/node.rs index 3c238b256b..f8226ae065 100644 --- a/core/payment/tests/mocks/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -1,6 +1,5 @@ #![allow(dead_code)] -use crate::mocks::market::FakeMarket; use actix_web::{middleware, App, HttpServer, Scope}; use anyhow::anyhow; use std::fs; @@ -8,6 +7,7 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::Duration; use url::Url; + use ya_client::payment::PaymentApi; use ya_client::web::WebClient; use ya_framework_basic::async_drop::DroppableTestContext; @@ -16,8 +16,9 @@ use ya_service_api_web::middleware::auth; use ya_service_api_web::middleware::cors::{AppKeyCors, CorsConfig}; use ya_service_api_web::rest_api_host_port; -use super::identity::MockIdentity; -use super::payment::MockPayment; +use crate::identity::MockIdentity; +use crate::market::FakeMarket; +use crate::payment::MockPayment; /// Represents Node abstraction in tests. /// Provides functionality to instantiate selected modules and make tests setup easier. diff --git a/core/payment/tests/mocks/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs similarity index 100% rename from core/payment/tests/mocks/payment.rs rename to test-utils/test-framework/framework-mocks/src/payment.rs From f605aae123a12496d39464c52ff0c591d9e6de2b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 2 Aug 2024 17:13:55 +0200 Subject: [PATCH 065/125] Move MockNet to ya-framework-mocks --- Cargo.lock | 1 + core/market/Cargo.toml | 2 + core/market/src/testing/mock_node.rs | 29 +-- core/market/tests/test_agreement.rs | 51 ++-- .../tests/test_agreement_cancellation.rs | 11 +- core/market/tests/test_agreement_events.rs | 13 +- core/market/tests/test_agreement_reason.rs | 3 +- core/market/tests/test_agreement_rejection.rs | 9 +- core/market/tests/test_app_session_id.rs | 13 +- core/market/tests/test_cleaner.rs | 19 +- core/market/tests/test_cyclic_broadcasts.rs | 11 +- core/market/tests/test_initial_proposal.rs | 25 +- core/market/tests/test_negotiations.rs | 37 +-- core/market/tests/test_offer_broadcast.rs | 15 +- core/market/tests/test_proposal.rs | 7 +- core/market/tests/test_resolver.rs | 9 +- core/market/tests/test_rest_api.rs | 27 +- core/market/tests/test_subscription.rs | 7 +- core/payment/tests/test_release_allocation.rs | 5 +- .../tests/tutorial_how_to_use_module_tests.rs | 4 +- .../framework-basic/src/mocks.rs | 1 - .../framework-basic/src/mocks/bcast.rs | 49 ---- .../framework-basic/src/mocks/net.rs | 241 +---------------- .../test-framework/framework-mocks/Cargo.toml | 3 - .../framework-mocks/src/identity.rs | 26 +- .../test-framework/framework-mocks/src/lib.rs | 1 + .../framework-mocks/src/market.rs | 6 +- .../test-framework/framework-mocks/src/net.rs | 242 ++++++++++++++++++ .../src/net/bcast.rs} | 29 +-- .../framework-mocks/src/node.rs | 4 +- .../framework-mocks/src/payment.rs | 2 - 31 files changed, 446 insertions(+), 456 deletions(-) delete mode 100644 test-utils/test-framework/framework-basic/src/mocks/bcast.rs create mode 100644 test-utils/test-framework/framework-mocks/src/net.rs rename test-utils/test-framework/{framework-basic/src/mocks/bcast/singleton.rs => framework-mocks/src/net/bcast.rs} (75%) diff --git a/Cargo.lock b/Cargo.lock index c51cf6fae1..778a3d147b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9321,6 +9321,7 @@ dependencies = [ "ya-core-model", "ya-diesel-utils", "ya-framework-basic", + "ya-framework-mocks", "ya-market-resolver", "ya-net", "ya-persistence", diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index e19e1acef9..92d0f43eb0 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -73,5 +73,7 @@ serial_test = { git = "https://github.com/golemfactory/serial_test.git", branch structopt = "0.3" tokio = { version = "1", features = ["macros", "rt"] } +ya-framework-mocks = "0.1" + [lints] workspace = true \ No newline at end of file diff --git a/core/market/src/testing/mock_node.rs b/core/market/src/testing/mock_node.rs index 00daf288d4..b06f0a68f2 100644 --- a/core/market/src/testing/mock_node.rs +++ b/core/market/src/testing/mock_node.rs @@ -32,19 +32,16 @@ use crate::protocol::negotiation::messages::*; use crate::testing::mock_identity::MockIdentity; use crate::testing::mock_node::default::*; -#[cfg(feature = "bcast-singleton")] -use ya_framework_basic::bcast::singleton::BCastService; -use ya_framework_basic::mocks::net::{gsb_market_prefixes, gsb_prefixes, MockNet}; - -use ya_framework_basic::mocks::bcast::BCast; -#[cfg(not(feature = "bcast-singleton"))] -use ya_framework_basic::mocks::bcast::BCastService; +use ya_framework_basic::mocks::net::{gsb_market_prefixes, gsb_prefixes, IMockNet}; /// Instantiates market test nodes inside one process. pub struct MarketsNetwork { + net: Box, nodes: Vec, + test_dir: PathBuf, test_name: String, + config: Arc, } @@ -123,7 +120,7 @@ impl MarketsNetwork { /// Remember that test_name should be unique between all tests. /// It will be used to create directories and GSB binding points, /// to avoid potential name clashes. - pub async fn new(test_name: Option<&str>) -> Self { + pub async fn new(test_name: Option<&str>, net: impl IMockNet + 'static) -> Self { std::env::set_var("RUST_LOG", "debug"); let _ = env_logger::builder().try_init(); @@ -133,11 +130,13 @@ impl MarketsNetwork { }; let test_name = test_name.map(String::from).unwrap_or_else(gen_test_name); - log::info!("Intializing MarketsNetwork. tn={}", test_name); + log::info!("Initializing MarketsNetwork. tn={}", test_name); - MockNet::default().bind_gsb(); + let net = Box::new(net); + net.bind_gsb(); MarketsNetwork { + net, nodes: vec![], test_dir: prepare_test_dir(&test_name).unwrap(), test_name, @@ -167,8 +166,8 @@ impl MarketsNetwork { let node_id = node.mock_identity.get_default_id().identity; log::info!("Creating mock node {}: [{}].", name, &node_id); - BCastService::default().register(&node_id, &self.test_name); - MockNet::default().register_node(&node_id, &public_gsb_prefix); + self.net.register_for_broadcasts(&node_id, &self.test_name); + self.net.register_node(&node_id, &public_gsb_prefix); self.nodes.push(node); self @@ -176,7 +175,7 @@ impl MarketsNetwork { pub fn break_networking_for(&self, node_name: &str) -> Result<()> { for (_, id) in self.list_ids(node_name) { - MockNet::default().unregister_node(&id.identity)? + self.net.unregister_node(&id.identity)? } Ok(()) } @@ -184,7 +183,7 @@ impl MarketsNetwork { pub fn enable_networking_for(&self, node_name: &str) -> Result<()> { for (_, id) in self.list_ids(node_name) { let (public_gsb_prefix, _) = gsb_prefixes(&self.test_name, node_name); - MockNet::default().register_node(&id.identity, &public_gsb_prefix); + self.net.register_node(&id.identity, &public_gsb_prefix); } Ok(()) } @@ -430,7 +429,7 @@ impl MarketsNetwork { let (public_gsb_prefix, _) = gsb_prefixes(&self.test_name, node_name); - MockNet::default().register_node(&id.identity, &public_gsb_prefix); + self.net.register_node(&id.identity, &public_gsb_prefix); id } diff --git a/core/market/tests/test_agreement.rs b/core/market/tests/test_agreement.rs index 4e958d26ca..3f6fbfa13b 100644 --- a/core/market/tests/test_agreement.rs +++ b/core/market/tests/test_agreement.rs @@ -3,6 +3,7 @@ use chrono::{Duration, Utc}; use ya_client::model::market::Role; use ya_core_model::market; +use ya_framework_mocks::net::MockNet; use ya_market::assert_err_eq; use ya_market::testing::{ agreement_utils::{gen_reason, negotiate_agreement}, @@ -22,7 +23,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_gsb_get_agreement() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -71,7 +72,7 @@ async fn test_gsb_get_agreement() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_gsb_list_agreements() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -116,7 +117,7 @@ async fn test_gsb_list_agreements() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_get_agreement() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -149,7 +150,7 @@ async fn test_get_agreement() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_get_not_existing_agreement() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -183,7 +184,7 @@ async fn test_rest_get_not_existing_agreement() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn full_market_interaction_aka_happy_path() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -262,7 +263,7 @@ async fn full_market_interaction_aka_happy_path() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn second_creation_should_fail() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -293,7 +294,7 @@ async fn second_creation_should_fail() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn second_confirmation_should_fail() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -343,7 +344,7 @@ async fn second_confirmation_should_fail() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn agreement_expired_before_confirmation() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -388,7 +389,7 @@ async fn agreement_expired_before_confirmation() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn agreement_expired_before_approval() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -431,7 +432,7 @@ async fn agreement_expired_before_approval() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn waiting_wo_confirmation_should_fail() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -464,7 +465,7 @@ async fn waiting_wo_confirmation_should_fail() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn approval_before_confirmation_should_fail() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -505,7 +506,7 @@ async fn approval_before_confirmation_should_fail() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn approval_without_waiting_should_pass() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -555,7 +556,7 @@ async fn approval_without_waiting_should_pass() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn waiting_after_approval_should_pass() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -611,7 +612,7 @@ async fn waiting_after_approval_should_pass() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn second_approval_should_fail() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -683,7 +684,7 @@ async fn second_approval_should_fail() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn second_waiting_should_pass() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -746,7 +747,7 @@ async fn second_waiting_should_pass() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn net_err_while_confirming() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -787,7 +788,7 @@ async fn net_err_while_confirming() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn net_err_while_approving() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -845,7 +846,7 @@ async fn net_err_while_approving() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn cant_promote_requestor_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -890,7 +891,7 @@ async fn cant_promote_requestor_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn cant_promote_initial_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -935,7 +936,7 @@ async fn cant_promote_initial_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn cant_promote_not_last_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -979,7 +980,7 @@ async fn cant_promote_not_last_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_terminate() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -1005,7 +1006,7 @@ async fn test_terminate() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_terminate_not_existing_agreement() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -1050,7 +1051,7 @@ async fn test_terminate_not_existing_agreement() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_terminate_from_wrong_states() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -1149,7 +1150,7 @@ async fn test_terminate_from_wrong_states() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_terminate_rejected_agreement() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -1217,7 +1218,7 @@ async fn test_terminate_rejected_agreement() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_terminate_invalid_reason() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await diff --git a/core/market/tests/test_agreement_cancellation.rs b/core/market/tests/test_agreement_cancellation.rs index bec682f401..efb47bf577 100644 --- a/core/market/tests/test_agreement_cancellation.rs +++ b/core/market/tests/test_agreement_cancellation.rs @@ -5,6 +5,7 @@ use tokio::time::timeout; use ya_client::model::market::agreement::State as ClientAgreementState; use ya_client::model::market::{AgreementEventType, Reason}; +use ya_framework_mocks::net::MockNet; use ya_market::assert_err_eq; use ya_market::testing::{ agreement_utils::{gen_reason, negotiate_agreement}, @@ -19,7 +20,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_cancelled() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -73,7 +74,7 @@ async fn test_agreement_cancelled() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_cancel_agreement_in_wrong_state() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -150,7 +151,7 @@ async fn test_cancel_agreement_in_wrong_state() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_cancelled_wait_for_approval() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -221,7 +222,7 @@ async fn test_agreement_cancelled_wait_for_approval() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_simultaneous_reject_cancel() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -344,7 +345,7 @@ async fn test_agreement_simultaneous_reject_cancel() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_simultaneous_approve_cancel() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await diff --git a/core/market/tests/test_agreement_events.rs b/core/market/tests/test_agreement_events.rs index d28ed94007..aeb4a20b68 100644 --- a/core/market/tests/test_agreement_events.rs +++ b/core/market/tests/test_agreement_events.rs @@ -8,6 +8,7 @@ use ya_market::testing::{ApprovalStatus, Owner}; use ya_client::model::market::agreement_event::AgreementTerminator; use ya_client::model::market::AgreementEventType; +use ya_framework_mocks::net::MockNet; const REQ_NAME: &str = "Node-1"; const PROV_NAME: &str = "Node-2"; @@ -15,7 +16,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_approved_event() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -110,7 +111,7 @@ async fn test_agreement_approved_event() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_events_and_wait_for_approval() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -195,7 +196,7 @@ async fn test_agreement_events_and_wait_for_approval() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_terminated_event() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -288,7 +289,7 @@ async fn test_agreement_terminated_event() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_waiting_for_agreement_event() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -339,7 +340,7 @@ async fn test_waiting_for_agreement_event() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_rejected_event() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -437,7 +438,7 @@ async fn test_agreement_rejected_event() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_cancelled_event() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await diff --git a/core/market/tests/test_agreement_reason.rs b/core/market/tests/test_agreement_reason.rs index fa6ee6e756..cef1cd8518 100644 --- a/core/market/tests/test_agreement_reason.rs +++ b/core/market/tests/test_agreement_reason.rs @@ -2,6 +2,7 @@ use chrono::{Duration, Utc}; use ya_client::model::market::agreement_event::AgreementTerminator; use ya_client::model::market::{AgreementEventType, Reason}; +use ya_framework_mocks::net::MockNet; use ya_market::assert_err_eq; use ya_market::testing::{ proposal_util::exchange_draft_proposals, AgreementError, ApprovalStatus, MarketsNetwork, Owner, @@ -13,7 +14,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_get_agreement_termination_reason() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await diff --git a/core/market/tests/test_agreement_rejection.rs b/core/market/tests/test_agreement_rejection.rs index c537791abf..002d09153d 100644 --- a/core/market/tests/test_agreement_rejection.rs +++ b/core/market/tests/test_agreement_rejection.rs @@ -3,6 +3,7 @@ use chrono::{Duration, Utc}; use ya_client::model::market::agreement::State as ClientAgreementState; use ya_client::model::market::{AgreementEventType, Reason}; +use ya_framework_mocks::net::MockNet; use ya_market::assert_err_eq; use ya_market::testing::{ agreement_utils::{gen_reason, negotiate_agreement}, @@ -17,7 +18,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_rejected() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -76,7 +77,7 @@ async fn test_agreement_rejected() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_agreement_rejected_wait_for_approval() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -143,7 +144,7 @@ async fn test_agreement_rejected_wait_for_approval() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_reject_agreement_in_wrong_state() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -221,7 +222,7 @@ async fn test_reject_agreement_in_wrong_state() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_reject_rejected_agreement() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await diff --git a/core/market/tests/test_app_session_id.rs b/core/market/tests/test_app_session_id.rs index 3e01c19ebd..e646b5c7c7 100644 --- a/core/market/tests/test_app_session_id.rs +++ b/core/market/tests/test_app_session_id.rs @@ -7,6 +7,7 @@ use ya_market::testing::MarketsNetwork; use ya_market::testing::Owner; use ya_client::model::market::AgreementEventType; +use ya_framework_mocks::net::MockNet; const REQ_NAME: &str = "Node-1"; const PROV_NAME: &str = "Node-2"; @@ -16,7 +17,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_session_events_filtering() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -192,7 +193,7 @@ async fn test_session_events_filtering() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_session_should_be_independent_on_both_sides() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -247,7 +248,7 @@ async fn test_session_should_be_independent_on_both_sides() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_session_negotiation_on_the_same_node() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node") .await; @@ -302,7 +303,7 @@ async fn test_session_negotiation_on_the_same_node() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_session_negotiation_on_the_same_node_same_session() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node") .await; @@ -358,7 +359,7 @@ async fn test_session_negotiation_on_the_same_node_same_session() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_session_timestamp_filtering() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -548,7 +549,7 @@ async fn test_session_timestamp_filtering() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_common_event_flow() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await diff --git a/core/market/tests/test_cleaner.rs b/core/market/tests/test_cleaner.rs index 7b1db860c3..eed1525272 100644 --- a/core/market/tests/test_cleaner.rs +++ b/core/market/tests/test_cleaner.rs @@ -1,6 +1,7 @@ use chrono::{Duration, NaiveDateTime, Utc}; use std::ops::Not; use structopt::StructOpt; +use ya_framework_mocks::net::MockNet; use ya_market::testing::cleaner::clean; use ya_market::testing::dao::TestingDao; @@ -32,7 +33,7 @@ async fn test_agreement() { let _ = env_logger::builder().try_init(); let valid_agreement = generate_agreement(1, future()); let expired_agreement = generate_agreement(2, past()); - let db = MarketsNetwork::new(None) + let db = MarketsNetwork::new(None, MockNet::new()) .await .init_database("test_agreement"); let agreement_dao = db.as_dao::(); @@ -79,7 +80,9 @@ async fn test_demand() { "c76161077d0343ab85ac986eb5f6ea38-edb0016d9f8bafb54540da34f05a8d510de8114488f23916276bdead05509a54", past(), ); - let db = MarketsNetwork::new(None).await.init_database("test_demand"); + let db = MarketsNetwork::new(None, MockNet::new()) + .await + .init_database("test_demand"); let demand_dao = db.as_dao::(); demand_dao.insert(&valid_demand).await.unwrap(); demand_dao.insert(&expired_demand).await.unwrap(); @@ -102,7 +105,9 @@ async fn test_offer() { "c76161077d0343ab85ac986eb5f6ea38-edb0016d9f8bafb54540da34f05a8d510de8114488f23916276bdead05509a54", past(), ); - let db = MarketsNetwork::new(None).await.init_database("test_offer"); + let db = MarketsNetwork::new(None, MockNet::new()) + .await + .init_database("test_offer"); let offer_dao = db.as_dao::(); let validation_ts = (Utc::now() - Duration::days(100)).naive_utc(); offer_dao @@ -124,7 +129,9 @@ async fn test_offer() { #[serial_test::serial] async fn test_events() { // insert two events - let db = MarketsNetwork::new(None).await.init_database("test_events"); + let db = MarketsNetwork::new(None, MockNet::new()) + .await + .init_database("test_events"); let valid_event = generate_event(1, future()); let expired_event = generate_event(2, past()); >::raw_insert(&db.ram_db.pool, valid_event.clone()) @@ -146,7 +153,7 @@ async fn test_events() { #[serial_test::serial] async fn test_proposal() { let _ = env_logger::builder().try_init(); - let db = MarketsNetwork::new(None) + let db = MarketsNetwork::new(None, MockNet::new()) .await .init_database("test_proposal"); let valid_negotiation = generate_negotiation(None); @@ -204,7 +211,7 @@ async fn test_proposal_lotsa_negotiations() { // Due to diesel limitations we have to take care of processing // big amount of negotiations (manually) #672 let _ = env_logger::builder().try_init(); - let db = MarketsNetwork::new(None) + let db = MarketsNetwork::new(None, MockNet::new()) .await .init_database("test_proposal_lotsa_negotiations"); let mut expired_negotiations: Vec = vec![]; diff --git a/core/market/tests/test_cyclic_broadcasts.rs b/core/market/tests/test_cyclic_broadcasts.rs index f13c3e4325..7ec5ec4877 100644 --- a/core/market/tests/test_cyclic_broadcasts.rs +++ b/core/market/tests/test_cyclic_broadcasts.rs @@ -1,5 +1,6 @@ use rand::seq::SliceRandom; use std::time::Duration; +use ya_framework_mocks::net::MockNet; use ya_market::assert_err_eq; use ya_market::testing::{ @@ -16,7 +17,7 @@ use ya_market::testing::{ async fn test_startup_offers_sharing() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -69,7 +70,7 @@ async fn test_startup_offers_sharing() { async fn test_unsubscribes_cyclic_broadcasts() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -173,7 +174,7 @@ async fn test_unsubscribes_cyclic_broadcasts() { #[serial_test::serial] async fn test_network_error_while_subscribing() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -210,7 +211,7 @@ async fn test_network_error_while_subscribing() { async fn test_sharing_someones_else_offers() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -263,7 +264,7 @@ async fn test_sharing_someones_else_offers() { async fn test_sharing_someones_else_unsubscribes() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await diff --git a/core/market/tests/test_initial_proposal.rs b/core/market/tests/test_initial_proposal.rs index 2afbc2f345..fd06e7d7c8 100644 --- a/core/market/tests/test_initial_proposal.rs +++ b/core/market/tests/test_initial_proposal.rs @@ -11,6 +11,7 @@ use ya_market::MarketService; use chrono::Utc; use std::sync::Arc; use std::time::Duration; +use ya_framework_mocks::net::MockNet; const REQ_NAME: &str = "Node-1"; const PROV_NAME: &str = "Node-2"; @@ -19,7 +20,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_query_events_non_existent_subscription() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -38,7 +39,7 @@ async fn test_query_events_non_existent_subscription() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_query_initial_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -85,7 +86,7 @@ async fn test_query_initial_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_query_multiple_events() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -139,7 +140,7 @@ async fn test_query_multiple_events() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_query_events_timeout() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -184,7 +185,7 @@ async fn test_query_events_timeout() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_query_events_unsubscribe_notification() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -230,7 +231,7 @@ async fn test_query_events_unsubscribe_notification() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_query_events_edge_cases() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -299,7 +300,7 @@ async fn test_query_events_edge_cases() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_query_events_for_multiple_subscriptions() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -351,7 +352,7 @@ async fn test_query_events_for_multiple_subscriptions() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_simultaneous_query_events() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -430,7 +431,7 @@ async fn test_simultaneous_query_events() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_unsubscribe_demand_while_query_events_for_other() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -492,7 +493,7 @@ async fn test_unsubscribe_demand_while_query_events_for_other() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_counter_initial_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -563,7 +564,7 @@ async fn test_counter_initial_proposal() { // #[cfg_attr(not(feature = "test-suite"), ignore)] // #[serial_test::serial] // async fn test_respawn_proposal_after_terminate_by_requestor() -> anyhow::Result<()> { -// let network = MarketsNetwork::new(None) +// let network = MarketsNetwork::new(None, MockNet::new()) // .await // .add_market_instance(REQ_NAME) // .await @@ -622,7 +623,7 @@ async fn test_counter_initial_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_respawn_proposal_after_terminate_by_provider() -> anyhow::Result<()> { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await diff --git a/core/market/tests/test_negotiations.rs b/core/market/tests/test_negotiations.rs index 4113dca7e1..b096724343 100644 --- a/core/market/tests/test_negotiations.rs +++ b/core/market/tests/test_negotiations.rs @@ -1,5 +1,6 @@ use chrono::{Duration, Utc}; use ya_client::model::market::{proposal::State, RequestorEvent}; +use ya_framework_mocks::net::MockNet; use ya_market::testing::{ agreement_utils::gen_reason, events_helper::{provider, requestor, ClientProposalHelper}, @@ -16,7 +17,7 @@ use ya_market::testing::{ #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_exchanging_draft_proposals() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -136,7 +137,7 @@ async fn test_exchanging_draft_proposals() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_counter_countered_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -221,7 +222,7 @@ async fn test_counter_countered_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_counter_own_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -314,7 +315,7 @@ async fn test_counter_own_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_counter_unsubscribed_demand() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -371,7 +372,7 @@ async fn test_counter_unsubscribed_demand() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_counter_unsubscribed_offer() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -442,7 +443,7 @@ async fn test_counter_unsubscribed_offer() { #[ignore] #[serial_test::serial] async fn test_counter_initial_unsubscribed_remote_offer() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -505,7 +506,7 @@ async fn test_counter_initial_unsubscribed_remote_offer() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_counter_draft_unsubscribed_remote_offer() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -562,7 +563,7 @@ async fn test_counter_draft_unsubscribed_remote_offer() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_counter_draft_unsubscribed_remote_demand() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -621,7 +622,7 @@ async fn test_counter_draft_unsubscribed_remote_demand() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_not_matching_counter_demand() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -656,7 +657,7 @@ async fn test_not_matching_counter_demand() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_not_matching_counter_offer() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -711,7 +712,7 @@ async fn test_not_matching_counter_offer() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_reject_negotiations_same_identity() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -743,7 +744,7 @@ async fn test_reject_negotiations_same_identity() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_reject_initial_offer() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Req-1") .await @@ -795,7 +796,7 @@ async fn test_reject_initial_offer() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_reject_demand() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Req-1") .await @@ -872,7 +873,7 @@ async fn test_reject_demand() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_proposal_events_last() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -950,7 +951,7 @@ async fn test_proposal_events_last() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_restart_negotiations() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Requestor1") .await @@ -1074,7 +1075,7 @@ async fn test_restart_negotiations() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_create_agreement_on_rejected_proposal_should_fail() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Requestor1") .await @@ -1150,7 +1151,7 @@ async fn test_create_agreement_on_rejected_proposal_should_fail() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_negotiations_after_agreement_rejected() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Requestor1") .await @@ -1245,7 +1246,7 @@ async fn test_negotiations_after_agreement_rejected() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_reject_initial_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await diff --git a/core/market/tests/test_offer_broadcast.rs b/core/market/tests/test_offer_broadcast.rs index 9fd10afecb..bbfbbe7617 100644 --- a/core/market/tests/test_offer_broadcast.rs +++ b/core/market/tests/test_offer_broadcast.rs @@ -4,6 +4,7 @@ use std::str::FromStr; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use tokio::time::Duration; +use ya_framework_mocks::net::MockNet; use ya_market::assert_err_eq; use ya_market::testing::discovery::{message::*, Discovery}; @@ -18,7 +19,7 @@ use ya_market::testing::{QueryOfferError, SubscriptionId}; #[serial_test::serial] async fn test_broadcast_offer() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -75,7 +76,7 @@ async fn test_broadcast_offer() { #[serial_test::serial] async fn test_broadcast_offer_callbacks() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -121,7 +122,7 @@ async fn test_broadcast_offer_callbacks() { #[serial_test::serial] async fn test_broadcast_offer_id_validation() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -163,7 +164,7 @@ async fn test_broadcast_offer_id_validation() { #[serial_test::serial] async fn test_broadcast_expired_offer() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -212,7 +213,7 @@ async fn test_broadcast_expired_offer() { #[serial_test::serial] async fn test_broadcast_stop_conditions() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -312,7 +313,7 @@ async fn test_broadcast_stop_conditions() { #[serial_test::serial] async fn test_discovery_get_offers() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -355,7 +356,7 @@ async fn test_discovery_get_offers() { // #[serial_test::serial] // async fn test_broadcast_50k() { // let _ = env_logger::builder().try_init(); -// let network = MarketsNetwork::new(None) +// let network = MarketsNetwork::new(None, MockNet::new()) // .await // .add_market_instance("Node-1") // .await; diff --git a/core/market/tests/test_proposal.rs b/core/market/tests/test_proposal.rs index 76790ac043..04bc568790 100644 --- a/core/market/tests/test_proposal.rs +++ b/core/market/tests/test_proposal.rs @@ -11,11 +11,12 @@ use tokio::time::Duration; use ya_client::model::market::proposal::State; use ya_client::model::market::RequestorEvent; use ya_core_model::NodeId; +use ya_framework_mocks::net::MockNet; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_get_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Requestor1") .await @@ -58,7 +59,7 @@ async fn test_get_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_get_proposal_not_found() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Requestor1") .await @@ -90,7 +91,7 @@ async fn test_get_proposal_not_found() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_proposal_random_shuffle() { - let mut network = MarketsNetwork::new(None) + let mut network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; diff --git a/core/market/tests/test_resolver.rs b/core/market/tests/test_resolver.rs index 3a7a7c9ff5..09397c9d57 100644 --- a/core/market/tests/test_resolver.rs +++ b/core/market/tests/test_resolver.rs @@ -1,5 +1,6 @@ use std::{future::Future, time::Duration}; use tokio::time::{timeout, Timeout}; +use ya_framework_mocks::net::MockNet; use ya_market::testing::{ client::{sample_demand, sample_offer}, @@ -12,7 +13,7 @@ use ya_market::testing::{ async fn test_single_not_resolve_offer() { // given let _ = env_logger::builder().try_init(); - let mut network = MarketsNetwork::new(None) + let mut network = MarketsNetwork::new(None, MockNet::new()) .await .add_matcher_instance("Node-1") .await; @@ -35,7 +36,7 @@ async fn test_single_not_resolve_offer() { async fn test_resolve_offer_demand() { // given let _ = env_logger::builder().try_init(); - let mut network = MarketsNetwork::new(None) + let mut network = MarketsNetwork::new(None, MockNet::new()) .await .add_matcher_instance("Provider-1") .await @@ -78,7 +79,7 @@ async fn test_resolve_offer_demand() { async fn test_single_not_resolve_demand() { // given let _ = env_logger::builder().try_init(); - let mut network = MarketsNetwork::new(None) + let mut network = MarketsNetwork::new(None, MockNet::new()) .await .add_matcher_instance("Node-1") .await; @@ -101,7 +102,7 @@ async fn test_single_not_resolve_demand() { async fn test_resolve_2xoffer_demand() { // given let _ = env_logger::builder().try_init(); - let mut network = MarketsNetwork::new(None) + let mut network = MarketsNetwork::new(None, MockNet::new()) .await .add_matcher_instance("Provider-1") .await diff --git a/core/market/tests/test_rest_api.rs b/core/market/tests/test_rest_api.rs index ad61116a70..c103edd303 100644 --- a/core/market/tests/test_rest_api.rs +++ b/core/market/tests/test_rest_api.rs @@ -10,6 +10,7 @@ use ya_client::model::market::{ }; use ya_client::model::ErrorMessage; use ya_client::web::QueryParamsBuilder; +use ya_framework_mocks::net::MockNet; use ya_market::testing::agreement_utils::negotiate_agreement; use ya_market::testing::events_helper::requestor::expect_approve; use ya_market::testing::{ @@ -27,7 +28,7 @@ const PROV_NAME: &str = "Node-2"; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_get_offers() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -90,7 +91,7 @@ async fn test_rest_get_offers() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_get_demands() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -119,7 +120,7 @@ async fn test_rest_get_demands() { #[serial_test::serial] async fn test_rest_invalid_subscription_id_should_return_400() { // given - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -150,7 +151,7 @@ async fn test_rest_invalid_subscription_id_should_return_400() { #[serial_test::serial] async fn test_rest_subscribe_unsubscribe_offer() { // given - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -212,7 +213,7 @@ async fn test_rest_subscribe_unsubscribe_offer() { #[serial_test::serial] async fn test_rest_subscribe_unsubscribe_demand() { // given - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -276,7 +277,7 @@ async fn test_rest_subscribe_unsubscribe_demand() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_get_proposal() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Provider") .await @@ -334,7 +335,7 @@ async fn test_rest_get_proposal() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_get_agreement() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -374,7 +375,7 @@ async fn test_rest_get_agreement() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_query_agreement_events() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await @@ -429,7 +430,7 @@ async fn test_rest_query_agreement_events() { #[serial_test::serial] async fn test_terminate_agreement() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -492,7 +493,7 @@ async fn test_terminate_agreement() { #[serial_test::serial] async fn test_terminate_agreement_without_reason() { let _ = env_logger::builder().try_init(); - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -552,7 +553,7 @@ async fn test_terminate_agreement_without_reason() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_agreement_rejected() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -614,7 +615,7 @@ async fn test_rest_agreement_rejected() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_rest_agreement_cancelled() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance(REQ_NAME) .await @@ -676,7 +677,7 @@ async fn test_rest_agreement_cancelled() { // #[actix_rt::test] // #[serial_test::serial] // async fn test_rest_get_proposal_wrong_subscription() { -// let network = MarketsNetwork::new(None) +// let network = MarketsNetwork::new(None, MockNet::new()) // .await // .add_market_instance("Node-1") // .await.unwrap() diff --git a/core/market/tests/test_subscription.rs b/core/market/tests/test_subscription.rs index 24f277cb5a..4bd7be3d6e 100644 --- a/core/market/tests/test_subscription.rs +++ b/core/market/tests/test_subscription.rs @@ -1,3 +1,4 @@ +use ya_framework_mocks::net::MockNet; use ya_market::assert_err_eq; use ya_market::testing::client::{sample_demand, sample_offer}; use ya_market::testing::mock_offer::flatten_json; @@ -9,7 +10,7 @@ use ya_market::testing::{MarketServiceExt, MarketsNetwork}; #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_subscribe_offer() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; @@ -20,7 +21,7 @@ async fn test_subscribe_offer() { let offer = sample_offer(); let subscription_id = market1.subscribe_offer(&offer, &identity1).await.unwrap(); - // Offer should be available in database after subscribe. + // Offer should be available in dataFailed to terminate agreementbase after subscribe. let got_offer = market1.get_offer(&subscription_id).await.unwrap(); let client_offer = got_offer.into_client_offer().unwrap(); assert_eq!(client_offer.offer_id, subscription_id.to_string()); @@ -52,7 +53,7 @@ async fn test_subscribe_offer() { #[cfg_attr(not(feature = "test-suite"), ignore)] #[serial_test::serial] async fn test_subscribe_demand() { - let network = MarketsNetwork::new(None) + let network = MarketsNetwork::new(None, MockNet::new()) .await .add_market_instance("Node-1") .await; diff --git a/core/payment/tests/test_release_allocation.rs b/core/payment/tests/test_release_allocation.rs index 1e2e9764d8..b1b00a4ca2 100644 --- a/core/payment/tests/test_release_allocation.rs +++ b/core/payment/tests/test_release_allocation.rs @@ -6,10 +6,9 @@ use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; -use ya_framework_basic::mocks::net::MockNet; use ya_framework_basic::{resource, temp_dir}; - use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::net::{IMockNet, MockNet}; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::Driver; @@ -21,7 +20,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu let dir = temp_dir!("test_release_allocation")?; - let net = MockNet::default(); + let net = MockNet::new(); net.bind_gsb(); let node = MockNode::new(net, "node-1", dir.path()) diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index fc1956225f..fa6fec3f16 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -6,10 +6,10 @@ use ya_client_model::payment::NewAllocation; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; -use ya_framework_basic::mocks::net::MockNet; use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::net::{IMockNet, MockNet}; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::Driver; @@ -25,7 +25,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Currently instantiating many MockNodes is not possible, but MockNet is necessary even // for communication on the same node, because messages directed to external GSB addresses `/net/0x437544...` // when NodeId belongs to local Node, need to be routed back. - let net = MockNet::default(); + let net = MockNet::new(); net.bind_gsb(); // Create MockNode which is container for all Golem modules and represents diff --git a/test-utils/test-framework/framework-basic/src/mocks.rs b/test-utils/test-framework/framework-basic/src/mocks.rs index ea422ac3e1..f9faf2ff65 100644 --- a/test-utils/test-framework/framework-basic/src/mocks.rs +++ b/test-utils/test-framework/framework-basic/src/mocks.rs @@ -1,2 +1 @@ -pub mod bcast; pub mod net; diff --git a/test-utils/test-framework/framework-basic/src/mocks/bcast.rs b/test-utils/test-framework/framework-basic/src/mocks/bcast.rs deleted file mode 100644 index ffaaaceed0..0000000000 --- a/test-utils/test-framework/framework-basic/src/mocks/bcast.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Broadcast support service -// Note: This file is derived from core/net module. It serves only as mock -// so we don't have to keep it compatible. -// It was moved here, because this file is not expected to be public in net module. -use std::cell::RefCell; -use std::collections::BTreeMap; -use std::sync::Arc; - -use ya_client::model::NodeId; -use ya_core_model::net; - -pub mod singleton; - -pub trait BCast: Clone { - /// registers node to be visible only in specific subnet - fn register(&self, _node_id: &NodeId, _subnet: &str) {} - /// subscribes endpoint to topic; endpoint prefix is // - fn add(&self, subscribe: net::local::Subscribe); - /// returns all nodes with same subnet as given node subscribed to given topic - fn resolve(&self, node_id: &str, topic: &str) -> Vec>; -} - -#[derive(Clone, Default)] -pub struct BCastService { - inner: Arc>, -} - -#[derive(Default)] -struct BCastServiceInner { - topics: BTreeMap>>, -} - -impl BCast for BCastService { - fn add(&self, subscribe: net::local::Subscribe) { - let mut me = self.inner.borrow_mut(); - me.topics - .entry(subscribe.topic().to_owned()) - .or_default() - .push(subscribe.endpoint().into()) - } - - fn resolve(&self, _node_id: &str, topic: &str) -> Vec> { - let me = self.inner.borrow(); - me.topics - .get(topic) - .map(|receivers| receivers.to_vec()) - .unwrap_or_default() - } -} diff --git a/test-utils/test-framework/framework-basic/src/mocks/net.rs b/test-utils/test-framework/framework-basic/src/mocks/net.rs index 719a3f6855..ef3bdccda9 100644 --- a/test-utils/test-framework/framework-basic/src/mocks/net.rs +++ b/test-utils/test-framework/framework-basic/src/mocks/net.rs @@ -1,239 +1,24 @@ use anyhow::Result; -use std::collections::HashMap; -use std::rc::Rc; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; -use ya_client::model::NodeId; +use ya_client_model::NodeId; use ya_core_model::net; -use ya_core_model::net::{local as local_net, local::SendBroadcastMessage}; -use ya_net::hybrid::testing::{parse_from_to_addr, parse_net_to_addr}; -use ya_service_bus::{serialization, typed as bus, untyped as local_bus, Error, RpcMessage}; -#[cfg(feature = "bcast-singleton")] -use super::bcast::singleton::BCastService; -use super::bcast::BCast; -#[cfg(not(feature = "bcast-singleton"))] -use super::bcast::BCastService; -use ya_core_model::net::local::SendBroadcastStub; - -#[derive(Clone)] -pub struct MockNet { - inner: Arc>, -} - -#[derive(Default)] -struct MockNetInner { - /// Maps NodeIds to gsb prefixes of other nodes. - pub nodes: HashMap, -} - -lazy_static::lazy_static! { - static ref NET : MockNet = MockNet { - inner: Arc::new(Mutex::new(MockNetInner::default())) - }; -} - -impl Default for MockNet { - fn default() -> Self { - log::debug!("Getting singleton MockNet"); - (*NET).clone() - } -} - -impl MockNet { - pub fn bind_gsb(&self) { - let inner = self.inner.lock().unwrap(); - inner.bind_gsb() - } - - pub fn register_node(&self, node_id: &NodeId, prefix: &str) { - log::info!("[MockNet] Registering node {node_id} at prefix: {prefix}"); - - let mut inner = self.inner.lock().unwrap(); - if inner.nodes.insert(*node_id, prefix.to_string()).is_some() { - panic!("[MockNet] Node [{}] already existed.", node_id); - } - } - - pub fn unregister_node(&self, node_id: &NodeId) -> Result<()> { - let mut inner = self.inner.lock().unwrap(); - inner - .nodes - .remove(node_id) - .map(|_| ()) - .ok_or_else(|| anyhow::anyhow!("node not registered: {}", node_id)) - } - - fn translate_to(&self, id: NodeId, addr: &str) -> Result { - let prefix = self.node_prefix(id)?; - let net_prefix = format!("/net/{}", id); - Ok(addr.replacen(&net_prefix, &prefix, 1)) - } - - fn node_prefix(&self, id: NodeId) -> Result { - let inner = self.inner.lock().unwrap(); - inner - .nodes - .get(&id) - .cloned() - .ok_or_else(|| anyhow::anyhow!("Node not registered: {id}")) - } - - pub fn node_by_prefix(&self, address: &str) -> Option { - let inner = self.inner.lock().unwrap(); - for (id, prefix) in inner.nodes.iter() { - if address.contains(prefix) { - return Some(*id); - } - } - None - } -} - -// TODO: all tests using this mock net implementation should be run sequentially -// because GSB router is a static singleton (shared state) and consecutive bindings -// for same addr (ie. local_net::BUS_ID) are being overwritten and only last is effective -// which means there might be interlace in BCastService instances being used -// `bcast::singleton` is a try to handle it, but unsuccessful yet -impl MockNetInner { - pub fn bind_gsb(&self) { - let bcast = BCastService::default(); - log::info!("initializing BCast on mock net"); - - let bcast_service_id = as RpcMessage>::ID; - - let bcast1 = bcast.clone(); - let _ = bus::bind(local_net::BUS_ID, move |subscribe: local_net::Subscribe| { - let bcast = bcast1.clone(); - async move { - log::debug!("subscribing BCast: {:?}", subscribe); - bcast.add(subscribe); - Ok(0) // ignored id - } - }); - - let addr = format!("{}/{}", local_net::BUS_ID, bcast_service_id); - let resp: Rc<[u8]> = serialization::to_vec(&Ok::<(), ()>(())).unwrap().into(); - let _ = local_bus::subscribe( - &addr, - move |caller: &str, _addr: &str, msg: &[u8]| { - let mock_net = MockNet::default(); - let resp = resp.clone(); - let bcast = bcast.clone(); - - let stub: SendBroadcastStub = serialization::from_slice(msg).unwrap(); - let caller = caller.to_string(); - - let msg = msg.to_vec(); - - let topic = stub.topic; - let endpoints = bcast.resolve(&caller, &topic); - - log::debug!("BCasting on {topic} to {endpoints:?} from {caller}"); - for endpoint in endpoints { - let addr = format!("{endpoint}/{bcast_service_id}"); - - // Normal net would have additional step: Broadcast message would be sent to other node first on /net/{node_id}. - // Net would receive message, check topic and translate it to local addresses interested in this topic. - // Here for simplicity we are skipping those additional steps and directly sending to all endpoints waiting for broadcast. - // - // But since all broadcast handlers are bound on `/local` and all addresses registered in net are on `/public`, - // we must replace `local` -> `public` to find NodeId of receiver. - let addr_local = addr.replacen("local", "public", 1); - - let node_id = match mock_net.node_by_prefix(&addr_local) { - Some(node_id) => node_id, - None => { - log::debug!( - "Not broadcasting on topic {topic} to {addr}. Node not found on list. \ - Probably networking was disabled for this Node." - ); - continue; - } - }; - - log::debug!("BCasting on {topic} to address: {addr}, node: [{node_id}]"); - - let caller = caller.clone(); - let msg = msg.clone(); - tokio::task::spawn_local(async move { - let _ = local_bus::send(addr.as_ref(), &caller, msg.as_ref()).await; - }); - } - async move { Ok(Vec::from(resp.as_ref())) } - }, - (), - ); - - Self::bind_local_bus(MockNet::default(), FROM_BUS_ID, from_address_resolver); - Self::bind_local_bus(MockNet::default(), FROM_UDP_BUS_ID, from_address_resolver); - Self::bind_local_bus( - MockNet::default(), - FROM_TRANSFER_BUS_ID, - from_address_resolver, - ); - - Self::bind_local_bus(MockNet::default(), net::BUS_ID, net_address_resolver); - Self::bind_local_bus(MockNet::default(), net::BUS_ID_UDP, net_address_resolver); - Self::bind_local_bus( - MockNet::default(), - net::BUS_ID_TRANSFER, - net_address_resolver, - ); - } - - fn bind_local_bus(net: MockNet, address: &'static str, resolver: F) - where - F: Fn(&str, &str) -> anyhow::Result<(String, NodeId, String)> + 'static, - { - let resolver = Arc::new(resolver); - - local_bus::subscribe( - address, - move |caller: &str, addr: &str, msg: &[u8]| { - let mock_net = net.clone(); - let data = Vec::from(msg); - let caller = caller.to_string(); - let addr = addr.to_string(); - let resolver_ = resolver.clone(); - - async move { - log::info!("[MockNet] Received message from [{caller}], on address [{addr}]."); - - let (from, to, address) = resolver_(&caller, &addr) - .map_err(|e| Error::GsbBadRequest(e.to_string()))?; - let translated = mock_net - .translate_to(to, &address) - .map_err(|e| Error::GsbBadRequest(e.to_string()))?; - - log::info!( - "[MockNet] Sending message from [{from}], to: [{to}], address [{translated}]." - ); - local_bus::send(&translated, &from.to_string(), &data).await - } - }, - // TODO: Implement stream handler - (), - ); - } -} - -fn from_address_resolver(_caller: &str, addr: &str) -> anyhow::Result<(String, NodeId, String)> { - let (from, to, addr) = - parse_from_to_addr(addr).map_err(|e| anyhow::anyhow!("invalid address: {}", e))?; - Ok((from.to_string(), to, addr)) +pub trait IMockBroadcast { + /// Registers node to be visible only in specific subnet + fn register_for_broadcasts(&self, _node_id: &NodeId, _subnet: &str); + /// Subscribes endpoint to topic; endpoint prefix is // + fn subscribe_topic(&self, subscribe: net::local::Subscribe); + /// Returns all nodes with same subnet as given node subscribed to given topic + fn resolve(&self, node_id: &str, topic: &str) -> Vec>; } -fn net_address_resolver(caller: &str, addr: &str) -> anyhow::Result<(String, NodeId, String)> { - let (to, addr) = - parse_net_to_addr(addr).map_err(|e| anyhow::anyhow!("invalid address: {}", e))?; - Ok((caller.to_string(), to, addr)) +pub trait IMockNet: IMockBroadcast { + fn bind_gsb(&self); + fn register_node(&self, node_id: &NodeId, prefix: &str); + fn unregister_node(&self, node_id: &NodeId) -> Result<()>; } -pub(crate) const FROM_BUS_ID: &str = "/from"; -pub(crate) const FROM_UDP_BUS_ID: &str = "/udp/from"; -pub(crate) const FROM_TRANSFER_BUS_ID: &str = "/transfer/from"; - pub fn gsb_prefixes(test_name: &str, name: &str) -> (String, String) { let public_gsb_prefix = format!("/{}/{}/public", test_name, name); let local_gsb_prefix = format!("/{}/{}/local", test_name, name); diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index d1429a4ef0..3c3c4c65bd 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -3,9 +3,6 @@ name = "ya-framework-mocks" version = "0.1.0" edition = "2021" -[features] -bcast-singleton = [] - [dependencies] ya-agreement-utils = { workspace = true } diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index d6e64e5ba8..eae25f7d1f 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -1,20 +1,19 @@ -#![allow(dead_code)] - use anyhow::{anyhow, bail}; use futures::FutureExt; use serde::de::DeserializeOwned; use std::fs; use std::path::Path; + use ya_client_model::NodeId; use ya_core_model::appkey::AppKey; use ya_core_model::identity::IdentityInfo; -use ya_framework_basic::mocks::net::MockNet; use ya_identity::cli::{AppKeyCommand, IdentityCommand}; - use ya_identity::service::Identity; use ya_persistence::executor::DbExecutor; use ya_service_api::{CliCtx, CommandOutput}; +use crate::net::{IMockNet, MockNet}; + #[derive(Clone)] pub struct MockIdentity { net: MockNet, @@ -54,8 +53,6 @@ impl MockIdentity { .create_appkey(name, identity.node_id) .await .map_err(|e| anyhow!("Creating AppKey: {e}"))?; - - self.register_identity_in_net(identity.node_id); Ok(appkey) } @@ -75,8 +72,6 @@ impl MockIdentity { .create_appkey(&name, identity.node_id) .await .map_err(|e| anyhow!("Creating AppKey: {e}"))?; - - self.register_identity_in_net(identity.node_id); Ok(appkey) } @@ -94,7 +89,6 @@ impl MockIdentity { } pub async fn create_identity(&self, name: &str) -> anyhow::Result { - let ctx = CliCtx::default(); let command = IdentityCommand::Create { no_password: true, alias: Some(name.to_string()), @@ -103,7 +97,7 @@ impl MockIdentity { from_private_key: None, }; - parse_output_result::(command.run_command(&ctx).boxed_local().await?) + self.run_create_identity(command).await } pub async fn load_identity( @@ -111,7 +105,6 @@ impl MockIdentity { name: &str, private_key: String, ) -> anyhow::Result { - let ctx = CliCtx::default(); let command = IdentityCommand::Create { no_password: true, alias: Some(name.to_string()), @@ -120,7 +113,16 @@ impl MockIdentity { from_private_key: Some(private_key), }; - parse_output_result::(command.run_command(&ctx).boxed_local().await?) + self.run_create_identity(command).await + } + + async fn run_create_identity(&self, command: IdentityCommand) -> anyhow::Result { + let ctx = CliCtx::default(); + let identity = + parse_output_result::(command.run_command(&ctx).boxed_local().await?)?; + + self.register_identity_in_net(identity.node_id); + Ok(identity) } pub async fn create_appkey(&self, name: &str, id: NodeId) -> anyhow::Result { diff --git a/test-utils/test-framework/framework-mocks/src/lib.rs b/test-utils/test-framework/framework-mocks/src/lib.rs index cffc8e639f..21ed5ff75b 100644 --- a/test-utils/test-framework/framework-mocks/src/lib.rs +++ b/test-utils/test-framework/framework-mocks/src/lib.rs @@ -1,4 +1,5 @@ pub mod identity; pub mod market; +pub mod net; pub mod node; pub mod payment; diff --git a/test-utils/test-framework/framework-mocks/src/market.rs b/test-utils/test-framework/framework-mocks/src/market.rs index 374d762291..3c3e790682 100644 --- a/test-utils/test-framework/framework-mocks/src/market.rs +++ b/test-utils/test-framework/framework-mocks/src/market.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use chrono::{Duration, Utc}; use serde_json::json; use std::collections::HashMap; @@ -23,7 +21,7 @@ use ya_service_bus::typed as bus; #[derive(Clone)] pub struct FakeMarket { name: String, - testdir: PathBuf, + _testdir: PathBuf, inner: Arc>, } @@ -36,7 +34,7 @@ impl FakeMarket { pub fn new(name: &str, testdir: &Path) -> Self { FakeMarket { name: name.to_string(), - testdir: testdir.to_path_buf(), + _testdir: testdir.to_path_buf(), inner: Arc::new(RwLock::new(FakeMarketInner { agreements: HashMap::new(), })), diff --git a/test-utils/test-framework/framework-mocks/src/net.rs b/test-utils/test-framework/framework-mocks/src/net.rs new file mode 100644 index 0000000000..1edd9d683d --- /dev/null +++ b/test-utils/test-framework/framework-mocks/src/net.rs @@ -0,0 +1,242 @@ +use anyhow::Result; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::{Arc, Mutex}; + +use ya_client::model::NodeId; +use ya_core_model::net; +use ya_core_model::net::local::{SendBroadcastStub, Subscribe}; +use ya_core_model::net::{local as local_net, local::SendBroadcastMessage}; +pub use ya_framework_basic::mocks::net::{IMockBroadcast, IMockNet}; +use ya_net::hybrid::testing::{parse_from_to_addr, parse_net_to_addr}; +use ya_service_bus::{serialization, typed as bus, untyped as local_bus, Error, RpcMessage}; + +use bcast::BCastService; + +pub mod bcast; + +#[derive(Clone)] +pub struct MockNet { + inner: Arc>, + broadcast: BCastService, +} + +#[derive(Default)] +struct MockNetInner { + /// Maps NodeIds to gsb prefixes of other nodes. + pub nodes: HashMap, +} + +impl Default for MockNet { + fn default() -> Self { + MockNet::new() + } +} + +impl IMockBroadcast for MockNet { + fn register_for_broadcasts(&self, node_id: &NodeId, subnet: &str) { + self.broadcast.register_for_broadcasts(node_id, subnet) + } + + fn subscribe_topic(&self, subscribe: Subscribe) { + self.broadcast.subscribe_topic(subscribe) + } + + fn resolve(&self, node_id: &str, topic: &str) -> Vec> { + self.broadcast.resolve(node_id, topic) + } +} + +impl IMockNet for MockNet { + fn bind_gsb(&self) { + self.bind_gsb_inner() + } + + fn register_node(&self, node_id: &NodeId, prefix: &str) { + log::info!("[MockNet] Registering node {node_id} at prefix: {prefix}"); + + let mut inner = self.inner.lock().unwrap(); + if inner.nodes.insert(*node_id, prefix.to_string()).is_some() { + panic!("[MockNet] Node [{}] already existed.", node_id); + } + } + + fn unregister_node(&self, node_id: &NodeId) -> Result<()> { + let mut inner = self.inner.lock().unwrap(); + inner + .nodes + .remove(node_id) + .map(|_| ()) + .ok_or_else(|| anyhow::anyhow!("node not registered: {}", node_id)) + } +} + +// TODO: all tests using this mock net implementation should be run sequentially +// because GSB router is a static singleton (shared state) and consecutive bindings +// for same addr (ie. local_net::BUS_ID) are being overwritten and only last is effective +// which means there might be interlace in BCastService instances being used +// `bcast::singleton` is a try to handle it, but unsuccessful yet +impl MockNet { + pub fn new() -> Self { + MockNet { + inner: Arc::new(Mutex::new(MockNetInner::default())), + broadcast: Default::default(), + } + } + + fn translate_to(&self, id: NodeId, addr: &str) -> Result { + let prefix = self.node_prefix(id)?; + let net_prefix = format!("/net/{}", id); + log::info!("Replacing {net_prefix} with {prefix} in {addr}"); + Ok(addr.replacen(&net_prefix, &prefix, 1)) + } + + fn node_prefix(&self, id: NodeId) -> Result { + let inner = self.inner.lock().unwrap(); + inner + .nodes + .get(&id) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Node not registered: {id}")) + } + + pub fn node_by_prefix(&self, address: &str) -> Option { + let inner = self.inner.lock().unwrap(); + for (id, prefix) in inner.nodes.iter() { + if address.contains(prefix) { + return Some(*id); + } + } + None + } + + fn bind_gsb_inner(&self) { + log::info!("initializing BCast on mock net"); + + let bcast_service_id = as RpcMessage>::ID; + + let bcast = self.broadcast.clone(); + let bcast1 = self.broadcast.clone(); + let _ = bus::bind(local_net::BUS_ID, move |subscribe: local_net::Subscribe| { + let bcast = bcast1.clone(); + async move { + log::debug!("subscribing BCast: {:?}", subscribe); + bcast.subscribe_topic(subscribe); + Ok(0) // ignored id + } + }); + + let mock_net = self.clone(); + + let addr = format!("{}/{}", local_net::BUS_ID, bcast_service_id); + let resp: Rc<[u8]> = serialization::to_vec(&Ok::<(), ()>(())).unwrap().into(); + let _ = local_bus::subscribe( + &addr, + move |caller: &str, _addr: &str, msg: &[u8]| { + let resp = resp.clone(); + let bcast = bcast.clone(); + + let stub: SendBroadcastStub = serialization::from_slice(msg).unwrap(); + let caller = caller.to_string(); + + let msg = msg.to_vec(); + + let topic = stub.topic; + let endpoints = bcast.resolve(&caller, &topic); + + log::debug!("BCasting on {topic} to {endpoints:?} from {caller}"); + for endpoint in endpoints { + let addr = format!("{endpoint}/{bcast_service_id}"); + + // Normal net would have additional step: Broadcast message would be sent to other node first on /net/{node_id}. + // Net would receive message, check topic and translate it to local addresses interested in this topic. + // Here for simplicity we are skipping those additional steps and directly sending to all endpoints waiting for broadcast. + // + // But since all broadcast handlers are bound on `/local` and all addresses registered in net are on `/public`, + // we must replace `local` -> `public` to find NodeId of receiver. + let addr_local = addr.replacen("local", "public", 1); + + let node_id = match mock_net.node_by_prefix(&addr_local) { + Some(node_id) => node_id, + None => { + log::debug!( + "Not broadcasting on topic {topic} to {addr}. Node not found on list. \ + Probably networking was disabled for this Node." + ); + continue; + } + }; + + log::debug!("BCasting on {topic} to address: {addr}, node: [{node_id}]"); + + let caller = caller.clone(); + let msg = msg.clone(); + tokio::task::spawn_local(async move { + let _ = local_bus::send(addr.as_ref(), &caller, msg.as_ref()).await; + }); + } + async move { Ok(Vec::from(resp.as_ref())) } + }, + (), + ); + + Self::bind_local_bus(self.clone(), FROM_BUS_ID, from_address_resolver); + Self::bind_local_bus(self.clone(), FROM_UDP_BUS_ID, from_address_resolver); + Self::bind_local_bus(self.clone(), FROM_TRANSFER_BUS_ID, from_address_resolver); + + Self::bind_local_bus(self.clone(), net::BUS_ID, net_address_resolver); + Self::bind_local_bus(self.clone(), net::BUS_ID_UDP, net_address_resolver); + Self::bind_local_bus(self.clone(), net::BUS_ID_TRANSFER, net_address_resolver); + } + + fn bind_local_bus(net: MockNet, address: &'static str, resolver: F) + where + F: Fn(&str, &str) -> anyhow::Result<(String, NodeId, String)> + 'static, + { + let resolver = Arc::new(resolver); + + local_bus::subscribe( + address, + move |caller: &str, addr: &str, msg: &[u8]| { + let mock_net = net.clone(); + let data = Vec::from(msg); + let caller = caller.to_string(); + let addr = addr.to_string(); + let resolver_ = resolver.clone(); + + async move { + log::info!("[MockNet] Received message from [{caller}], on address [{addr}]."); + + let (from, to, address) = resolver_(&caller, &addr) + .map_err(|e| Error::GsbBadRequest(e.to_string()))?; + let translated = mock_net + .translate_to(to, &address) + .map_err(|e| Error::GsbBadRequest(e.to_string()))?; + + log::info!( + "[MockNet] Sending message from [{from}], to: [{to}], address [{translated}]." + ); + local_bus::send(&translated, &from.to_string(), &data).await + } + }, + // TODO: Implement stream handler + (), + ); + } +} + +fn from_address_resolver(_caller: &str, addr: &str) -> anyhow::Result<(String, NodeId, String)> { + let (from, to, addr) = + parse_from_to_addr(addr).map_err(|e| anyhow::anyhow!("invalid address: {}", e))?; + Ok((from.to_string(), to, addr)) +} + +fn net_address_resolver(caller: &str, addr: &str) -> anyhow::Result<(String, NodeId, String)> { + let (to, addr) = + parse_net_to_addr(addr).map_err(|e| anyhow::anyhow!("invalid address: {}", e))?; + Ok((caller.to_string(), to, addr)) +} + +pub(crate) const FROM_BUS_ID: &str = "/from"; +pub(crate) const FROM_UDP_BUS_ID: &str = "/udp/from"; +pub(crate) const FROM_TRANSFER_BUS_ID: &str = "/transfer/from"; diff --git a/test-utils/test-framework/framework-basic/src/mocks/bcast/singleton.rs b/test-utils/test-framework/framework-mocks/src/net/bcast.rs similarity index 75% rename from test-utils/test-framework/framework-basic/src/mocks/bcast/singleton.rs rename to test-utils/test-framework/framework-mocks/src/net/bcast.rs index a91d6cd29c..00516e8246 100644 --- a/test-utils/test-framework/framework-basic/src/mocks/bcast/singleton.rs +++ b/test-utils/test-framework/framework-mocks/src/net/bcast.rs @@ -1,14 +1,10 @@ -//! Broadcast singleton support service -// Singleton version of BCast module in hope to solve problems with running -// multithreaded integration tests with singleton GSB router - use std::collections::btree_map::Entry::{Occupied, Vacant}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; -use super::BCast; use ya_client::model::NodeId; use ya_core_model::net::local as local_net; +use ya_framework_basic::mocks::net::IMockBroadcast; #[derive(Clone)] pub struct BCastService { @@ -21,21 +17,22 @@ struct BCastServiceInner { node_subnet: BTreeMap, } -lazy_static::lazy_static! { - static ref BCAST : BCastService = BCastService { - inner: Arc::new(Mutex::new(BCastServiceInner::default())) - }; -} - impl Default for BCastService { fn default() -> Self { - log::debug!("getting singleton mock BCast"); - (*BCAST).clone() + BCastService::new() + } +} + +impl BCastService { + pub fn new() -> Self { + Self { + inner: Arc::new(Mutex::new(Default::default())), + } } } -impl BCast for BCastService { - fn register(&self, node_id: &NodeId, subnet: &str) { +impl IMockBroadcast for BCastService { + fn register_for_broadcasts(&self, node_id: &NodeId, subnet: &str) { let mut me = self.inner.lock().unwrap(); log::info!("registering node {} within subnet: {}", node_id, subnet); @@ -48,7 +45,7 @@ impl BCast for BCastService { }; } - fn add(&self, subscribe: local_net::Subscribe) { + fn subscribe_topic(&self, subscribe: local_net::Subscribe) { let mut me = self.inner.lock().unwrap(); me.topics_endpoints .entry(subscribe.topic().to_owned()) diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index f8226ae065..08bfe56841 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use actix_web::{middleware, App, HttpServer, Scope}; use anyhow::anyhow; use std::fs; @@ -11,13 +9,13 @@ use url::Url; use ya_client::payment::PaymentApi; use ya_client::web::WebClient; use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::mocks::net::MockNet; use ya_service_api_web::middleware::auth; use ya_service_api_web::middleware::cors::{AppKeyCors, CorsConfig}; use ya_service_api_web::rest_api_host_port; use crate::identity::MockIdentity; use crate::market::FakeMarket; +use crate::net::MockNet; use crate::payment::MockPayment; /// Represents Node abstraction in tests. diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index 46194c7331..96d4f48844 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -1,5 +1,3 @@ -#![allow(dead_code)] - use anyhow::anyhow; use std::path::{Path, PathBuf}; use std::sync::Arc; From 76273383006e15b4095f9358a0fc70994228b50b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 2 Aug 2024 18:12:08 +0200 Subject: [PATCH 066/125] Create test from validate_allocation example --- ...lease_allocation.rs => test_allocation.rs} | 125 +++++++++++++++++- .../framework-mocks/src/identity.rs | 10 +- .../framework-mocks/src/node.rs | 6 +- .../framework-mocks/src/payment.rs | 14 +- 4 files changed, 144 insertions(+), 11 deletions(-) rename core/payment/tests/{test_release_allocation.rs => test_allocation.rs} (58%) diff --git a/core/payment/tests/test_release_allocation.rs b/core/payment/tests/test_allocation.rs similarity index 58% rename from core/payment/tests/test_release_allocation.rs rename to core/payment/tests/test_allocation.rs index b1b00a4ca2..dfb1c1fd86 100644 --- a/core/payment/tests/test_release_allocation.rs +++ b/core/payment/tests/test_allocation.rs @@ -2,8 +2,9 @@ use bigdecimal::BigDecimal; use chrono::Utc; use test_context::test_context; -use ya_client_model::payment::allocation::PaymentPlatformEnum; +use ya_client_model::payment::allocation::{PaymentPlatform, PaymentPlatformEnum}; use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; +use ya_core_model::payment::local::GetStatus; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; @@ -160,3 +161,125 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu log::info!(" 👍🏻 Example completed successfully ❤️"); Ok(()) } + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_validate_allocation(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_validate_allocation")?; + + let net = MockNet::new(); + net.bind_gsb(); + + let node = MockNode::new(net, "node-1", dir.path()) + .with_identity() + .with_payment() + .with_fake_market(); + node.bind_gsb().await?; + node.start_server(ctx).await?; + + let appkey_req = node + .get_identity()? + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; + + let requestor = node.rest_payments(&appkey_req.key)?; + + let payment = node.get_payment()?; + payment + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) + .await?; + + let payment_platform = PaymentPlatform { + driver: Some(Driver::Erc20.gsb_name()), + network: Some("holesky".to_string()), + token: Some("tglm".to_string()), + }; + + let status = payment + .gsb_local_endpoint() + .call(GetStatus { + address: appkey_req.identity.to_string(), + driver: payment_platform.driver.clone().unwrap(), + network: payment_platform.network.clone(), + //token: payment_platform.token.clone(), + token: None, + after_timestamp: 0, + }) + .await??; + + log::info!( + "Requestor balance: {}, platform: {:?}", + status.amount, + payment_platform + ); + + log::info!("Attempting to create allocation with invalid address..."); + let result = requestor + .create_allocation(&NewAllocation { + address: Some("Definitely not a valid address".to_string()), + payment_platform: Some(PaymentPlatformEnum::PaymentPlatform( + payment_platform.clone(), + )), + total_amount: BigDecimal::from(1u64), + timeout: None, + make_deposit: false, + deposit: None, + extend_timeout: None, + }) + .await; + assert!(result.is_err()); + log::info!("Failed to create allocation (as expected)."); + + let new_allocation = NewAllocation { + address: None, // Use default address (i.e. identity) + payment_platform: Some(PaymentPlatformEnum::PaymentPlatform( + payment_platform.clone(), + )), + total_amount: status.amount / 2, + timeout: None, + make_deposit: false, + deposit: None, + extend_timeout: None, + }; + + log::info!( + "Creating allocation for {} tGLM...", + &new_allocation.total_amount + ); + requestor.create_allocation(&new_allocation).await?; + log::info!("Allocation created."); + + log::info!( + "Creating another allocation for {} tGLM...", + &new_allocation.total_amount + ); + let allocation = requestor.create_allocation(&new_allocation).await?; + log::info!("Allocation created."); + + log::info!( + "Attempting to create another allocation for {} tGLM...", + &new_allocation.total_amount + ); + let result = requestor.create_allocation(&new_allocation).await; + assert!(result.is_err()); + log::info!("Failed to create allocation (as expected)."); + + log::info!("Releasing an allocation..."); + requestor + .release_allocation(&allocation.allocation_id) + .await?; + log::info!("Allocation released."); + + log::info!( + "Creating another allocation for {} tGLM...", + &new_allocation.total_amount + ); + requestor.create_allocation(&new_allocation).await?; + log::info!("Allocation created."); + + log::info!(" 👍🏻 Example completed successfully ❤️"); + Ok(()) +} diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index eae25f7d1f..f4e81d4ce4 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -22,8 +22,8 @@ pub struct MockIdentity { } impl MockIdentity { - pub fn new(net: MockNet, name: &str) -> Self { - let db = Self::create_db(&format!("{name}.identity.db")).unwrap(); + pub fn new(net: MockNet, testdir: &Path, name: &str) -> Self { + let db = Self::create_db(testdir, "identity.db").unwrap(); MockIdentity { net, @@ -32,9 +32,9 @@ impl MockIdentity { } } - fn create_db(name: &str) -> anyhow::Result { - let db = DbExecutor::in_memory(name) - .map_err(|e| anyhow!("Failed to create in memory db [{name:?}]. Error: {e}"))?; + fn create_db(testdir: &Path, name: &str) -> anyhow::Result { + let db = DbExecutor::from_data_dir(testdir, name) + .map_err(|e| anyhow!("Failed to create db [{name:?}]. Error: {e}"))?; Ok(db) } diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index 08bfe56841..e0be67334a 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -56,7 +56,11 @@ impl MockNode { /// Use full wrapped Identity module for this node. pub fn with_identity(mut self) -> Self { - self.identity = Some(MockIdentity::new(self.net.clone(), &self.name)); + self.identity = Some(MockIdentity::new( + self.net.clone(), + &self.testdir, + &self.name, + )); self } diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index 96d4f48844..17596ff146 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use ya_core_model::driver::{driver_bus_id, Fund}; +use ya_core_model::payment::local::BUS_ID; use ya_payment::api::web_scope; use ya_payment::config::Config; use ya_payment::migrations; @@ -10,6 +11,7 @@ use ya_payment::processor::PaymentProcessor; use ya_payment::service::BindOptions; use ya_persistence::executor::DbExecutor; use ya_service_bus::typed as bus; +use ya_service_bus::typed::Endpoint; use ya_dummy_driver as dummy; use ya_erc20_driver as erc20; @@ -42,7 +44,7 @@ pub struct MockPayment { impl MockPayment { pub fn new(name: &str, testdir: &Path) -> Self { - let db = Self::create_db(&format!("{name}.payment.db")).unwrap(); + let db = Self::create_db(testdir, "payment.db").unwrap(); let processor = Arc::new(PaymentProcessor::new(db.clone())); MockPayment { @@ -53,9 +55,9 @@ impl MockPayment { } } - fn create_db(name: &str) -> anyhow::Result { - let db = DbExecutor::in_memory(name) - .map_err(|e| anyhow!("Failed to create in memory db [{name:?}]. Error: {e}"))?; + fn create_db(testdir: &Path, name: &str) -> anyhow::Result { + let db = DbExecutor::from_data_dir(testdir, name) + .map_err(|e| anyhow!("Failed to create db [{name:?}]. Error: {e}"))?; db.apply_migration(migrations::run_with_output)?; Ok(db) } @@ -100,4 +102,8 @@ impl MockPayment { .await??; Ok(()) } + + pub fn gsb_local_endpoint(&self) -> Endpoint { + bus::service(BUS_ID) + } } From 5152381668e2ff0750882b4ad35274517abe69d6 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 2 Aug 2024 19:27:24 +0200 Subject: [PATCH 067/125] Create test from debit_note_flow example; Implement FakeActivity; Better tutorial description --- Cargo.lock | 2 +- core/payment/tests/test_allocation.rs | 5 +- core/payment/tests/test_debit_notes.rs | 220 ++++++++++++++++++ .../tests/tutorial_how_to_use_module_tests.rs | 52 ++++- .../test-framework/framework-mocks/Cargo.toml | 2 +- .../framework-mocks/src/activity.rs | 88 +++++++ .../test-framework/framework-mocks/src/lib.rs | 1 + .../test-framework/framework-mocks/src/net.rs | 3 +- .../framework-mocks/src/node.rs | 19 ++ 9 files changed, 381 insertions(+), 11 deletions(-) create mode 100644 core/payment/tests/test_debit_notes.rs create mode 100644 test-utils/test-framework/framework-mocks/src/activity.rs diff --git a/Cargo.lock b/Cargo.lock index 778a3d147b..e1683af4bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9089,7 +9089,7 @@ dependencies = [ "test-context", "tokio", "url", - "uuid 1.8.0", + "uuid 0.8.2", "ya-agreement-utils", "ya-client", "ya-client-model", diff --git a/core/payment/tests/test_allocation.rs b/core/payment/tests/test_allocation.rs index dfb1c1fd86..937dd88edc 100644 --- a/core/payment/tests/test_allocation.rs +++ b/core/payment/tests/test_allocation.rs @@ -17,7 +17,7 @@ use ya_framework_mocks::payment::Driver; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("test_release_allocation")?; @@ -166,7 +166,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_validate_allocation(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("test_validate_allocation")?; @@ -204,7 +204,6 @@ async fn test_validate_allocation(ctx: &mut DroppableTestContext) -> anyhow::Res address: appkey_req.identity.to_string(), driver: payment_platform.driver.clone().unwrap(), network: payment_platform.network.clone(), - //token: payment_platform.token.clone(), token: None, after_timestamp: 0, }) diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs new file mode 100644 index 0000000000..feeedccb5c --- /dev/null +++ b/core/payment/tests/test_debit_notes.rs @@ -0,0 +1,220 @@ +use bigdecimal::BigDecimal; +use chrono::Utc; +use std::time::Duration; +use test_context::test_context; + +use ya_client_model::payment::allocation::PaymentPlatformEnum; +use ya_client_model::payment::{Acceptance, DocumentStatus, NewAllocation, NewDebitNote}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::mocks::net::IMockNet; +use ya_framework_basic::{resource, temp_dir}; +use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::net::MockNet; +use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::Driver; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_debit_note_flow")?; + let dir = dir.path(); + + let net = MockNet::new(); + net.bind_gsb(); + + let node = MockNode::new(net, "node-1", &dir) + .with_identity() + .with_payment() + .with_fake_market() + .with_fake_activity(); + node.bind_gsb().await?; + node.start_server(ctx).await?; + + let appkey_prov = node.get_identity()?.create_identity_key("provider").await?; + let appkey_req = node + .get_identity()? + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; + + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node.get_market()?.add_agreement(agreement.clone()).await; + let activity_id = node + .get_activity()? + .create_activity(&agreement.agreement_id) + .await; + + let requestor = node.rest_payments(&appkey_req.key)?; + let provider = node.rest_payments(&appkey_prov.key)?; + + node.get_payment()? + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) + .await?; + + let app_session_id = Some("app_session_id".to_string()); + let payment_platform = + PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); + + let debit_note_date = Utc::now(); + let debit_note = NewDebitNote { + activity_id: activity_id.clone(), + total_amount_due: BigDecimal::from(1u64), + usage_counter_vector: None, + payment_due_date: Some(Utc::now()), + }; + log::info!( + "Issuing debit note (total amount due: {} GLM)...", + &debit_note.total_amount_due + ); + let debit_note = provider.issue_debit_note(&debit_note).await?; + log::info!("Debit note issued."); + + log::info!("Sending debit note..."); + provider.send_debit_note(&debit_note.debit_note_id).await?; + log::info!("Debit note sent."); + + let debit_note_events_received = requestor + .get_debit_note_events::( + Some(&debit_note_date), + Some(Duration::from_secs(10)), + None, + app_session_id.clone(), + ) + .await + .unwrap(); + log::debug!("events 1: {:?}", &debit_note_events_received); + log::debug!( + "DATE: {:?}", + Some(&debit_note_events_received.first().unwrap().event_date) + ); + + log::info!("Creating allocation..."); + let allocation = requestor + .create_allocation(&NewAllocation { + address: None, // Use default address (i.e. identity) + payment_platform: Some(payment_platform.clone()), + total_amount: BigDecimal::from(10u64), + make_deposit: false, + deposit: None, + timeout: None, + extend_timeout: None, + }) + .await?; + log::info!("Allocation created."); + + log::debug!( + "DEBIT_NOTES1: {:?}", + requestor.get_debit_notes::(None, None).await + ); + log::debug!( + "DEBIT_NOTES2: {:?}", + requestor + .get_debit_notes::(Some(debit_note_date), None) + .await + ); + log::debug!( + "DEBIT_NOTES3: {:?}", + requestor + .get_debit_notes::(Some(Utc::now()), None) + .await + ); + + log::info!("Accepting debit note..."); + let now = Utc::now(); + requestor + .accept_debit_note( + &debit_note.debit_note_id, + &Acceptance { + total_amount_accepted: debit_note.total_amount_due.clone(), + allocation_id: allocation.allocation_id.clone(), + }, + ) + .await?; + log::info!("Debit note accepted."); + + log::info!("Waiting for payment..."); + let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer + let mut payments = provider + .get_payments(Some(&now), timeout, None, None) + .await?; + assert_eq!(payments.len(), 1); + let payment = payments.pop().unwrap(); + assert_eq!(&payment.amount, &debit_note.total_amount_due); + log::info!("Payment verified correctly."); + + log::info!("Verifying debit note status..."); + let debit_note = provider.get_debit_note(&debit_note.debit_note_id).await?; + assert_eq!(debit_note.status, DocumentStatus::Settled); + log::info!("Debit note status verified correctly."); + + let debit_note2 = NewDebitNote { + activity_id: activity_id.clone(), + total_amount_due: BigDecimal::from(2u64), + usage_counter_vector: None, + payment_due_date: Some(Utc::now()), + }; + log::info!( + "Issuing debit note (total amount due: {} GLM)...", + debit_note2.total_amount_due + ); + let debit_note2 = provider.issue_debit_note(&debit_note2).await?; + log::info!("Debit note issued."); + + log::info!("Sending debit note..."); + provider.send_debit_note(&debit_note2.debit_note_id).await?; + log::info!("Debit note sent."); + + log::info!("Accepting debit note..."); + let now = Utc::now(); + requestor + .accept_debit_note( + &debit_note2.debit_note_id, + &Acceptance { + total_amount_accepted: debit_note2.total_amount_due.clone(), + allocation_id: allocation.allocation_id, + }, + ) + .await?; + log::info!("Debit note accepted."); + + log::info!("Waiting for payment..."); + let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer + let mut payments = provider + .get_payments(Some(&now), timeout, None, app_session_id.clone()) + .await?; + assert_eq!(payments.len(), 1); + let payment = payments.pop().unwrap(); + assert_eq!( + &payment.amount, + &(&debit_note2.total_amount_due - &debit_note.total_amount_due) + ); + log::info!("Payment verified correctly."); + + log::info!("Verifying debit note status..."); + let debit_note2 = provider.get_debit_note(&debit_note2.debit_note_id).await?; + assert_eq!(debit_note2.status, DocumentStatus::Settled); + log::info!("Debit note status verified correctly."); + + // Not implemented + // log::debug!( + // "get_payments_for_debit_note1: {:?}", + // requestor.get_payments_for_debit_note::(&debit_note2.debit_note_id, None, None).await + // ); + // log::debug!( + // "get_payments_for_debit_note2: {:?}", + // requestor + // .get_payments_for_debit_note::(&debit_note2.debit_note_id, Some(debit_note_date), None) + // .await + // ); + // log::debug!( + // "get_payments_for_debit_note3: {:?}", + // requestor.get_payments_for_debit_note::(&debit_note2.debit_note_id, Some(Utc::now()), None).await + // ); + + log::info!(" 👍🏻 Example completed successfully ❤️"); + Ok(()) +} diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index fa6fec3f16..7396f4cee0 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -1,8 +1,9 @@ use bigdecimal::BigDecimal; use test_context::test_context; -use ya_client_model::payment::allocation::PaymentPlatformEnum; +use ya_client_model::payment::allocation::{PaymentPlatform, PaymentPlatformEnum}; use ya_client_model::payment::NewAllocation; +use ya_core_model::payment::local::GetStatus; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; @@ -13,13 +14,24 @@ use ya_framework_mocks::net::{IMockNet, MockNet}; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::Driver; +// Tests should be always wrapped in these macros. +// `serial_test` forces sequential execution. It is nor possible to run them concurrently, because +// we bind to single GSB. +// `DroppableTestContext` is a helper struct which will clean up after tests (for example shutdown servers etc). #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { enable_logs(true); + // This line create temporary directory for test data, that will be removed after `TempDir` is dropped. + // Directory will be placed in cargo `target/tmp`. + // You can use `TempDir` library in 2 ways: + // - `dir.path()` to get path which can be used further in tests + // - `dir.into_path()` which steals PathBuf preventing temporary directory from removing on the end of the test. + // Use the second option during debugging your tests and switch back to the first one before merging PR. let dir = temp_dir!("tutorial_how_to_use_module_tests")?; + let dir = dir.path(); // MockNet routes traffic between MockNodes. // Currently instantiating many MockNodes is not possible, but MockNet is necessary even @@ -30,7 +42,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Create MockNode which is container for all Golem modules and represents // single node in tests. - let node = MockNode::new(net, "node-1", dir.path()) + let node = MockNode::new(net, "node-1", dir) // Request instantiating wrappers around real Golem modules. .with_identity() .with_payment() @@ -61,14 +73,21 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) .await?; + // Create REST API client for give node, to test payments endpoints. let api = node.rest_payments(&appkey_req.key)?; - let payment_platform = - PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); + let payment_platform = PaymentPlatform { + driver: Some(Driver::Erc20.gsb_name()), + network: Some("holesky".to_string()), + token: Some("tglm".to_string()), + }; + let _allocation = api .create_allocation(&NewAllocation { address: None, // Use default address (i.e. identity) - payment_platform: Some(payment_platform.clone()), + payment_platform: Some(PaymentPlatformEnum::PaymentPlatform( + payment_platform.clone(), + )), total_amount: BigDecimal::from(10u64), timeout: None, make_deposit: false, @@ -78,9 +97,32 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any .await .unwrap(); + // Some interactions will require mock Agreements between Requestor and Provider. let agreement = FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); node.get_market()?.add_agreement(agreement.clone()).await; + // Always use `gsb_local_endpoint()` to determine address on which to call GSB. + // GSB addresses can be different from regular yagna usage when we will be able to instantiate + // multiple MockNodes in single tests. + let status = node + .get_payment()? + .gsb_local_endpoint() + .call(GetStatus { + address: appkey_req.identity.to_string(), + driver: payment_platform.driver.clone().unwrap(), + network: payment_platform.network.clone(), + token: None, + after_timestamp: 0, + }) + .await??; + + log::info!( + "Requestor balance: {}, allocated: {}/{}", + status.amount, + status.reserved, + status.amount + ); + Ok(()) } diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index 3c3c4c65bd..73186003a3 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -43,6 +43,6 @@ serde = "1.0" serde_json = "1.0" test-context = "0.1.4" tokio = "1" -uuid = "1" +uuid = { version = "0.8", features = ["v4", "serde"] } url = "2.5" diff --git a/test-utils/test-framework/framework-mocks/src/activity.rs b/test-utils/test-framework/framework-mocks/src/activity.rs new file mode 100644 index 0000000000..74bee8adfa --- /dev/null +++ b/test-utils/test-framework/framework-mocks/src/activity.rs @@ -0,0 +1,88 @@ +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::sync::RwLock; +use uuid::Uuid; + +use ya_core_model::activity; +use ya_service_bus::typed as bus; + +#[derive(Clone)] +pub struct FakeActivity { + name: String, + _testdir: PathBuf, + + inner: Arc>, +} + +#[derive(Clone, Debug)] +pub struct Activity { + id: String, + agreement_id: String, +} + +pub struct FakeActivityInner { + agreement_mapping: HashMap>, + activities: HashMap, +} + +impl FakeActivity { + pub fn new(name: &str, testdir: &Path) -> Self { + FakeActivity { + name: name.to_string(), + _testdir: testdir.to_path_buf(), + inner: Arc::new(RwLock::new(FakeActivityInner { + agreement_mapping: Default::default(), + activities: Default::default(), + })), + } + } + + pub async fn bind_gsb(&self) -> anyhow::Result<()> { + log::info!("Activity ({}) - binding GSB", self.name); + + let self_ = self.clone(); + bus::bind( + activity::local::BUS_ID, + move |msg: activity::local::GetAgreementId| { + let self_ = self_.clone(); + async move { + log::info!( + "[FakeActivity] - responding to GetAgreementId for activity: {}", + msg.activity_id + ); + self_.get_agreement_id(&msg.activity_id).await.ok_or( + activity::RpcMessageError::NotFound(format!( + "Activity id: {}", + msg.activity_id + )), + ) + } + }, + ); + Ok(()) + } + + pub async fn create_activity(&self, agreement_id: &str) -> String { + let id = Uuid::new_v4().to_simple().to_string(); + let activity = Activity { + id: id.clone(), + agreement_id: agreement_id.to_string(), + }; + + let mut lock = self.inner.write().await; + lock.activities.insert(id.clone(), activity.clone()); + lock.agreement_mapping + .entry(agreement_id.to_string()) + .or_default() + .push(id); + activity.id + } + + pub async fn get_agreement_id(&self, activity_id: &str) -> Option { + let lock = self.inner.read().await; + lock.activities + .get(activity_id) + .map(|a| a.agreement_id.clone()) + } +} diff --git a/test-utils/test-framework/framework-mocks/src/lib.rs b/test-utils/test-framework/framework-mocks/src/lib.rs index 21ed5ff75b..a81a4b6bd1 100644 --- a/test-utils/test-framework/framework-mocks/src/lib.rs +++ b/test-utils/test-framework/framework-mocks/src/lib.rs @@ -1,3 +1,4 @@ +mod activity; pub mod identity; pub mod market; pub mod net; diff --git a/test-utils/test-framework/framework-mocks/src/net.rs b/test-utils/test-framework/framework-mocks/src/net.rs index 1edd9d683d..1cabc286d2 100644 --- a/test-utils/test-framework/framework-mocks/src/net.rs +++ b/test-utils/test-framework/framework-mocks/src/net.rs @@ -87,7 +87,8 @@ impl MockNet { fn translate_to(&self, id: NodeId, addr: &str) -> Result { let prefix = self.node_prefix(id)?; let net_prefix = format!("/net/{}", id); - log::info!("Replacing {net_prefix} with {prefix} in {addr}"); + + log::debug!("Replacing {net_prefix} with {prefix} in {addr}"); Ok(addr.replacen(&net_prefix, &prefix, 1)) } diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index e0be67334a..a7bdc624ba 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -6,6 +6,7 @@ use std::str::FromStr; use std::time::Duration; use url::Url; +use crate::activity::FakeActivity; use ya_client::payment::PaymentApi; use ya_client::web::WebClient; use ya_framework_basic::async_drop::DroppableTestContext; @@ -36,6 +37,7 @@ pub struct MockNode { pub identity: Option, pub payment: Option, pub market: Option, + pub activity: Option, } impl MockNode { @@ -51,6 +53,7 @@ impl MockNode { identity: None, payment: None, market: None, + activity: None, } } @@ -76,6 +79,12 @@ impl MockNode { self } + /// Use fake Activity module for this node. + pub fn with_fake_activity(mut self) -> Self { + self.activity = Some(FakeActivity::new(&self.name, &self.testdir)); + self + } + pub fn get_identity(&self) -> anyhow::Result { self.identity .clone() @@ -94,6 +103,12 @@ impl MockNode { .ok_or_else(|| anyhow!("Market ({}) is not initialized", self.name)) } + pub fn get_activity(&self) -> anyhow::Result { + self.activity + .clone() + .ok_or_else(|| anyhow!("Activity ({}) is not initialized", self.name)) + } + /// Binds GSB router and all initialized modules to GSB. /// If you want to bind only chosen modules, you should bind them manually. pub async fn bind_gsb(&self) -> anyhow::Result<()> { @@ -110,6 +125,10 @@ impl MockNode { if let Some(market) = &self.market { market.bind_gsb().await?; } + + if let Some(activity) = &self.activity { + activity.bind_gsb().await?; + } Ok(()) } From 9da6e6631e4c43d15fc288192908f4edc7ac0016 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 2 Aug 2024 19:50:39 +0200 Subject: [PATCH 068/125] Fix: debit notes events constraint failure, when Provider and Requestor are on the same node --- .../down.sql | 31 +++++++++++++++++++ .../up.sql | 31 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql create mode 100644 core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql diff --git a/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql new file mode 100644 index 0000000000..4f3f7c3216 --- /dev/null +++ b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql @@ -0,0 +1,31 @@ +CREATE TABLE pay_debit_note_event_copy( + debit_note_id VARCHAR(50) NOT NULL, + owner_id VARCHAR(50) NOT NULL, + event_type VARCHAR(50) NOT NULL, + timestamp DATETIME NOT NULL DEFAULT(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), + details TEXT NULL, + PRIMARY KEY(debit_note_id, event_type), + FOREIGN KEY(owner_id, debit_note_id) REFERENCES pay_invoice (owner_id, id), + FOREIGN KEY(event_type) REFERENCES pay_event_type (event_type) +); +INSERT INTO pay_debit_note_event_copy (debit_note_id, owner_id, event_type, timestamp, details) +SELECT debit_note_id, owner_id, event_type, timestamp, details FROM pay_debit_note_event; + +DROP VIEW pay_debit_note_event_read; + +DROP TABLE IF EXISTS pay_debit_note_event; +ALTER TABLE pay_debit_note_event_copy RENAME TO pay_debit_note_event; + +CREATE VIEW pay_debit_note_event_read AS +SELECT + inv.role, + ie.debit_note_id, + ie.owner_id, + ie.event_type, + ie.timestamp, + ie.details, + agr.app_session_id +FROM + pay_debit_note_event ie + INNER JOIN pay_invoice inv ON ie.owner_id = inv.owner_id AND ie.debit_note_id = inv.id + INNER JOIN pay_agreement agr ON ie.owner_id = agr.owner_id AND inv.agreement_id = agr.id; diff --git a/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql new file mode 100644 index 0000000000..afa5a156fa --- /dev/null +++ b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql @@ -0,0 +1,31 @@ +CREATE TABLE pay_debit_note_event_copy( + debit_note_id VARCHAR(50) NOT NULL, + owner_id VARCHAR(50) NOT NULL, + event_type VARCHAR(50) NOT NULL, + timestamp DATETIME NOT NULL DEFAULT(STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')), + details TEXT NULL, + PRIMARY KEY(debit_note_id, event_type, owner_id), + FOREIGN KEY(owner_id, debit_note_id) REFERENCES pay_debit_note (owner_id, id), + FOREIGN KEY(event_type) REFERENCES pay_event_type (event_type) +); +INSERT INTO pay_debit_note_event_copy (debit_note_id, owner_id, event_type, timestamp, details) +SELECT debit_note_id, owner_id, event_type, timestamp, details FROM pay_debit_note_event; + +DROP VIEW pay_debit_note_event_read; + +DROP TABLE IF EXISTS pay_debit_note_event; +ALTER TABLE pay_debit_note_event_copy RENAME TO pay_debit_note_event; + +CREATE VIEW pay_debit_note_event_read AS +SELECT + inv.role, + ie.debit_note_id, + ie.owner_id, + ie.event_type, + ie.timestamp, + ie.details, + agr.app_session_id +FROM + pay_debit_note_event ie + INNER JOIN pay_invoice inv ON ie.owner_id = inv.owner_id AND ie.debit_note_id = inv.id + INNER JOIN pay_agreement agr ON ie.owner_id = agr.owner_id AND inv.agreement_id = agr.id; \ No newline at end of file From 761e107bb5066bd0e1f4e17ee23d608b31049b92 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 2 Aug 2024 20:08:16 +0200 Subject: [PATCH 069/125] Add invoice flow test to see if it fails the same way --- core/payment/tests/test_debit_notes.rs | 2 +- core/payment/tests/test_invoices.rs | 183 +++++++++++++++++++++++++ 2 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 core/payment/tests/test_invoices.rs diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs index feeedccb5c..2ea2c7b912 100644 --- a/core/payment/tests/test_debit_notes.rs +++ b/core/payment/tests/test_debit_notes.rs @@ -26,7 +26,7 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< let net = MockNet::new(); net.bind_gsb(); - let node = MockNode::new(net, "node-1", &dir) + let node = MockNode::new(net, "node-1", dir) .with_identity() .with_payment() .with_fake_market() diff --git a/core/payment/tests/test_invoices.rs b/core/payment/tests/test_invoices.rs new file mode 100644 index 0000000000..11cd855558 --- /dev/null +++ b/core/payment/tests/test_invoices.rs @@ -0,0 +1,183 @@ +use bigdecimal::BigDecimal; +use chrono::Utc; +use std::str::FromStr; +use std::time::Duration; +use test_context::test_context; + +use ya_client_model::payment::allocation::PaymentPlatformEnum; +use ya_client_model::payment::{Acceptance, DocumentStatus, NewAllocation, NewInvoice}; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::mocks::net::IMockNet; +use ya_framework_basic::{resource, temp_dir}; +use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::net::MockNet; +use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::Driver; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_invoice_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_invoice_flow")?; + let dir = dir.path(); + + let net = MockNet::new(); + net.bind_gsb(); + + let node = MockNode::new(net, "node-1", dir) + .with_identity() + .with_payment() + .with_fake_market(); + node.bind_gsb().await?; + node.start_server(ctx).await?; + + let appkey_prov = node.get_identity()?.create_identity_key("provider").await?; + let appkey_req = node + .get_identity()? + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; + + let app_session_id = Some("app_session_id".to_string()); + let mut agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + agreement.app_session_id = app_session_id.clone(); + node.get_market()?.add_agreement(agreement.clone()).await; + + let requestor = node.rest_payments(&appkey_req.key)?; + let provider = node.rest_payments(&appkey_prov.key)?; + + node.get_payment()? + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) + .await?; + + let payment_platform = + PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); + + let invoice_date = Utc::now(); + + log::info!("Issuing invoice..."); + let invoice = provider + .issue_invoice(&NewInvoice { + agreement_id: agreement.agreement_id.to_string(), + activity_ids: None, + amount: BigDecimal::from_str("1.230028519070000")?, + payment_due_date: Utc::now(), + }) + .await?; + log::debug!("invoice={:?}", invoice); + log::info!("Invoice issued."); + + log::info!("Sending invoice..."); + provider.send_invoice(&invoice.invoice_id).await?; + log::info!("Invoice sent."); + + let invoice_events_received = requestor + .get_invoice_events::( + Some(&invoice_date), + Some(Duration::from_secs(1000)), + None, + app_session_id.clone(), + ) + .await + .unwrap(); + log::debug!("events 1: {:?}", &invoice_events_received); + log::debug!( + "DATE: {:?}", + Some(&invoice_events_received.first().unwrap().event_date) + ); + + log::info!("Creating allocation..."); + let allocation = requestor + .create_allocation(&NewAllocation { + address: None, // Use default address (i.e. identity) + payment_platform: Some(payment_platform.clone()), + total_amount: BigDecimal::from(10u64), + timeout: None, + make_deposit: false, + deposit: None, + extend_timeout: None, + }) + .await?; + log::debug!("allocation={:?}", allocation); + log::info!("Allocation created."); + + log::debug!( + "INVOICES1: {:?}", + requestor.get_invoices::(None, None).await + ); + log::debug!( + "INVOICES2: {:?}", + requestor + .get_invoices::(Some(invoice_date), None) + .await + ); + log::debug!( + "INVOICES3: {:?}", + requestor.get_invoices::(Some(Utc::now()), None).await + ); + + log::info!("Accepting invoice..."); + let now = Utc::now(); + requestor + .accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id, + }, + ) + .await?; + log::info!("Invoice accepted."); + + let invoice_events_accepted = provider + .get_invoice_events::( + Some(&invoice_events_received.first().unwrap().event_date), + Some(Duration::from_secs(1000)), + None, + app_session_id.clone(), + ) + .await + .unwrap(); + log::debug!("events 2: {:?}", &invoice_events_accepted); + + log::info!("Waiting for payment..."); + let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer + + let mut payments = provider + .get_payments(Some(&now), timeout, None, app_session_id.clone()) + .await?; + + let signed_payments = provider + .get_signed_payments(Some(&now), None, None, app_session_id.clone()) + .await?; + + assert_eq!(payments.len(), 1); + assert_eq!(signed_payments.len(), 1); + + let payment = payments.pop().unwrap(); + assert!(payment.amount >= invoice.amount); + + log::info!("Payment verified correctly."); + + log::info!("Verifying invoice status..."); + let invoice = provider.get_invoice(&invoice.invoice_id).await?; + assert_eq!(invoice.status, DocumentStatus::Settled); + log::info!("Invoice status verified correctly."); + + let invoice_events_settled = provider + .get_invoice_events::( + Some(&invoice_events_accepted.first().unwrap().event_date), + Some(Duration::from_secs(1000)), + None, + app_session_id.clone(), + ) + .await + .unwrap(); + log::debug!("events 3: {:?}", &invoice_events_settled); + + log::info!(" 👍🏻 Example completed successfully ❤️"); + Ok(()) +} From 3dcd471f858acf04f15d9a0ff63ad85c2009add0 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 5 Aug 2024 18:06:49 +0200 Subject: [PATCH 070/125] Fix LogErr trait module discovery --- utils/std-utils/Cargo.toml | 5 ++ utils/std-utils/src/result.rs | 93 ++++++++++++++++++++++++++++------- 2 files changed, 81 insertions(+), 17 deletions(-) diff --git a/utils/std-utils/Cargo.toml b/utils/std-utils/Cargo.toml index 83760e4d89..66a125c455 100644 --- a/utils/std-utils/Cargo.toml +++ b/utils/std-utils/Cargo.toml @@ -8,3 +8,8 @@ edition = "2018" [dependencies] log = "0.4" backtrace = "0.3.50" + + +[dev-dependencies] +anyhow = "1.0" +testing_logger = "0.1" diff --git a/utils/std-utils/src/result.rs b/utils/std-utils/src/result.rs index c50acc8f91..2eafc32e20 100644 --- a/utils/std-utils/src/result.rs +++ b/utils/std-utils/src/result.rs @@ -1,5 +1,6 @@ use log::{Level, Record}; use std::fmt::{Debug, Display}; +use std::sync::atomic::{AtomicBool, Ordering}; pub trait LogErr { /// If Result is `Err`, this function logs it on error level @@ -31,30 +32,36 @@ impl LogErr for Result { fn log_error(self, message: &str, log_level: Level) -> Result { if let Err(e) = self { + let std_utils_symbols = AtomicBool::new(false); + let cont = AtomicBool::new(true); + backtrace::trace(|frame| { - let mut cont = true; backtrace::resolve_frame(frame, |symbol| { if let Some(name) = symbol.name() { - let module_path = name.to_string(); - if module_path.starts_with(" LogErr for Result { } } } + +fn log(module: &str, level: Level, message: &str, e: &E) { + let mut msg = message; + if msg.is_empty() { + msg = "Error occurred"; + } + + log::logger().log( + &Record::builder() + .level(level) + .args(format_args!("{}: {}", msg, e)) + .module_path(Some(module)) + .target(module) + .build(), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::anyhow; + use testing_logger; + + #[test] + fn test_log_err_and_warn() { + testing_logger::setup(); + + let _result: anyhow::Result<()> = Err(anyhow!("Message-Message")).log_err(); + + testing_logger::validate(|captured_logs| { + let last = captured_logs.last().unwrap(); + assert_eq!(last.body, "Error occurred: Message-Message"); + assert_eq!(last.level, Level::Error); + assert_eq!( + last.target, + "ya_std_utils::result::tests::test_log_err_and_warn" + ); + }); + + let _result: anyhow::Result<()> = Err(anyhow!("Message-Message")).log_warn_msg("Warning"); + + testing_logger::validate(|captured_logs| { + let last = captured_logs.last().unwrap(); + assert_eq!(last.body, "Warning: Message-Message"); + assert_eq!(last.level, Level::Warn); + assert_eq!( + last.target, + "ya_std_utils::result::tests::test_log_err_and_warn" + ); + }); + } +} From a7a2b45f8e32527cee69561b96ba5a5366beee9b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 5 Aug 2024 19:06:23 +0200 Subject: [PATCH 071/125] Fix test_invoice_flow --- Cargo.lock | 16 ++++++- core/payment/Cargo.toml | 1 + core/payment/src/processor.rs | 45 +++++++++-------- core/payment/src/service.rs | 21 ++++++-- core/payment/tests/test_debit_notes.rs | 26 +++++++--- core/payment/tests/test_invoices.rs | 26 ++++++++-- core/serv/src/main.rs | 1 + .../test-framework/framework-basic/src/log.rs | 4 +- .../framework-mocks/src/payment.rs | 48 +++++++++++++++++++ 9 files changed, 148 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1683af4bf..598cfc92ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -568,9 +568,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "appdirs" @@ -7504,6 +7504,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "testing_logger" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d92b727cb45d33ae956f7f46b966b25f1bc712092aeef9dba5ac798fc89f720" +dependencies = [ + "log", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -9498,6 +9507,7 @@ dependencies = [ "ya-service-api-interfaces", "ya-service-api-web", "ya-service-bus", + "ya-std-utils", ] [[package]] @@ -9972,8 +9982,10 @@ dependencies = [ name = "ya-std-utils" version = "0.1.0" dependencies = [ + "anyhow", "backtrace", "log", + "testing_logger", ] [[package]] diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index 69c4900b8d..d13480d79e 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -25,6 +25,7 @@ ya-service-api = "0.1" ya-service-api-interfaces = "0.2" ya-service-api-web = "0.2" ya-service-bus = { workspace = true } +ya-std-utils = "0.1" actix-web = "4" anyhow = "1.0" diff --git a/core/payment/src/processor.rs b/core/payment/src/processor.rs index 933b0bcd79..60aa018f8f 100644 --- a/core/payment/src/processor.rs +++ b/core/payment/src/processor.rs @@ -512,37 +512,38 @@ impl PaymentProcessor { tokio::task::spawn_local( async move { + let send_result = + Self::send_to_gsb(payer_id, payee_id, msg_with_bytes.clone()).await; + + let mark_sent = match send_result { + Ok(_) => true, + // If sending SendPaymentWithBytes is not supported then use SendPayment as fallback. + Err(PaymentSendToGsbError::NotSupported) => { + match Self::send_to_gsb(payer_id, payee_id, msg).await { + Ok(_) => true, + Err(PaymentSendToGsbError::Rejected) => true, + Err(PaymentSendToGsbError::Failed) => false, + Err(PaymentSendToGsbError::NotSupported) => false, + } + } + Err(_) => false, + }; + let db_executor = db_executor.timeout_lock(DB_LOCK_TIMEOUT).await?; let payment_dao: PaymentDao = db_executor.as_dao(); let sync_dao: SyncNotifsDao = db_executor.as_dao(); - let send_result = - Self::send_to_gsb(payer_id, payee_id, msg_with_bytes.clone()).await; - - let mark_sent = if send_result.is_ok() { + if mark_sent { + payment_dao.mark_sent(payment_id.clone()).await?; + // Always add new type of signature. Compatibility is for older Provider nodes only. payment_dao .add_signature( - payment_id.clone(), + payment_id, msg_with_bytes.signature.clone(), msg_with_bytes.signed_bytes.clone(), ) - .await - .is_ok() - } else if send_result.is_err_and(|err| err == PaymentSendToGsbError::NotSupported) { - // if sending SendPaymentWithBytes is not supported then try sending SendPayment - match Self::send_to_gsb(payer_id, payee_id, msg).await { - Ok(_) => true, - Err(PaymentSendToGsbError::Rejected) => true, - Err(PaymentSendToGsbError::Failed) => false, - Err(PaymentSendToGsbError::NotSupported) => false, - } - } else { - false - }; - - if mark_sent { - payment_dao.mark_sent(payment_id).await.ok(); + .await?; } else { sync_dao.upsert(payee_id).await?; SYNC_NOTIFS_NOTIFY.notify_one(); @@ -703,7 +704,9 @@ impl PaymentProcessor { } { + log::debug!("`verify_payment` acquiring db lock"); let db_executor = self.db_executor.timeout_lock(DB_LOCK_TIMEOUT).await?; + log::debug!("`verify_payment` db lock acquired"); // Verify agreement payments let agreement_dao: AgreementDao = db_executor.as_dao(); diff --git a/core/payment/src/service.rs b/core/payment/src/service.rs index f35966bbc3..a4c63c7915 100644 --- a/core/payment/src/service.rs +++ b/core/payment/src/service.rs @@ -59,6 +59,7 @@ mod local { use std::time::Instant; use std::{collections::BTreeMap, convert::TryInto}; use tracing::{debug, trace}; + use ya_client_model::{ payment::{ Account, DebitNoteEventType, DocumentStatus, DriverDetails, DriverStatusProperty, @@ -817,6 +818,7 @@ mod local { mod public { use std::str::FromStr; + use tracing::debug; use super::*; @@ -830,6 +832,7 @@ mod public { use ya_client_model::{payment::*, NodeId}; use ya_core_model::payment::public::*; use ya_persistence::types::Role; + use ya_std_utils::LogErr; pub fn bind_service( db: &DbExecutor, @@ -1317,6 +1320,7 @@ mod public { signature: Vec, signed_bytes: Option>, ) -> Result { + let payment_id = payment.payment_id.clone(); if sender_id != payment.payer_id.to_string() { return Err(SendError::BadRequest("Invalid payer ID".to_owned())); } @@ -1325,7 +1329,12 @@ mod public { let amount = payment.amount.clone(); let num_paid_invoices = payment.agreement_payments.len() as u64; - log::debug!("Verify payment processor started"); + debug!( + entity = "payment", + action = "verify", + payment_id, + "Verify payment processor started." + ); let res = match processor .verify_payment(payment, signature, canonicalized, signed_bytes) .await @@ -1342,8 +1351,14 @@ mod public { VerifyPaymentError::Validation(e) => Err(SendError::BadRequest(e)), _ => Err(SendError::ServiceError(e.to_string())), }, - }; - log::debug!("Verify payment processor finished"); + }.log_err_msg("Payment verification failure"); + + debug!( + entity = "payment", + action = "verify", + payment_id, + "Verify payment processor finished." + ); res } diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs index 2ea2c7b912..afd4a16542 100644 --- a/core/payment/tests/test_debit_notes.rs +++ b/core/payment/tests/test_debit_notes.rs @@ -12,7 +12,7 @@ use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; -use ya_framework_mocks::payment::Driver; +use ya_framework_mocks::payment::{Driver, PaymentRestExt}; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] @@ -40,9 +40,12 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< .create_from_private_key(&resource!("ci-requestor-1.key.priv")) .await?; - let agreement = + let app_session_id = Some("app_session_id".to_string()); + let mut agreement = FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + agreement.app_session_id = app_session_id.clone(); node.get_market()?.add_agreement(agreement.clone()).await; + let activity_id = node .get_activity()? .create_activity(&agreement.agreement_id) @@ -55,7 +58,6 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) .await?; - let app_session_id = Some("app_session_id".to_string()); let payment_platform = PaymentPlatformEnum::PaymentPlatformName("erc20-holesky-tglm".to_string()); @@ -137,9 +139,14 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< log::info!("Debit note accepted."); log::info!("Waiting for payment..."); - let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer let mut payments = provider - .get_payments(Some(&now), timeout, None, None) + .wait_for_payment( + Some(&now), + // Should be enough for GLM transfer + Duration::from_secs(1000), + None, + app_session_id.clone(), + ) .await?; assert_eq!(payments.len(), 1); let payment = payments.pop().unwrap(); @@ -182,9 +189,14 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< log::info!("Debit note accepted."); log::info!("Waiting for payment..."); - let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer let mut payments = provider - .get_payments(Some(&now), timeout, None, app_session_id.clone()) + .wait_for_payment( + Some(&now), + // Should be enough for GLM transfer + Duration::from_secs(1000), + None, + app_session_id.clone(), + ) .await?; assert_eq!(payments.len(), 1); let payment = payments.pop().unwrap(); diff --git a/core/payment/tests/test_invoices.rs b/core/payment/tests/test_invoices.rs index 11cd855558..cc1f950287 100644 --- a/core/payment/tests/test_invoices.rs +++ b/core/payment/tests/test_invoices.rs @@ -13,7 +13,7 @@ use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; -use ya_framework_mocks::payment::Driver; +use ya_framework_mocks::payment::{Driver, PaymentRestExt}; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] @@ -143,13 +143,29 @@ async fn test_invoice_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .unwrap(); log::debug!("events 2: {:?}", &invoice_events_accepted); - log::info!("Waiting for payment..."); - let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer + log::info!("Waiting for payment on requestor..."); + let mut payments = requestor + .wait_for_payment( + Some(&now), + // Should be enough for GLM transfer + Duration::from_secs(5 * 60), + None, + app_session_id.clone(), + ) + .await?; + assert_eq!(payments.len(), 1); + let payment = payments.pop().unwrap(); + assert!(payment.amount >= invoice.amount); + log::info!("Waiting for payment on provider..."); let mut payments = provider - .get_payments(Some(&now), timeout, None, app_session_id.clone()) + .wait_for_payment( + Some(&now), + Duration::from_secs(60), + None, + app_session_id.clone(), + ) .await?; - let signed_payments = provider .get_signed_payments(Some(&now), None, None, app_session_id.clone()) .await?; diff --git a/core/serv/src/main.rs b/core/serv/src/main.rs index 0f6584bc5d..16e5a91fce 100644 --- a/core/serv/src/main.rs +++ b/core/serv/src/main.rs @@ -511,6 +511,7 @@ impl ServiceCommand { ("web3", log::LevelFilter::Info), ("tokio_util", log::LevelFilter::Off), ("mio", log::LevelFilter::Off), + ("sqlx", log::LevelFilter::Info), ]; // if RUST_LOG is default, then set ya_payment::service to debug (investigating rare deadlocks) diff --git a/test-utils/test-framework/framework-basic/src/log.rs b/test-utils/test-framework/framework-basic/src/log.rs index 9d85d40bf1..4632200de2 100644 --- a/test-utils/test-framework/framework-basic/src/log.rs +++ b/test-utils/test-framework/framework-basic/src/log.rs @@ -7,9 +7,9 @@ pub fn enable_logs(enable: bool) { env_logger::try_init_from_env(Env::default()).ok(); } else { env_logger::builder() - .filter_level(log::LevelFilter::Info) + .filter_level(log::LevelFilter::Debug) .filter(Some("web3"), log::LevelFilter::Warn) - .filter(Some("sqlx_core"), log::LevelFilter::Warn) + .filter(Some("sqlx"), log::LevelFilter::Info) .filter(Some("hyper"), log::LevelFilter::Warn) .filter(Some("erc20_payment_lib"), log::LevelFilter::Info) .filter(Some("trust_dns_proto"), log::LevelFilter::Warn) diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index 17596ff146..45e98cb034 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -1,7 +1,12 @@ use anyhow::anyhow; +use chrono::{DateTime, TimeZone, Utc}; +use std::fmt::Display; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::Duration; +use ya_client::payment::PaymentApi; +use ya_client_model::payment::Payment; use ya_core_model::driver::{driver_bus_id, Fund}; use ya_core_model::payment::local::BUS_ID; use ya_payment::api::web_scope; @@ -107,3 +112,46 @@ impl MockPayment { bus::service(BUS_ID) } } + +#[async_trait::async_trait(?Send)] +pub trait PaymentRestExt { + async fn wait_for_payment( + &self, + after_timestamp: Option<&DateTime>, + timeout: Duration, + max_events: Option, + app_session_id: Option, + ) -> anyhow::Result> + where + Tz: TimeZone, + Tz::Offset: Display; +} + +#[async_trait::async_trait(?Send)] +impl PaymentRestExt for PaymentApi { + async fn wait_for_payment( + &self, + after_timestamp: Option<&DateTime>, + timeout: Duration, + max_events: Option, + app_session_id: Option, + ) -> anyhow::Result> + where + Tz: TimeZone, + Tz::Offset: Display, + { + let start = Utc::now(); + // Workaround: Can't pass timeout to `get_payments`, because serde_urlencoded is unable to deserialize it. + // https://github.com/nox/serde_urlencoded/issues/33 + while start + timeout > Utc::now() { + let payments = self + .get_payments(after_timestamp, None, max_events, app_session_id.clone()) + .await?; + + if !payments.is_empty() { + return Ok(payments); + } + } + Err(anyhow!("Timeout {timeout:?} waiting for payments.")) + } +} From 5349a6421bb85f50904987072d3eb113fb98e45f Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 5 Aug 2024 20:04:16 +0200 Subject: [PATCH 072/125] Fix test_debit_note_flow --- .../down.sql | 19 ++++++++++--------- .../up.sql | 19 ++++++++++--------- core/payment/src/processor.rs | 2 -- core/payment/tests/test_debit_notes.rs | 2 +- core/payment/tests/test_invoices.rs | 2 +- utils/std-utils/src/result.rs | 1 - 6 files changed, 22 insertions(+), 23 deletions(-) diff --git a/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql index 4f3f7c3216..c47619f47c 100644 --- a/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql +++ b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/down.sql @@ -18,14 +18,15 @@ ALTER TABLE pay_debit_note_event_copy RENAME TO pay_debit_note_event; CREATE VIEW pay_debit_note_event_read AS SELECT - inv.role, - ie.debit_note_id, - ie.owner_id, - ie.event_type, - ie.timestamp, - ie.details, + dn.role, + dne.debit_note_id, + dne.owner_id, + dne.event_type, + dne.timestamp, + dne.details, agr.app_session_id FROM - pay_debit_note_event ie - INNER JOIN pay_invoice inv ON ie.owner_id = inv.owner_id AND ie.debit_note_id = inv.id - INNER JOIN pay_agreement agr ON ie.owner_id = agr.owner_id AND inv.agreement_id = agr.id; + pay_debit_note_event dne + INNER JOIN pay_debit_note dn ON dne.owner_id = dn.owner_id AND dne.debit_note_id = dn.id + INNER JOIN pay_activity act ON dne.owner_id = act.owner_id AND dn.activity_id = act.id + INNER JOIN pay_agreement agr ON dne.owner_id = agr.owner_id AND act.agreement_id = agr.id diff --git a/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql index afa5a156fa..cf6276e0a3 100644 --- a/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql +++ b/core/payment/migrations/2024-08-02-173140_fix-debitnote-events-primary-key/up.sql @@ -18,14 +18,15 @@ ALTER TABLE pay_debit_note_event_copy RENAME TO pay_debit_note_event; CREATE VIEW pay_debit_note_event_read AS SELECT - inv.role, - ie.debit_note_id, - ie.owner_id, - ie.event_type, - ie.timestamp, - ie.details, + dn.role, + dne.debit_note_id, + dne.owner_id, + dne.event_type, + dne.timestamp, + dne.details, agr.app_session_id FROM - pay_debit_note_event ie - INNER JOIN pay_invoice inv ON ie.owner_id = inv.owner_id AND ie.debit_note_id = inv.id - INNER JOIN pay_agreement agr ON ie.owner_id = agr.owner_id AND inv.agreement_id = agr.id; \ No newline at end of file + pay_debit_note_event dne + INNER JOIN pay_debit_note dn ON dne.owner_id = dn.owner_id AND dne.debit_note_id = dn.id + INNER JOIN pay_activity act ON dne.owner_id = act.owner_id AND dn.activity_id = act.id + INNER JOIN pay_agreement agr ON dne.owner_id = agr.owner_id AND act.agreement_id = agr.id diff --git a/core/payment/src/processor.rs b/core/payment/src/processor.rs index 60aa018f8f..48c7072347 100644 --- a/core/payment/src/processor.rs +++ b/core/payment/src/processor.rs @@ -704,9 +704,7 @@ impl PaymentProcessor { } { - log::debug!("`verify_payment` acquiring db lock"); let db_executor = self.db_executor.timeout_lock(DB_LOCK_TIMEOUT).await?; - log::debug!("`verify_payment` db lock acquired"); // Verify agreement payments let agreement_dao: AgreementDao = db_executor.as_dao(); diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs index afd4a16542..e9bab9ae11 100644 --- a/core/payment/tests/test_debit_notes.rs +++ b/core/payment/tests/test_debit_notes.rs @@ -18,7 +18,7 @@ use ya_framework_mocks::payment::{Driver, PaymentRestExt}; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("test_debit_note_flow")?; let dir = dir.path(); diff --git a/core/payment/tests/test_invoices.rs b/core/payment/tests/test_invoices.rs index cc1f950287..d5a3338cec 100644 --- a/core/payment/tests/test_invoices.rs +++ b/core/payment/tests/test_invoices.rs @@ -19,7 +19,7 @@ use ya_framework_mocks::payment::{Driver, PaymentRestExt}; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_invoice_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("test_invoice_flow")?; let dir = dir.path(); diff --git a/utils/std-utils/src/result.rs b/utils/std-utils/src/result.rs index 2eafc32e20..eb6186d316 100644 --- a/utils/std-utils/src/result.rs +++ b/utils/std-utils/src/result.rs @@ -90,7 +90,6 @@ fn log(module: &str, level: Level, message: &str, e: &E) { mod tests { use super::*; use anyhow::anyhow; - use testing_logger; #[test] fn test_log_err_and_warn() { From b9a443de823ab51d1bf27126c4b94d633b248c5a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Aug 2024 19:10:12 +0200 Subject: [PATCH 073/125] Fix (workaround) problems with LogErr::* functions --- utils/std-utils/src/result.rs | 52 ++++++++--------------------------- 1 file changed, 12 insertions(+), 40 deletions(-) diff --git a/utils/std-utils/src/result.rs b/utils/std-utils/src/result.rs index eb6186d316..98a80d858f 100644 --- a/utils/std-utils/src/result.rs +++ b/utils/std-utils/src/result.rs @@ -1,6 +1,7 @@ use log::{Level, Record}; use std::fmt::{Debug, Display}; -use std::sync::atomic::{AtomicBool, Ordering}; + +pub use std::format_args; pub trait LogErr { /// If Result is `Err`, this function logs it on error level @@ -14,55 +15,32 @@ pub trait LogErr { } impl LogErr for Result { + #[track_caller] fn log_err(self) -> Result { self.log_err_msg("") } + #[track_caller] fn log_warn(self) -> Result { self.log_warn_msg("") } + #[track_caller] fn log_err_msg(self, message: &str) -> Result { self.log_error(message, Level::Error) } + #[track_caller] fn log_warn_msg(self, message: &str) -> Result { self.log_error(message, Level::Warn) } + #[track_caller] fn log_error(self, message: &str, log_level: Level) -> Result { if let Err(e) = self { - let std_utils_symbols = AtomicBool::new(false); - let cont = AtomicBool::new(true); - - backtrace::trace(|frame| { - backtrace::resolve_frame(frame, |symbol| { - if let Some(name) = symbol.name() { - let module = name.to_string(); - - // Skip all symbols from this library until we find first symbol of caller function. - if std_utils_symbols.load(Ordering::SeqCst) - && !module.contains("ya_std_utils::result::LogErr") - { - cont.store(false, Ordering::SeqCst); - - // Find out the module path and print log. - let suffix = module.rfind("::"); - if let Some(suffix) = suffix { - log(&module[..suffix], log_level, message, &e); - } else { - log(&module, log_level, message, &e); - } - } - - // Mark first symbol of function called in this library. - if module.contains("ya_std_utils::result::LogErr") { - std_utils_symbols.store(true, Ordering::SeqCst); - } - } - }); - cont.load(Ordering::SeqCst) - }); + // It will return file not module path, so it will differ from the original log macro. + let module = std::panic::Location::caller().file(); + log(module, log_level, message, &e); Err(e) } else { self @@ -101,10 +79,7 @@ mod tests { let last = captured_logs.last().unwrap(); assert_eq!(last.body, "Error occurred: Message-Message"); assert_eq!(last.level, Level::Error); - assert_eq!( - last.target, - "ya_std_utils::result::tests::test_log_err_and_warn" - ); + assert_eq!(last.target, "utils/std-utils/src/result.rs"); }); let _result: anyhow::Result<()> = Err(anyhow!("Message-Message")).log_warn_msg("Warning"); @@ -113,10 +88,7 @@ mod tests { let last = captured_logs.last().unwrap(); assert_eq!(last.body, "Warning: Message-Message"); assert_eq!(last.level, Level::Warn); - assert_eq!( - last.target, - "ya_std_utils::result::tests::test_log_err_and_warn" - ); + assert_eq!(last.target, "utils/std-utils/src/result.rs"); }); } } From c90c279d8e73a3b590923d088f31aac4dec5ff1b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Aug 2024 19:37:51 +0200 Subject: [PATCH 074/125] Remove examples that were rewritten to tests --- core/payment/examples/debit_note_flow.rs | 198 ------------------- core/payment/examples/invoice_flow.rs | 167 ---------------- core/payment/examples/release_allocation.rs | 136 ------------- core/payment/examples/validate_allocation.rs | 122 ------------ 4 files changed, 623 deletions(-) delete mode 100644 core/payment/examples/debit_note_flow.rs delete mode 100644 core/payment/examples/invoice_flow.rs delete mode 100644 core/payment/examples/release_allocation.rs delete mode 100644 core/payment/examples/validate_allocation.rs diff --git a/core/payment/examples/debit_note_flow.rs b/core/payment/examples/debit_note_flow.rs deleted file mode 100644 index e4e78f9530..0000000000 --- a/core/payment/examples/debit_note_flow.rs +++ /dev/null @@ -1,198 +0,0 @@ -use bigdecimal::BigDecimal; -use chrono::Utc; -use std::time::Duration; -use structopt::StructOpt; -use ya_client::payment::PaymentApi; -use ya_client::web::{rest_api_url, WebClient}; -use ya_client_model::payment::allocation::PaymentPlatformEnum; -use ya_client_model::payment::{Acceptance, DocumentStatus, NewAllocation, NewDebitNote}; - -#[derive(Clone, Debug, StructOpt)] -struct Args { - #[structopt(long)] - app_session_id: Option, - #[structopt(long, default_value = "dummy-glm")] - platform: String, -} - -#[actix_rt::main] -async fn main() -> anyhow::Result<()> { - let log_level = - std::env::var("RUST_LOG").unwrap_or_else(|_| "debit_note_flow=debug,info".to_owned()); - std::env::set_var("RUST_LOG", log_level); - env_logger::init(); - - let args: Args = Args::from_args(); - - // Create requestor / provider PaymentApi - let provider_url = format!("{}provider/", rest_api_url()).parse().unwrap(); - let provider: PaymentApi = WebClient::builder() - .api_url(provider_url) - .build() - .interface()?; - let requestor_url = format!("{}requestor/", rest_api_url()).parse().unwrap(); - let requestor: PaymentApi = WebClient::builder() - .api_url(requestor_url) - .build() - .interface()?; - - let debit_note_date = Utc::now(); - - let debit_note = NewDebitNote { - activity_id: "activity_id".to_string(), - total_amount_due: BigDecimal::from(1u64), - usage_counter_vector: None, - payment_due_date: Some(Utc::now()), - }; - log::info!( - "Issuing debit note (total amount due: {} GLM)...", - &debit_note.total_amount_due - ); - let debit_note = provider.issue_debit_note(&debit_note).await?; - log::info!("Debit note issued."); - - log::info!("Sending debit note..."); - provider.send_debit_note(&debit_note.debit_note_id).await?; - log::info!("Debit note sent."); - - let debit_note_events_received = requestor - .get_debit_note_events::( - Some(&debit_note_date), - Some(Duration::from_secs(10)), - None, - args.app_session_id.clone(), - ) - .await - .unwrap(); - log::debug!("events 1: {:?}", &debit_note_events_received); - log::debug!( - "DATE: {:?}", - Some(&debit_note_events_received.first().unwrap().event_date) - ); - - log::info!("Creating allocation..."); - let allocation = requestor - .create_allocation(&NewAllocation { - address: None, // Use default address (i.e. identity) - payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName(args.platform)), - total_amount: BigDecimal::from(10u64), - make_deposit: false, - deposit: None, - timeout: None, - extend_timeout: None, - }) - .await?; - log::info!("Allocation created."); - - log::debug!( - "DEBIT_NOTES1: {:?}", - requestor.get_debit_notes::(None, None).await - ); - log::debug!( - "DEBIT_NOTES2: {:?}", - requestor - .get_debit_notes::(Some(debit_note_date), None) - .await - ); - log::debug!( - "DEBIT_NOTES3: {:?}", - requestor - .get_debit_notes::(Some(Utc::now()), None) - .await - ); - - log::info!("Accepting debit note..."); - let now = Utc::now(); - requestor - .accept_debit_note( - &debit_note.debit_note_id, - &Acceptance { - total_amount_accepted: debit_note.total_amount_due.clone(), - allocation_id: allocation.allocation_id.clone(), - }, - ) - .await?; - log::info!("Debit note accepted."); - - log::info!("Waiting for payment..."); - let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer - let mut payments = provider - .get_payments(Some(&now), timeout, None, None) - .await?; - assert_eq!(payments.len(), 1); - let payment = payments.pop().unwrap(); - assert_eq!(&payment.amount, &debit_note.total_amount_due); - log::info!("Payment verified correctly."); - - log::info!("Verifying debit note status..."); - let debit_note = provider.get_debit_note(&debit_note.debit_note_id).await?; - assert_eq!(debit_note.status, DocumentStatus::Settled); - log::info!("Debit note status verified correctly."); - - let debit_note2 = NewDebitNote { - activity_id: "activity_id".to_string(), - total_amount_due: BigDecimal::from(2u64), - usage_counter_vector: None, - payment_due_date: Some(Utc::now()), - }; - log::info!( - "Issuing debit note (total amount due: {} GLM)...", - debit_note2.total_amount_due - ); - let debit_note2 = provider.issue_debit_note(&debit_note2).await?; - log::info!("Debit note issued."); - - log::info!("Sending debit note..."); - provider.send_debit_note(&debit_note2.debit_note_id).await?; - log::info!("Debit note sent."); - - log::info!("Accepting debit note..."); - let now = Utc::now(); - requestor - .accept_debit_note( - &debit_note2.debit_note_id, - &Acceptance { - total_amount_accepted: debit_note2.total_amount_due.clone(), - allocation_id: allocation.allocation_id, - }, - ) - .await?; - log::info!("Debit note accepted."); - - log::info!("Waiting for payment..."); - let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer - let mut payments = provider - .get_payments(Some(&now), timeout, None, args.app_session_id.clone()) - .await?; - assert_eq!(payments.len(), 1); - let payment = payments.pop().unwrap(); - assert_eq!( - &payment.amount, - &(&debit_note2.total_amount_due - &debit_note.total_amount_due) - ); - log::info!("Payment verified correctly."); - - log::info!("Verifying debit note status..."); - let debit_note2 = provider.get_debit_note(&debit_note2.debit_note_id).await?; - assert_eq!(debit_note2.status, DocumentStatus::Settled); - log::info!("Debit note status verified correctly."); - - // Not implemented - // log::debug!( - // "get_payments_for_debit_note1: {:?}", - // requestor.get_payments_for_debit_note::(&debit_note2.debit_note_id, None, None).await - // ); - // log::debug!( - // "get_payments_for_debit_note2: {:?}", - // requestor - // .get_payments_for_debit_note::(&debit_note2.debit_note_id, Some(debit_note_date), None) - // .await - // ); - // log::debug!( - // "get_payments_for_debit_note3: {:?}", - // requestor.get_payments_for_debit_note::(&debit_note2.debit_note_id, Some(Utc::now()), None).await - // ); - - log::info!(" 👍🏻 Example completed successfully ❤️"); - Ok(()) -} diff --git a/core/payment/examples/invoice_flow.rs b/core/payment/examples/invoice_flow.rs deleted file mode 100644 index a37b40b846..0000000000 --- a/core/payment/examples/invoice_flow.rs +++ /dev/null @@ -1,167 +0,0 @@ -use bigdecimal::BigDecimal; -use chrono::Utc; -use std::str::FromStr; -use std::time::Duration; -use structopt::StructOpt; -use ya_client::payment::PaymentApi; -use ya_client::web::{rest_api_url, WebClient}; -use ya_client_model::payment::allocation::PaymentPlatformEnum; -use ya_client_model::payment::{Acceptance, DocumentStatus, NewAllocation, NewInvoice}; - -#[derive(Clone, Debug, StructOpt)] -struct Args { - #[structopt(long)] - app_session_id: Option, - #[structopt(long, default_value = "erc20-holesky-tglm")] - platform: String, -} - -#[actix_rt::main] -async fn main() -> anyhow::Result<()> { - let log_level = - std::env::var("RUST_LOG").unwrap_or_else(|_| "invoice_flow=debug,info".to_owned()); - std::env::set_var("RUST_LOG", log_level); - env_logger::init(); - - let args: Args = Args::from_args(); - - // Create requestor / provider PaymentApi - let provider_url = format!("{}provider/", rest_api_url()).parse().unwrap(); - let provider: PaymentApi = WebClient::builder() - .timeout(Duration::from_secs(600 * 60)) - .api_url(provider_url) - .build() - .interface()?; - - let requestor_url = format!("{}requestor/", rest_api_url()).parse().unwrap(); - let requestor: PaymentApi = WebClient::builder() - .timeout(Duration::from_secs(600 * 60)) - .api_url(requestor_url) - .build() - .interface()?; - - let invoice_date = Utc::now(); - - log::info!("Issuing invoice..."); - let invoice = provider - .issue_invoice(&NewInvoice { - agreement_id: "agreement_id".to_string(), - activity_ids: None, - amount: BigDecimal::from_str("1.230028519070000")?, - payment_due_date: Utc::now(), - }) - .await?; - log::debug!("invoice={:?}", invoice); - log::info!("Invoice issued."); - - log::info!("Sending invoice..."); - provider.send_invoice(&invoice.invoice_id).await?; - log::info!("Invoice sent."); - - let invoice_events_received = requestor - .get_invoice_events::( - Some(&invoice_date), - Some(Duration::from_secs(1000)), - None, - args.app_session_id.clone(), - ) - .await - .unwrap(); - log::debug!("events 1: {:?}", &invoice_events_received); - log::debug!( - "DATE: {:?}", - Some(&invoice_events_received.first().unwrap().event_date) - ); - - log::info!("Creating allocation..."); - let allocation = requestor - .create_allocation(&NewAllocation { - address: None, // Use default address (i.e. identity) - payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName(args.platform)), - total_amount: BigDecimal::from(10u64), - timeout: None, - make_deposit: false, - deposit: None, - extend_timeout: None, - }) - .await?; - log::debug!("allocation={:?}", allocation); - log::info!("Allocation created."); - - log::debug!( - "INVOICES1: {:?}", - requestor.get_invoices::(None, None).await - ); - log::debug!( - "INVOICES2: {:?}", - requestor - .get_invoices::(Some(invoice_date), None) - .await - ); - log::debug!( - "INVOICES3: {:?}", - requestor.get_invoices::(Some(Utc::now()), None).await - ); - - log::info!("Accepting invoice..."); - let now = Utc::now(); - requestor - .accept_invoice( - &invoice.invoice_id, - &Acceptance { - total_amount_accepted: invoice.amount.clone(), - allocation_id: allocation.allocation_id, - }, - ) - .await?; - log::info!("Invoice accepted."); - - let invoice_events_accepted = provider - .get_invoice_events::( - Some(&invoice_events_received.first().unwrap().event_date), - Some(Duration::from_secs(1000)), - None, - args.app_session_id.clone(), - ) - .await - .unwrap(); - log::debug!("events 2: {:?}", &invoice_events_accepted); - - log::info!("Waiting for payment..."); - let timeout = Some(Duration::from_secs(1000)); // Should be enough for GLM transfer - - let mut payments = provider - .get_payments(Some(&now), timeout, None, args.app_session_id.clone()) - .await?; - - let signed_payments = provider - .get_signed_payments(Some(&now), None, None, args.app_session_id.clone()) - .await?; - - assert_eq!(payments.len(), 1); - assert_eq!(signed_payments.len(), 1); - - let payment = payments.pop().unwrap(); - assert!(payment.amount >= invoice.amount); - - log::info!("Payment verified correctly."); - - log::info!("Verifying invoice status..."); - let invoice = provider.get_invoice(&invoice.invoice_id).await?; - assert_eq!(invoice.status, DocumentStatus::Settled); - log::info!("Invoice status verified correctly."); - - let invoice_events_settled = provider - .get_invoice_events::( - Some(&invoice_events_accepted.first().unwrap().event_date), - Some(Duration::from_secs(1000)), - None, - args.app_session_id.clone(), - ) - .await - .unwrap(); - log::debug!("events 3: {:?}", &invoice_events_settled); - - log::info!(" 👍🏻 Example completed successfully ❤️"); - Ok(()) -} diff --git a/core/payment/examples/release_allocation.rs b/core/payment/examples/release_allocation.rs deleted file mode 100644 index ad1742af4b..0000000000 --- a/core/payment/examples/release_allocation.rs +++ /dev/null @@ -1,136 +0,0 @@ -use bigdecimal::BigDecimal; -use chrono::Utc; -use ya_client::payment::PaymentApi; -use ya_client::web::{rest_api_url, WebClient}; -use ya_client_model::payment::allocation::PaymentPlatformEnum; -use ya_client_model::payment::{Acceptance, NewAllocation, NewInvoice}; - -#[actix_rt::main] -async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); - std::env::set_var("RUST_LOG", log_level); - env_logger::init(); - - // Create requestor / provider PaymentApi - let provider_url = format!("{}provider/", rest_api_url()).parse().unwrap(); - let provider: PaymentApi = WebClient::builder() - .api_url(provider_url) - .build() - .interface()?; - let requestor_url = format!("{}requestor/", rest_api_url()).parse().unwrap(); - let requestor: PaymentApi = WebClient::builder() - .api_url(requestor_url) - .build() - .interface()?; - - log::info!("Creating allocation..."); - let accounts = requestor.get_requestor_accounts().await?; - let account = accounts.first().expect("No account available"); - let allocation = requestor - .create_allocation(&NewAllocation { - address: Some(account.address.clone()), - payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName( - account.platform.clone(), - )), - total_amount: BigDecimal::from(10u64), - timeout: None, - make_deposit: false, - deposit: None, - extend_timeout: None, - }) - .await?; - log::info!("Allocation created."); - - log::info!("Verifying allocation..."); - let allocations = requestor.get_allocations::(None, None).await?; - assert_eq!(allocations.len(), 1); - assert_eq!(allocations[0], allocation); - let allocation1 = requestor.get_allocation(&allocation.allocation_id).await?; - assert_eq!(allocation1, allocation); - log::info!("Done."); - - log::info!("Releasing allocation..."); - requestor - .release_allocation(&allocation.allocation_id) - .await?; - log::info!("Allocation released."); - - log::info!("Verifying allocation removal..."); - let allocations = requestor.get_allocations::(None, None).await?; - assert_eq!(allocations.len(), 0); - let result = requestor.get_allocation(&allocation.allocation_id).await; - assert!(result.is_err()); - log::info!("Done."); - - log::info!("Issuing invoice..."); - let invoice = provider - .issue_invoice(&NewInvoice { - agreement_id: "agreement_id".to_string(), - activity_ids: None, - amount: BigDecimal::from(1u64), - payment_due_date: Utc::now(), - }) - .await?; - log::info!("Invoice issued."); - - log::info!("Sending invoice..."); - provider.send_invoice(&invoice.invoice_id).await?; - log::info!("Invoice sent."); - - log::info!("Attempting to accept invoice..."); - let result = requestor - .accept_invoice( - &invoice.invoice_id, - &Acceptance { - total_amount_accepted: invoice.amount.clone(), - allocation_id: allocation.allocation_id, - }, - ) - .await; - assert!(result.is_err()); - log::info!("Failed to accept invoice (as expected)."); - - log::info!("Creating another allocation..."); - let allocation = requestor - .create_allocation(&NewAllocation { - address: Some(account.address.clone()), - payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName( - account.platform.clone(), - )), - total_amount: BigDecimal::from(10u64), - timeout: None, - make_deposit: false, - deposit: None, - extend_timeout: None, - }) - .await?; - log::info!("Allocation created."); - - log::info!("Accepting invoice..."); - requestor - .accept_invoice( - &invoice.invoice_id, - &Acceptance { - total_amount_accepted: invoice.amount.clone(), - allocation_id: allocation.allocation_id.clone(), - }, - ) - .await?; - log::info!("Invoice accepted."); - - log::info!("Releasing allocation..."); - requestor - .release_allocation(&allocation.allocation_id) - .await?; - log::info!("Allocation released."); - - log::info!("Verifying allocation removal..."); - let allocations = requestor.get_allocations::(None, None).await?; - assert_eq!(allocations.len(), 0); - let result = requestor.get_allocation(&allocation.allocation_id).await; - assert!(result.is_err()); - log::info!("Done."); - - log::info!(" 👍🏻 Example completed successfully ❤️"); - Ok(()) -} diff --git a/core/payment/examples/validate_allocation.rs b/core/payment/examples/validate_allocation.rs deleted file mode 100644 index 8909a11f0e..0000000000 --- a/core/payment/examples/validate_allocation.rs +++ /dev/null @@ -1,122 +0,0 @@ -use bigdecimal::BigDecimal; -use ya_client::payment::PaymentApi; -use ya_client::web::{rest_api_url, WebClient}; -use ya_client_model::payment::allocation::PaymentPlatformEnum; -use ya_client_model::payment::NewAllocation; -use ya_core_model::payment::local as pay; -use ya_service_bus::typed as bus; - -async fn get_requestor_balance_and_platform() -> anyhow::Result<(BigDecimal, String)> { - let account_list = bus::service(pay::BUS_ID) - .call(pay::GetAccounts {}) - .await??; - - for account in account_list.into_iter() { - if account.send { - let status = bus::service(pay::BUS_ID) - .call(pay::GetStatus { - address: account.address.clone(), - driver: account.driver, - network: Some(account.network), - token: Some(account.token), - after_timestamp: 0, - }) - .await??; - return Ok((status.amount, account.platform)); - } - } - - anyhow::bail!("Requestor account not found") -} - -#[actix_rt::main] -async fn main() -> anyhow::Result<()> { - let log_level = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned()); - std::env::set_var("RUST_LOG", log_level); - env_logger::init(); - - let requestor_url = format!("{}requestor/", rest_api_url()).parse().unwrap(); - let requestor: PaymentApi = WebClient::builder() - .api_url(requestor_url) - .build() - .interface()?; - - let (requestor_balance, payment_platform) = get_requestor_balance_and_platform().await?; - log::info!( - "Requestor balance: {}, platform: {}", - requestor_balance, - payment_platform - ); - - if "dummy-glm" == &payment_platform { - log::info!( - " 🖐 Example will not work with Dummy driver as it does not validate requests 💛" - ); - return Ok(()); - } - - log::info!("Attempting to create allocation with invalid address..."); - let result = requestor - .create_allocation(&NewAllocation { - address: Some("Definitely not a valid address".to_string()), - payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName( - payment_platform.clone(), - )), - total_amount: BigDecimal::from(1u64), - timeout: None, - make_deposit: false, - deposit: None, - extend_timeout: None, - }) - .await; - assert!(result.is_err()); - log::info!("Failed to create allocation (as expected)."); - - let new_allocation = NewAllocation { - address: None, // Use default address (i.e. identity) - payment_platform: Some(PaymentPlatformEnum::PaymentPlatformName(payment_platform)), - total_amount: requestor_balance, - timeout: None, - make_deposit: false, - deposit: None, - extend_timeout: None, - }; - - log::info!( - "Creating allocation for {} tGLM...", - &new_allocation.total_amount - ); - requestor.create_allocation(&new_allocation).await?; - log::info!("Allocation created."); - - log::info!( - "Creating another allocation for {} tGLM...", - &new_allocation.total_amount - ); - let allocation = requestor.create_allocation(&new_allocation).await?; - log::info!("Allocation created."); - - log::info!( - "Attempting to create another allocation for {} tGLM...", - &new_allocation.total_amount - ); - let result = requestor.create_allocation(&new_allocation).await; - assert!(result.is_err()); - log::info!("Failed to create allocation (as expected)."); - - log::info!("Releasing an allocation..."); - requestor - .release_allocation(&allocation.allocation_id) - .await?; - log::info!("Allocation released."); - - log::info!( - "Creating another allocation for {} tGLM...", - &new_allocation.total_amount - ); - requestor.create_allocation(&new_allocation).await?; - log::info!("Allocation created."); - - log::info!(" 👍🏻 Example completed successfully ❤️"); - Ok(()) -} From 44f8f7045d6c135a5b4aac45203969549179c241 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Aug 2024 19:40:18 +0200 Subject: [PATCH 075/125] Remove mentions of removed examples --- core/payment/examples/README.md | 173 ++++---------------------------- 1 file changed, 19 insertions(+), 154 deletions(-) diff --git a/core/payment/examples/README.md b/core/payment/examples/README.md index 6f06a40331..fec9cd33f1 100644 --- a/core/payment/examples/README.md +++ b/core/payment/examples/README.md @@ -3,168 +3,33 @@ ### Startup To start the API server (both provider & requestor) run the following commands: + ```shell script cd core/payment cp ../../.env-template .env (rm payment.db* || true) && cargo run --example payment_api ``` -To use erc20 instead of Dummy driver use `cargo run --example payment_api -- --driver=erc20 --platform=erc20-goerli-tglm` instead. +To use erc20 instead of Dummy driver +use `cargo run --example payment_api -- --driver=erc20 --platform=erc20-goerli-tglm` instead. ### Examples -To make sense of the included examples it is important to understand what parameters the example accepts and how they -can be used to change payment platform (driver, network, token). We list examples along with their parameters starting -from `payment_api` with is required to run any other example with payment platform that matches the `payment_api`'s platform. - -| Example | Parameters | Defaults | -|---------------------|------------------------------------------------|------------------------------------------------------------------------------------------------| -| payment_api | driver, platform | driver=`dummy`, platform=`dummy-glm` | -| account_ballance | | Same as `payment_api` | -| cancel_invoice | driver, network | driver=`dummy`, network=None | -| debit_note_flow | platform | platform=`dummy-glm` | +To make sense of the included examples it is important to understand what parameters the example accepts and how they +can be used to change payment platform (driver, network, token). We list examples along with their parameters starting +from `payment_api` with is required to run any other example with payment platform that matches the `payment_api`'s +platform. + +| Example | Parameters | Defaults | +|---------------------|----------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------| +| payment_api | driver, platform | driver=`dummy`, platform=`dummy-glm` | +| account_ballance | | Same as `payment_api` | +| cancel_invoice | driver, network | driver=`dummy`, network=None | +| debit_note_flow | platform | platform=`dummy-glm` | | get_accounts | <`provider_addr`>
<`requestor_addr`>
platform | `provider_addr` and `requestor_addr` are required, positional, `0x`-hex-encoded parameters. Platform=`dummy-glm` | -| invoice_flow | platform | platform=`dummy-glm` | -| market_decoration | | Same as `payment_api` | -| release_allocation | | Same as `payment_api` | -| validate_allocation | | Same as `payment_api` | - - -### Debit note flow - -To test the whole flow start the API server (see above) and run the debit_note_flow -example in another terminal: -```shell script -cd core/payment -cargo run --example debit_note_flow -``` -(**NOTE:** The example expects a clean database so might need to remove `payment.db` -and restart the API server.) - -Running examples with erc-20 payment driver, please wait until `payment_api` get funded and then run `debit_note_flow` with `--platform=erc20-rinkeby-tglm` parameter. - -##### Issue a debit node: -`POST` `http://127.0.0.1:7465/payment-api/v1/provider/debitNotes` - -Payload: -```json -{ - "activityId": "activity_id", - "totalAmountDue": "1.123456789012345678", - "usageCounterVector": { - "comment": "This field can contain anything", - "values": [1.222, 2.333, 4.555] - }, - "paymentDueDate": "2020-02-05T15:07:45.956Z" -} -``` -Don't forget to copy `debitNoteId` from the response! - -##### Send the issued debit note to requestor: -`POST` `http://127.0.0.1:7465/payment-api/v1/provider/debitNotes//send` - -##### See debit notes issued by the provider: -`GET` `http://127.0.0.1:7465/payment-api/v1/provider/debitNotes` - -##### See debit notes received by the requestor: -`GET` `http://127.0.0.1:7465/payment-api/v1/requestor/debitNotes` - -##### Accept a debit note: -`POST` `http://127.0.0.1:7465/payment-api/v1/requestor/debitNotes//accept` - -Payload: -```json -{ - "totalAmountAccepted": "1.123456789012345678", - "allocationId": "" -} -``` - -##### Listen for requestor's debit note events: -`GET` `http://127.0.0.1:7465/payment-api/v1/requestor/debitNoteEvents?timeout=` - -##### Listen for provider's debit note events: -`GET` `http://127.0.0.1:7465/payment-api/v1/provider/debitNoteEvents?timeout=` - -### Invoice flow - -To test the whole flow start the API server (see above) and run the invoice_flow -example in another terminal: -```shell script -cargo run --example invoice_flow -``` -(**NOTE:** The example expects a clean database so might need to remove `payment.db` -and restart the API server.) - -Running examples with erc-20 payment driver, please wait until `payment_api` get funded and then run `invoice_flow` with `--platform=erc20-rinkeby-tglm` parameter. - -##### Issue an invoice: -`POST` `http://127.0.0.1:7465/payment-api/v1/provider/invoices` - -Payload: -```json -{ - "agreementId": "agreement_id", - "activityIds": ["activity_id1", "activity_id2"], - "amount": "10.123456789012345678", - "paymentDueDate": "2020-02-05T15:07:45.956Z" -} -``` -Don't forget to copy `invoiceId` from the response! - -##### Send the issued invoice to requestor: -`POST` `http://127.0.0.1:7465/payment-api/v1/provider/invoices//send` - -##### See invoices issued by the provider: -`GET` `http://127.0.0.1:7465/payment-api/v1/provider/invoices` - -##### See invoices received by the requestor: -`GET` `http://127.0.0.1:7465/payment-api/v1/requestor/invoices` - -##### Accept an invoice: -`POST` `http://127.0.0.1:7465/payment-api/v1/requestor/invoices//accept` - -Payload: -```json -{ - "totalAmountAccepted": "10.123456789012345678", - "allocationId": "" -} -``` - -##### Listen for requestor's invoice events: -`GET` `http://127.0.0.1:7465/payment-api/v1/requestor/invoiceEvents?timeout=` +| invoice_flow | platform | platform=`dummy-glm` | +| market_decoration | | Same as `payment_api` | +| release_allocation | | Same as `payment_api` | +| validate_allocation | | Same as `payment_api` | -##### Listen for provider's invoice events: -`GET` `http://127.0.0.1:7465/payment-api/v1/provider/invoiceEvents?timeout=` - -### Allocations - -##### Create an allocation: -`POST` `http://127.0.0.1:7465/payment-api/v1/requestor/allocations` - -Payload: -```json -{ - "totalAmount": "100.123456789012345678", - "timeout": "2020-02-17T11:42:56.739Z", - "makeDeposit": false -} -``` -Don't forget to copy `allocationId` from the response! - -##### See all created allocations: -`GET` `http://127.0.0.1:7465/payment-api/v1/requestor/allocations` - -##### Release an allocation: -`DELETE` `http://127.0.0.1:7465/payment-api/v1/requestor/allocations/` - -### Payments - -##### See requestor's (sent) payments: -`GET` `http://127.0.0.1:7465/payment-api/v1/requestor/payments` - -##### See provider's (received) payments: -`GET` `http://127.0.0.1:7465/payment-api/v1/provider/payments` - -One can also listen for payments by adding `?timeout=` parameter. + From cb71ddd49b3b0ca66e59ff2762874d13888f9137 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 6 Aug 2024 20:23:53 +0200 Subject: [PATCH 076/125] Implement FakePayments that reponds with Ack to every GSB call --- core/payment/tests/test_allocation.rs | 8 +- core/payment/tests/test_debit_notes.rs | 4 +- core/payment/tests/test_invoices.rs | 4 +- core/payment/tests/test_payment_sync.rs | 53 ++++++ .../tests/tutorial_how_to_use_module_tests.rs | 5 +- .../framework-mocks/src/identity.rs | 6 +- .../test-framework/framework-mocks/src/net.rs | 4 + .../framework-mocks/src/node.rs | 32 +++- .../framework-mocks/src/payment.rs | 13 +- .../src/payment/fake_payment.rs | 169 ++++++++++++++++++ 10 files changed, 269 insertions(+), 29 deletions(-) create mode 100644 core/payment/tests/test_payment_sync.rs create mode 100644 test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs diff --git a/core/payment/tests/test_allocation.rs b/core/payment/tests/test_allocation.rs index 937dd88edc..ac3f59c83f 100644 --- a/core/payment/tests/test_allocation.rs +++ b/core/payment/tests/test_allocation.rs @@ -9,7 +9,7 @@ use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; -use ya_framework_mocks::net::{IMockNet, MockNet}; +use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::Driver; @@ -21,8 +21,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu let dir = temp_dir!("test_release_allocation")?; - let net = MockNet::new(); - net.bind_gsb(); + let net = MockNet::new().bind(); let node = MockNode::new(net, "node-1", dir.path()) .with_identity() @@ -170,8 +169,7 @@ async fn test_validate_allocation(ctx: &mut DroppableTestContext) -> anyhow::Res let dir = temp_dir!("test_validate_allocation")?; - let net = MockNet::new(); - net.bind_gsb(); + let net = MockNet::new().bind(); let node = MockNode::new(net, "node-1", dir.path()) .with_identity() diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs index e9bab9ae11..79e22db446 100644 --- a/core/payment/tests/test_debit_notes.rs +++ b/core/payment/tests/test_debit_notes.rs @@ -7,7 +7,6 @@ use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::{Acceptance, DocumentStatus, NewAllocation, NewDebitNote}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; -use ya_framework_basic::mocks::net::IMockNet; use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; use ya_framework_mocks::net::MockNet; @@ -23,8 +22,7 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< let dir = temp_dir!("test_debit_note_flow")?; let dir = dir.path(); - let net = MockNet::new(); - net.bind_gsb(); + let net = MockNet::new().bind(); let node = MockNode::new(net, "node-1", dir) .with_identity() diff --git a/core/payment/tests/test_invoices.rs b/core/payment/tests/test_invoices.rs index d5a3338cec..b530b3a979 100644 --- a/core/payment/tests/test_invoices.rs +++ b/core/payment/tests/test_invoices.rs @@ -8,7 +8,6 @@ use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::{Acceptance, DocumentStatus, NewAllocation, NewInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; -use ya_framework_basic::mocks::net::IMockNet; use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; use ya_framework_mocks::net::MockNet; @@ -24,8 +23,7 @@ async fn test_invoice_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let dir = temp_dir!("test_invoice_flow")?; let dir = dir.path(); - let net = MockNet::new(); - net.bind_gsb(); + let net = MockNet::new().bind(); let node = MockNode::new(net, "node-1", dir) .with_identity() diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs new file mode 100644 index 0000000000..d681330e91 --- /dev/null +++ b/core/payment/tests/test_payment_sync.rs @@ -0,0 +1,53 @@ +use test_context::test_context; + +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::{resource, temp_dir}; +use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::net::MockNet; +use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::Driver; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(false); + + let dir = temp_dir!("test_payment_sync")?; + let dir = dir.path(); + + let net = MockNet::new().bind(); + + let node1 = MockNode::new(net.clone(), "node-1", dir) + .with_identity() + .with_payment() + .with_fake_market(); + node1.bind_gsb().await?; + node1.start_server(ctx).await?; + + let node2 = MockNode::new(net, "node-2", dir).with_fake_payment(); + + let appkey_prov = node1 + .get_identity()? + .create_identity_key("provider") + .await?; + let appkey_req = node1 + .get_identity()? + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; + + let mut agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + node1 + .get_payment()? + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) + .await?; + + let requestor = node1.rest_payments(&appkey_req.key)?; + let provider = node1.rest_payments(&appkey_prov.key)?; + + Ok(()) +} diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 7396f4cee0..b3b1ee279e 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -10,7 +10,7 @@ use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; -use ya_framework_mocks::net::{IMockNet, MockNet}; +use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::Driver; @@ -37,8 +37,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Currently instantiating many MockNodes is not possible, but MockNet is necessary even // for communication on the same node, because messages directed to external GSB addresses `/net/0x437544...` // when NodeId belongs to local Node, need to be routed back. - let net = MockNet::new(); - net.bind_gsb(); + let net = MockNet::new().bind(); // Create MockNode which is container for all Golem modules and represents // single node in tests. diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index f4e81d4ce4..e5d1426c5a 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -15,17 +15,17 @@ use ya_service_api::{CliCtx, CommandOutput}; use crate::net::{IMockNet, MockNet}; #[derive(Clone)] -pub struct MockIdentity { +pub struct RealIdentity { net: MockNet, name: String, db: DbExecutor, } -impl MockIdentity { +impl RealIdentity { pub fn new(net: MockNet, testdir: &Path, name: &str) -> Self { let db = Self::create_db(testdir, "identity.db").unwrap(); - MockIdentity { + RealIdentity { net, name: name.to_string(), db, diff --git a/test-utils/test-framework/framework-mocks/src/net.rs b/test-utils/test-framework/framework-mocks/src/net.rs index 1cabc286d2..1ed1127f4f 100644 --- a/test-utils/test-framework/framework-mocks/src/net.rs +++ b/test-utils/test-framework/framework-mocks/src/net.rs @@ -83,6 +83,10 @@ impl MockNet { broadcast: Default::default(), } } + pub fn bind(self) -> Self { + self.bind_gsb(); + self + } fn translate_to(&self, id: NodeId, addr: &str) -> Result { let prefix = self.node_prefix(id)?; diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index a7bdc624ba..a6b3d88e39 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -6,7 +6,6 @@ use std::str::FromStr; use std::time::Duration; use url::Url; -use crate::activity::FakeActivity; use ya_client::payment::PaymentApi; use ya_client::web::WebClient; use ya_framework_basic::async_drop::DroppableTestContext; @@ -14,10 +13,12 @@ use ya_service_api_web::middleware::auth; use ya_service_api_web::middleware::cors::{AppKeyCors, CorsConfig}; use ya_service_api_web::rest_api_host_port; -use crate::identity::MockIdentity; +use crate::activity::FakeActivity; +use crate::identity::RealIdentity; use crate::market::FakeMarket; use crate::net::MockNet; -use crate::payment::MockPayment; +use crate::payment::fake_payment::FakePayment; +use crate::payment::RealPayment; /// Represents Node abstraction in tests. /// Provides functionality to instantiate selected modules and make tests setup easier. @@ -34,8 +35,10 @@ pub struct MockNode { rest_url: Url, - pub identity: Option, - pub payment: Option, + pub identity: Option, + pub payment: Option, + pub fake_payment: Option, + pub market: Option, pub activity: Option, } @@ -52,6 +55,7 @@ impl MockNode { rest_url: Self::generate_rest_url(), identity: None, payment: None, + fake_payment: None, market: None, activity: None, } @@ -59,7 +63,7 @@ impl MockNode { /// Use full wrapped Identity module for this node. pub fn with_identity(mut self) -> Self { - self.identity = Some(MockIdentity::new( + self.identity = Some(RealIdentity::new( self.net.clone(), &self.testdir, &self.name, @@ -69,7 +73,13 @@ impl MockNode { /// Use full wrapped Payment module for this node. pub fn with_payment(mut self) -> Self { - self.payment = Some(MockPayment::new(&self.name, &self.testdir)); + self.payment = Some(RealPayment::new(&self.name, &self.testdir)); + self + } + + /// Use fake Market module for this node. + pub fn with_fake_payment(mut self) -> Self { + self.fake_payment = Some(FakePayment::new(&self.name, &self.testdir)); self } @@ -85,13 +95,13 @@ impl MockNode { self } - pub fn get_identity(&self) -> anyhow::Result { + pub fn get_identity(&self) -> anyhow::Result { self.identity .clone() .ok_or_else(|| anyhow!("Identity ({}) is not initialized", self.name)) } - pub fn get_payment(&self) -> anyhow::Result { + pub fn get_payment(&self) -> anyhow::Result { self.payment .clone() .ok_or_else(|| anyhow!("Payment ({}) is not initialized", self.name)) @@ -122,6 +132,10 @@ impl MockNode { payment.bind_gsb().await?; } + if let Some(payment) = &self.fake_payment { + payment.bind_gsb().await?; + } + if let Some(market) = &self.market { market.bind_gsb().await?; } diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index 45e98cb034..bda34650c1 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -9,6 +9,7 @@ use ya_client::payment::PaymentApi; use ya_client_model::payment::Payment; use ya_core_model::driver::{driver_bus_id, Fund}; use ya_core_model::payment::local::BUS_ID; +use ya_core_model::payment::public; use ya_payment::api::web_scope; use ya_payment::config::Config; use ya_payment::migrations; @@ -21,6 +22,8 @@ use ya_service_bus::typed::Endpoint; use ya_dummy_driver as dummy; use ya_erc20_driver as erc20; +pub mod fake_payment; + #[derive(Clone, Debug, derive_more::Display)] pub enum Driver { #[display(fmt = "dummy")] @@ -39,7 +42,7 @@ impl Driver { } #[derive(Clone)] -pub struct MockPayment { +pub struct RealPayment { name: String, testdir: PathBuf, @@ -47,12 +50,12 @@ pub struct MockPayment { processor: Arc, } -impl MockPayment { +impl RealPayment { pub fn new(name: &str, testdir: &Path) -> Self { let db = Self::create_db(testdir, "payment.db").unwrap(); let processor = Arc::new(PaymentProcessor::new(db.clone())); - MockPayment { + RealPayment { name: name.to_string(), testdir: testdir.to_path_buf(), db, @@ -111,6 +114,10 @@ impl MockPayment { pub fn gsb_local_endpoint(&self) -> Endpoint { bus::service(BUS_ID) } + + pub fn gsb_public_endpoint(&self) -> Endpoint { + bus::service(public::BUS_ID) + } } #[async_trait::async_trait(?Send)] diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs new file mode 100644 index 0000000000..ee433868f1 --- /dev/null +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -0,0 +1,169 @@ +#![allow(unused)] + +use anyhow::anyhow; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use ya_core_model::payment::public::{ + AcceptDebitNote, AcceptInvoice, AcceptRejectError, Ack, CancelDebitNote, CancelError, + CancelInvoice, PaymentSync, PaymentSyncError, PaymentSyncRequest, PaymentSyncWithBytes, + RejectDebitNote, RejectInvoiceV2, SendDebitNote, SendError, SendInvoice, SendPayment, + SendSignedPayment, BUS_ID, +}; +use ya_payment::migrations; +use ya_payment::processor::PaymentProcessor; +use ya_persistence::executor::DbExecutor; +use ya_service_bus::typed::ServiceBinder; + +#[derive(Clone)] +pub struct FakePayment { + name: String, + testdir: PathBuf, + + db: DbExecutor, +} + +impl FakePayment { + pub fn new(name: &str, testdir: &Path) -> Self { + let db = Self::create_db(testdir, "payment").unwrap(); + FakePayment { + name: name.to_string(), + testdir: testdir.to_path_buf(), + db, + } + } + + fn create_db(_testdir: &Path, name: &str) -> anyhow::Result { + let db = DbExecutor::in_memory(&format!("{name}.payment.db")) + .map_err(|e| anyhow!("Failed to create db [{name:?}]. Error: {e}"))?; + db.apply_migration(migrations::run_with_output)?; + Ok(db) + } + + pub async fn bind_gsb(&self) -> anyhow::Result<()> { + log::info!("FakePayment ({}) - binding GSB", self.name); + + ServiceBinder::new(BUS_ID, &self.db, ()) + .bind(send_debit_note) + .bind(accept_debit_note) + .bind(reject_debit_note) + .bind(cancel_debit_note) + .bind(send_invoice) + .bind(accept_invoice) + .bind(reject_invoice) + .bind(cancel_invoice) + .bind(sync_request) + .bind(send_payment) + .bind(send_payment_with_bytes) + .bind(sync_payment) + .bind(sync_payment_with_bytes); + + Ok(()) + } +} + +async fn send_debit_note( + db: DbExecutor, + sender_id: String, + msg: SendDebitNote, +) -> Result { + Ok(Ack {}) +} + +async fn accept_debit_note( + db: DbExecutor, + sender_id: String, + msg: AcceptDebitNote, +) -> Result { + Ok(Ack {}) +} + +async fn reject_debit_note( + db: DbExecutor, + sender: String, + msg: RejectDebitNote, +) -> Result { + Ok(Ack {}) +} + +async fn cancel_debit_note( + db: DbExecutor, + sender: String, + msg: CancelDebitNote, +) -> Result { + Ok(Ack {}) +} + +// *************************** INVOICE **************************** + +async fn send_invoice( + db: DbExecutor, + sender_id: String, + msg: SendInvoice, +) -> Result { + Ok(Ack {}) +} + +async fn accept_invoice( + db: DbExecutor, + sender_id: String, + msg: AcceptInvoice, +) -> Result { + Ok(Ack {}) +} + +async fn reject_invoice( + db: DbExecutor, + sender_id: String, + msg: RejectInvoiceV2, +) -> Result { + Ok(Ack {}) +} + +async fn cancel_invoice( + db: DbExecutor, + sender_id: String, + msg: CancelInvoice, +) -> Result { + Ok(Ack {}) +} + +async fn send_payment( + db: DbExecutor, + sender_id: String, + msg: SendPayment, +) -> Result { + Ok(Ack {}) +} + +async fn send_payment_with_bytes( + db: DbExecutor, + sender_id: String, + msg: SendSignedPayment, +) -> Result { + Ok(Ack {}) +} + +async fn sync_payment( + db: DbExecutor, + sender_id: String, + msg: PaymentSync, +) -> Result { + Ok(Ack {}) +} + +async fn sync_payment_with_bytes( + db: DbExecutor, + sender_id: String, + msg: PaymentSyncWithBytes, +) -> Result { + Ok(Ack {}) +} + +async fn sync_request( + db: DbExecutor, + sender_id: String, + msg: PaymentSyncRequest, +) -> Result { + Ok(Ack {}) +} From df6b706ddb1badd26d5e26008b6f8452cab4bed8 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 7 Aug 2024 11:21:36 +0200 Subject: [PATCH 077/125] Fix LogErr trait module log on windows --- utils/std-utils/src/result.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/utils/std-utils/src/result.rs b/utils/std-utils/src/result.rs index 98a80d858f..11376461c9 100644 --- a/utils/std-utils/src/result.rs +++ b/utils/std-utils/src/result.rs @@ -39,8 +39,9 @@ impl LogErr for Result { fn log_error(self, message: &str, log_level: Level) -> Result { if let Err(e) = self { // It will return file not module path, so it will differ from the original log macro. - let module = std::panic::Location::caller().file(); - log(module, log_level, message, &e); + let module = std::panic::Location::caller().file().to_string(); + let module = module.replace('\\', "/"); + log(&module, log_level, message, &e); Err(e) } else { self From 7b3824d7e618a71b8adc2c9ecaf1eeb991036fe9 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Wed, 7 Aug 2024 12:12:22 +0200 Subject: [PATCH 078/125] Updated to newest Goth (update after final changes in LockContract) --- goth_tests/assets/docker/docker-compose.yml | 6 +- goth_tests/poetry.lock | 578 ++++++++++---------- goth_tests/pyproject.toml | 2 +- 3 files changed, 300 insertions(+), 286 deletions(-) diff --git a/goth_tests/assets/docker/docker-compose.yml b/goth_tests/assets/docker/docker-compose.yml index 607ab6a75e..11fe2bcca0 100644 --- a/goth_tests/assets/docker/docker-compose.yml +++ b/goth_tests/assets/docker/docker-compose.yml @@ -26,17 +26,17 @@ services: - "host.docker.internal:host-gateway" ethereum-mainnet: - image: ghcr.io/golemfactory/gnt2/gnt2-docker-yagna:b628fa5bfe0d + image: ghcr.io/golemfactory/gnt2/gnt2-docker-yagna:4c020aca5a0d environment: - GANACHE_CHAIN_ID=1 ethereum-holesky: - image: ghcr.io/golemfactory/gnt2/gnt2-docker-yagna:b628fa5bfe0d + image: ghcr.io/golemfactory/gnt2/gnt2-docker-yagna:4c020aca5a0d environment: - GANACHE_CHAIN_ID=17000 ethereum-polygon: - image: ghcr.io/golemfactory/gnt2/gnt2-docker-yagna:b628fa5bfe0d + image: ghcr.io/golemfactory/gnt2/gnt2-docker-yagna:4c020aca5a0d environment: - GANACHE_CHAIN_ID=137 diff --git a/goth_tests/poetry.lock b/goth_tests/poetry.lock index 34d6f41382..6ca3e39dca 100644 --- a/goth_tests/poetry.lock +++ b/goth_tests/poetry.lock @@ -1,91 +1,103 @@ # This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +[[package]] +name = "aiohappyeyeballs" +version = "2.3.4" +description = "Happy Eyeballs for asyncio" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "aiohappyeyeballs-2.3.4-py3-none-any.whl", hash = "sha256:40a16ceffcf1fc9e142fd488123b2e218abc4188cf12ac20c67200e1579baa42"}, + {file = "aiohappyeyeballs-2.3.4.tar.gz", hash = "sha256:7e1ae8399c320a8adec76f6c919ed5ceae6edd4c3672f4d9eae2b27e37c80ff6"}, +] + [[package]] name = "aiohttp" -version = "3.9.5" +version = "3.10.1" description = "Async http client/server framework (asyncio)" optional = false python-versions = ">=3.8" files = [ - {file = "aiohttp-3.9.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7"}, - {file = "aiohttp-3.9.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d6b3f1fabe465e819aed2c421a6743d8debbde79b6a8600739300630a01bf2c"}, - {file = "aiohttp-3.9.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ae79c1bc12c34082d92bf9422764f799aee4746fd7a392db46b7fd357d4a17a"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d3ebb9e1316ec74277d19c5f482f98cc65a73ccd5430540d6d11682cd857430"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84dabd95154f43a2ea80deffec9cb44d2e301e38a0c9d331cc4aa0166fe28ae3"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8a02fbeca6f63cb1f0475c799679057fc9268b77075ab7cf3f1c600e81dd46b"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c26959ca7b75ff768e2776d8055bf9582a6267e24556bb7f7bd29e677932be72"}, - {file = "aiohttp-3.9.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:714d4e5231fed4ba2762ed489b4aec07b2b9953cf4ee31e9871caac895a839c0"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7a6a8354f1b62e15d48e04350f13e726fa08b62c3d7b8401c0a1314f02e3558"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c413016880e03e69d166efb5a1a95d40f83d5a3a648d16486592c49ffb76d0db"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ad7f2919d7dac062f24d6f5fe95d401597fbb015a25771f85e692d043c9d7832"}, - {file = "aiohttp-3.9.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:702e2c7c187c1a498a4e2b03155d52658fdd6fda882d3d7fbb891a5cf108bb10"}, - {file = "aiohttp-3.9.5-cp310-cp310-win32.whl", hash = "sha256:67c3119f5ddc7261d47163ed86d760ddf0e625cd6246b4ed852e82159617b5fb"}, - {file = "aiohttp-3.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:471f0ef53ccedec9995287f02caf0c068732f026455f07db3f01a46e49d76bbb"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e0ae53e33ee7476dd3d1132f932eeb39bf6125083820049d06edcdca4381f342"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c088c4d70d21f8ca5c0b8b5403fe84a7bc8e024161febdd4ef04575ef35d474d"}, - {file = "aiohttp-3.9.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:639d0042b7670222f33b0028de6b4e2fad6451462ce7df2af8aee37dcac55424"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f26383adb94da5e7fb388d441bf09c61e5e35f455a3217bfd790c6b6bc64b2ee"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:66331d00fb28dc90aa606d9a54304af76b335ae204d1836f65797d6fe27f1ca2"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ff550491f5492ab5ed3533e76b8567f4b37bd2995e780a1f46bca2024223233"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f22eb3a6c1080d862befa0a89c380b4dafce29dc6cd56083f630073d102eb595"}, - {file = "aiohttp-3.9.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a81b1143d42b66ffc40a441379387076243ef7b51019204fd3ec36b9f69e77d6"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f64fd07515dad67f24b6ea4a66ae2876c01031de91c93075b8093f07c0a2d93d"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:93e22add827447d2e26d67c9ac0161756007f152fdc5210277d00a85f6c92323"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:55b39c8684a46e56ef8c8d24faf02de4a2b2ac60d26cee93bc595651ff545de9"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4715a9b778f4293b9f8ae7a0a7cef9829f02ff8d6277a39d7f40565c737d3771"}, - {file = "aiohttp-3.9.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:afc52b8d969eff14e069a710057d15ab9ac17cd4b6753042c407dcea0e40bf75"}, - {file = "aiohttp-3.9.5-cp311-cp311-win32.whl", hash = "sha256:b3df71da99c98534be076196791adca8819761f0bf6e08e07fd7da25127150d6"}, - {file = "aiohttp-3.9.5-cp311-cp311-win_amd64.whl", hash = "sha256:88e311d98cc0bf45b62fc46c66753a83445f5ab20038bcc1b8a1cc05666f428a"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:c7a4b7a6cf5b6eb11e109a9755fd4fda7d57395f8c575e166d363b9fc3ec4678"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0a158704edf0abcac8ac371fbb54044f3270bdbc93e254a82b6c82be1ef08f3c"}, - {file = "aiohttp-3.9.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d153f652a687a8e95ad367a86a61e8d53d528b0530ef382ec5aaf533140ed00f"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82a6a97d9771cb48ae16979c3a3a9a18b600a8505b1115cfe354dfb2054468b4"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60cdbd56f4cad9f69c35eaac0fbbdf1f77b0ff9456cebd4902f3dd1cf096464c"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8676e8fd73141ded15ea586de0b7cda1542960a7b9ad89b2b06428e97125d4fa"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da00da442a0e31f1c69d26d224e1efd3a1ca5bcbf210978a2ca7426dfcae9f58"}, - {file = "aiohttp-3.9.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18f634d540dd099c262e9f887c8bbacc959847cfe5da7a0e2e1cf3f14dbf2daf"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:320e8618eda64e19d11bdb3bd04ccc0a816c17eaecb7e4945d01deee2a22f95f"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:2faa61a904b83142747fc6a6d7ad8fccff898c849123030f8e75d5d967fd4a81"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:8c64a6dc3fe5db7b1b4d2b5cb84c4f677768bdc340611eca673afb7cf416ef5a"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:393c7aba2b55559ef7ab791c94b44f7482a07bf7640d17b341b79081f5e5cd1a"}, - {file = "aiohttp-3.9.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c671dc117c2c21a1ca10c116cfcd6e3e44da7fcde37bf83b2be485ab377b25da"}, - {file = "aiohttp-3.9.5-cp312-cp312-win32.whl", hash = "sha256:5a7ee16aab26e76add4afc45e8f8206c95d1d75540f1039b84a03c3b3800dd59"}, - {file = "aiohttp-3.9.5-cp312-cp312-win_amd64.whl", hash = "sha256:5ca51eadbd67045396bc92a4345d1790b7301c14d1848feaac1d6a6c9289e888"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:694d828b5c41255e54bc2dddb51a9f5150b4eefa9886e38b52605a05d96566e8"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0605cc2c0088fcaae79f01c913a38611ad09ba68ff482402d3410bf59039bfb8"}, - {file = "aiohttp-3.9.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4558e5012ee03d2638c681e156461d37b7a113fe13970d438d95d10173d25f78"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dbc053ac75ccc63dc3a3cc547b98c7258ec35a215a92bd9f983e0aac95d3d5b"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4109adee842b90671f1b689901b948f347325045c15f46b39797ae1bf17019de"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6ea1a5b409a85477fd8e5ee6ad8f0e40bf2844c270955e09360418cfd09abac"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3c2890ca8c59ee683fd09adf32321a40fe1cf164e3387799efb2acebf090c11"}, - {file = "aiohttp-3.9.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3916c8692dbd9d55c523374a3b8213e628424d19116ac4308e434dbf6d95bbdd"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8d1964eb7617907c792ca00b341b5ec3e01ae8c280825deadbbd678447b127e1"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d5ab8e1f6bee051a4bf6195e38a5c13e5e161cb7bad83d8854524798bd9fcd6e"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:52c27110f3862a1afbcb2af4281fc9fdc40327fa286c4625dfee247c3ba90156"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:7f64cbd44443e80094309875d4f9c71d0401e966d191c3d469cde4642bc2e031"}, - {file = "aiohttp-3.9.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b4f72fbb66279624bfe83fd5eb6aea0022dad8eec62b71e7bf63ee1caadeafe"}, - {file = "aiohttp-3.9.5-cp38-cp38-win32.whl", hash = "sha256:6380c039ec52866c06d69b5c7aad5478b24ed11696f0e72f6b807cfb261453da"}, - {file = "aiohttp-3.9.5-cp38-cp38-win_amd64.whl", hash = "sha256:da22dab31d7180f8c3ac7c7635f3bcd53808f374f6aa333fe0b0b9e14b01f91a"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1732102949ff6087589408d76cd6dea656b93c896b011ecafff418c9661dc4ed"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6021d296318cb6f9414b48e6a439a7f5d1f665464da507e8ff640848ee2a58a"}, - {file = "aiohttp-3.9.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:239f975589a944eeb1bad26b8b140a59a3a320067fb3cd10b75c3092405a1372"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b7b30258348082826d274504fbc7c849959f1989d86c29bc355107accec6cfb"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cd2adf5c87ff6d8b277814a28a535b59e20bfea40a101db6b3bdca7e9926bc24"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9a3d838441bebcf5cf442700e3963f58b5c33f015341f9ea86dcd7d503c07e2"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e3a1ae66e3d0c17cf65c08968a5ee3180c5a95920ec2731f53343fac9bad106"}, - {file = "aiohttp-3.9.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9c69e77370cce2d6df5d12b4e12bdcca60c47ba13d1cbbc8645dd005a20b738b"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0cbf56238f4bbf49dab8c2dc2e6b1b68502b1e88d335bea59b3f5b9f4c001475"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d1469f228cd9ffddd396d9948b8c9cd8022b6d1bf1e40c6f25b0fb90b4f893ed"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:45731330e754f5811c314901cebdf19dd776a44b31927fa4b4dbecab9e457b0c"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:3fcb4046d2904378e3aeea1df51f697b0467f2aac55d232c87ba162709478c46"}, - {file = "aiohttp-3.9.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8cf142aa6c1a751fcb364158fd710b8a9be874b81889c2bd13aa8893197455e2"}, - {file = "aiohttp-3.9.5-cp39-cp39-win32.whl", hash = "sha256:7b179eea70833c8dee51ec42f3b4097bd6370892fa93f510f76762105568cf09"}, - {file = "aiohttp-3.9.5-cp39-cp39-win_amd64.whl", hash = "sha256:38d80498e2e169bc61418ff36170e0aad0cd268da8b38a17c4cf29d254a8b3f1"}, - {file = "aiohttp-3.9.5.tar.gz", hash = "sha256:edea7d15772ceeb29db4aff55e482d4bcfb6ae160ce144f2682de02f6d693551"}, + {file = "aiohttp-3.10.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:47b4c2412960e64d97258f40616efddaebcb34ff664c8a972119ed38fac2a62c"}, + {file = "aiohttp-3.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e7dbf637f87dd315fa1f36aaed8afa929ee2c607454fb7791e74c88a0d94da59"}, + {file = "aiohttp-3.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c8fb76214b5b739ce59e2236a6489d9dc3483649cfd6f563dbf5d8e40dbdd57d"}, + {file = "aiohttp-3.10.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c577cdcf8f92862363b3d598d971c6a84ed8f0bf824d4cc1ce70c2fb02acb4a"}, + {file = "aiohttp-3.10.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:777e23609899cb230ad2642b4bdf1008890f84968be78de29099a8a86f10b261"}, + {file = "aiohttp-3.10.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b07286a1090483799599a2f72f76ac396993da31f6e08efedb59f40876c144fa"}, + {file = "aiohttp-3.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9db600a86414a9a653e3c1c7f6a2f6a1894ab8f83d11505247bd1b90ad57157"}, + {file = "aiohttp-3.10.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01c3f1eb280008e51965a8d160a108c333136f4a39d46f516c64d2aa2e6a53f2"}, + {file = "aiohttp-3.10.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f5dd109a925fee4c9ac3f6a094900461a2712df41745f5d04782ebcbe6479ccb"}, + {file = "aiohttp-3.10.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:8c81ff4afffef9b1186639506d70ea90888218f5ddfff03870e74ec80bb59970"}, + {file = "aiohttp-3.10.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:2a384dfbe8bfebd203b778a30a712886d147c61943675f4719b56725a8bbe803"}, + {file = "aiohttp-3.10.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:b9fb6508893dc31cfcbb8191ef35abd79751db1d6871b3e2caee83959b4d91eb"}, + {file = "aiohttp-3.10.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:88596384c3bec644a96ae46287bb646d6a23fa6014afe3799156aef42669c6bd"}, + {file = "aiohttp-3.10.1-cp310-cp310-win32.whl", hash = "sha256:68164d43c580c2e8bf8e0eb4960142919d304052ccab92be10250a3a33b53268"}, + {file = "aiohttp-3.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:d6bbe2c90c10382ca96df33b56e2060404a4f0f88673e1e84b44c8952517e5f3"}, + {file = "aiohttp-3.10.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f6979b4f20d3e557a867da9d9227de4c156fcdcb348a5848e3e6190fd7feb972"}, + {file = "aiohttp-3.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:03c0c380c83f8a8d4416224aafb88d378376d6f4cadebb56b060688251055cd4"}, + {file = "aiohttp-3.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c2b104e81b3c3deba7e6f5bc1a9a0e9161c380530479970766a6655b8b77c7c"}, + {file = "aiohttp-3.10.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b023b68c61ab0cd48bd38416b421464a62c381e32b9dc7b4bdfa2905807452a4"}, + {file = "aiohttp-3.10.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a07c76a82390506ca0eabf57c0540cf5a60c993c442928fe4928472c4c6e5e6"}, + {file = "aiohttp-3.10.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:41d8dab8c64ded1edf117d2a64f353efa096c52b853ef461aebd49abae979f16"}, + {file = "aiohttp-3.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:615348fab1a9ef7d0960a905e83ad39051ae9cb0d2837da739b5d3a7671e497a"}, + {file = "aiohttp-3.10.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:256ee6044214ee9d66d531bb374f065ee94e60667d6bbeaa25ca111fc3997158"}, + {file = "aiohttp-3.10.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d5bb926805022508b7ddeaad957f1fce7a8d77532068d7bdb431056dc630cd"}, + {file = "aiohttp-3.10.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:028faf71b338f069077af6315ad54281612705d68889f5d914318cbc2aab0d50"}, + {file = "aiohttp-3.10.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5c12310d153b27aa630750be44e79313acc4e864c421eb7d2bc6fa3429c41bf8"}, + {file = "aiohttp-3.10.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:de1a91d5faded9054957ed0a9e01b9d632109341942fc123947ced358c5d9009"}, + {file = "aiohttp-3.10.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9c186b270979fb1dee3ababe2d12fb243ed7da08b30abc83ebac3a928a4ddb15"}, + {file = "aiohttp-3.10.1-cp311-cp311-win32.whl", hash = "sha256:4a9ce70f5e00380377aac0e568abd075266ff992be2e271765f7b35d228a990c"}, + {file = "aiohttp-3.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:a77c79bac8d908d839d32c212aef2354d2246eb9deb3e2cb01ffa83fb7a6ea5d"}, + {file = "aiohttp-3.10.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:2212296cdb63b092e295c3e4b4b442e7b7eb41e8a30d0f53c16d5962efed395d"}, + {file = "aiohttp-3.10.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4dcb127ca3eb0a61205818a606393cbb60d93b7afb9accd2fd1e9081cc533144"}, + {file = "aiohttp-3.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb8b79a65332e1a426ccb6290ce0409e1dc16b4daac1cc5761e059127fa3d134"}, + {file = "aiohttp-3.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68cc24f707ed9cb961f6ee04020ca01de2c89b2811f3cf3361dc7c96a14bfbcc"}, + {file = "aiohttp-3.10.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cb54f5725b4b37af12edf6c9e834df59258c82c15a244daa521a065fbb11717"}, + {file = "aiohttp-3.10.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:51d03e948e53b3639ce4d438f3d1d8202898ec6655cadcc09ec99229d4adc2a9"}, + {file = "aiohttp-3.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:786299d719eb5d868f161aeec56d589396b053925b7e0ce36e983d30d0a3e55c"}, + {file = "aiohttp-3.10.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abda4009a30d51d3f06f36bc7411a62b3e647fa6cc935ef667e3e3d3a7dd09b1"}, + {file = "aiohttp-3.10.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:67f7639424c313125213954e93a6229d3a1d386855d70c292a12628f600c7150"}, + {file = "aiohttp-3.10.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8e5a26d7aac4c0d8414a347da162696eea0629fdce939ada6aedf951abb1d745"}, + {file = "aiohttp-3.10.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:120548d89f14b76a041088b582454d89389370632ee12bf39d919cc5c561d1ca"}, + {file = "aiohttp-3.10.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f5293726943bdcea24715b121d8c4ae12581441d22623b0e6ab12d07ce85f9c4"}, + {file = "aiohttp-3.10.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1f8605e573ed6c44ec689d94544b2c4bb1390aaa723a8b5a2cc0a5a485987a68"}, + {file = "aiohttp-3.10.1-cp312-cp312-win32.whl", hash = "sha256:e7168782621be4448d90169a60c8b37e9b0926b3b79b6097bc180c0a8a119e73"}, + {file = "aiohttp-3.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:8fbf8c0ded367c5c8eaf585f85ca8dd85ff4d5b73fb8fe1e6ac9e1b5e62e11f7"}, + {file = "aiohttp-3.10.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:54b7f4a20d7cc6bfa4438abbde069d417bb7a119f870975f78a2b99890226d55"}, + {file = "aiohttp-3.10.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2fa643ca990323db68911b92f3f7a0ca9ae300ae340d0235de87c523601e58d9"}, + {file = "aiohttp-3.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d8311d0d690487359fe2247ec5d2cac9946e70d50dced8c01ce9e72341c21151"}, + {file = "aiohttp-3.10.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222821c60b8f6a64c5908cb43d69c0ee978a1188f6a8433d4757d39231b42cdb"}, + {file = "aiohttp-3.10.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e7b55d9ede66af7feb6de87ff277e0ccf6d51c7db74cc39337fe3a0e31b5872d"}, + {file = "aiohttp-3.10.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5a95151a5567b3b00368e99e9c5334a919514f60888a6b6d2054fea5e66e527e"}, + {file = "aiohttp-3.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e9e9171d2fe6bfd9d3838a6fe63b1e91b55e0bf726c16edf265536e4eafed19"}, + {file = "aiohttp-3.10.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a57e73f9523e980f6101dc9a83adcd7ac0006ea8bf7937ca3870391c7bb4f8ff"}, + {file = "aiohttp-3.10.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:0df51a3d70a2bfbb9c921619f68d6d02591f24f10e9c76de6f3388c89ed01de6"}, + {file = "aiohttp-3.10.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:b0de63ff0307eac3961b4af74382d30220d4813f36b7aaaf57f063a1243b4214"}, + {file = "aiohttp-3.10.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:8db9b749f589b5af8e4993623dbda6716b2b7a5fcb0fa2277bf3ce4b278c7059"}, + {file = "aiohttp-3.10.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:6b14c19172eb53b63931d3e62a9749d6519f7c121149493e6eefca055fcdb352"}, + {file = "aiohttp-3.10.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5cd57ad998e3038aa87c38fe85c99ed728001bf5dde8eca121cadee06ee3f637"}, + {file = "aiohttp-3.10.1-cp38-cp38-win32.whl", hash = "sha256:df31641e3f02b77eb3c5fb63c0508bee0fc067cf153da0e002ebbb0db0b6d91a"}, + {file = "aiohttp-3.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:93094eba50bc2ad4c40ff4997ead1fdcd41536116f2e7d6cfec9596a8ecb3615"}, + {file = "aiohttp-3.10.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:440954ddc6b77257e67170d57b1026aa9545275c33312357472504eef7b4cc0b"}, + {file = "aiohttp-3.10.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f9f8beed277488a52ee2b459b23c4135e54d6a819eaba2e120e57311015b58e9"}, + {file = "aiohttp-3.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8a8221a63602008550022aa3a4152ca357e1dde7ab3dd1da7e1925050b56863"}, + {file = "aiohttp-3.10.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a702bd3663b5cbf3916e84bf332400d24cdb18399f0877ca6b313ce6c08bfb43"}, + {file = "aiohttp-3.10.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1988b370536eb14f0ce7f3a4a5b422ab64c4e255b3f5d7752c5f583dc8c967fc"}, + {file = "aiohttp-3.10.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7ccf1f0a304352c891d124ac1a9dea59b14b2abed1704aaa7689fc90ef9c5be1"}, + {file = "aiohttp-3.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc3ea6ef2a83edad84bbdb5d96e22f587b67c68922cd7b6f9d8f24865e655bcf"}, + {file = "aiohttp-3.10.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:89b47c125ab07f0831803b88aeb12b04c564d5f07a1c1a225d4eb4d2f26e8b5e"}, + {file = "aiohttp-3.10.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:21778552ef3d44aac3278cc6f6d13a6423504fa5f09f2df34bfe489ed9ded7f5"}, + {file = "aiohttp-3.10.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:bde0693073fd5e542e46ea100aa6c1a5d36282dbdbad85b1c3365d5421490a92"}, + {file = "aiohttp-3.10.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:bf66149bb348d8e713f3a8e0b4f5b952094c2948c408e1cfef03b49e86745d60"}, + {file = "aiohttp-3.10.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:587237571a85716d6f71f60d103416c9df7d5acb55d96d3d3ced65f39bff9c0c"}, + {file = "aiohttp-3.10.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:bfe33cba6e127d0b5b417623c9aa621f0a69f304742acdca929a9fdab4593693"}, + {file = "aiohttp-3.10.1-cp39-cp39-win32.whl", hash = "sha256:9fbff00646cf8211b330690eb2fd64b23e1ce5b63a342436c1d1d6951d53d8dd"}, + {file = "aiohttp-3.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:5951c328f9ac42d7bce7a6ded535879bc9ae13032818d036749631fa27777905"}, + {file = "aiohttp-3.10.1.tar.gz", hash = "sha256:8b0d058e4e425d3b45e8ec70d49b402f4d6b21041e674798b1f91ba027c73f28"}, ] [package.dependencies] +aiohappyeyeballs = ">=2.3.0" aiosignal = ">=1.1.2" async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" @@ -94,7 +106,7 @@ multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "brotlicffi"] +speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"] [[package]] name = "aiosignal" @@ -145,22 +157,22 @@ files = [ [[package]] name = "attrs" -version = "23.2.0" +version = "24.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, - {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, ] [package.extras] -cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[tests]", "pre-commit"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] -tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] -tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "black" @@ -346,24 +358,24 @@ websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "dpath" -version = "2.1.6" +version = "2.2.0" description = "Filesystem-like pathing and searching for dictionaries" optional = false python-versions = ">=3.7" files = [ - {file = "dpath-2.1.6-py3-none-any.whl", hash = "sha256:31407395b177ab63ef72e2f6ae268c15e938f2990a8ecf6510f5686c02b6db73"}, - {file = "dpath-2.1.6.tar.gz", hash = "sha256:f1e07c72e8605c6a9e80b64bc8f42714de08a789c7de417e49c3f87a19692e47"}, + {file = "dpath-2.2.0-py3-none-any.whl", hash = "sha256:b330a375ded0a0d2ed404440f6c6a715deae5313af40bbb01c8a41d891900576"}, + {file = "dpath-2.2.0.tar.gz", hash = "sha256:34f7e630dc55ea3f219e555726f5da4b4b25f2200319c8e6902c394258dd6a3e"}, ] [[package]] name = "exceptiongroup" -version = "1.2.1" +version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, - {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, ] [package.extras] @@ -371,13 +383,13 @@ test = ["pytest (>=6)"] [[package]] name = "fastcore" -version = "1.5.38" +version = "1.6.3" description = "Python supercharged for fastai development" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "fastcore-1.5.38-py3-none-any.whl", hash = "sha256:327f011613c986e7f627f63d1d9993c8d6de116c586df94d85806fbfbe45e52a"}, - {file = "fastcore-1.5.38.tar.gz", hash = "sha256:7732403778de9bc2b25bf52617c7fbb9e7ae96010f534a5f00f7e6dee73f1d39"}, + {file = "fastcore-1.6.3-py3-none-any.whl", hash = "sha256:2d079ac224e6f9e1db81aa6d1206b99a54fcf324c3dab914523bbff5bfcaa887"}, + {file = "fastcore-1.6.3.tar.gz", hash = "sha256:87731839bd729698ac177969bf494596f5f703df8739f67c52f51a178ceab464"}, ] [package.dependencies] @@ -527,8 +539,8 @@ ya-tmp-pyapi = "=0.7.0" [package.source] type = "git" url = "https://github.com/golemfactory/goth.git" -reference = "cda4aa83a72622cd97de45d8ce9066750c3b522e" -resolved_reference = "cda4aa83a72622cd97de45d8ce9066750c3b522e" +reference = "3891382c4a18e0e82a735dfe2280a6287e8c268b" +resolved_reference = "3891382c4a18e0e82a735dfe2280a6287e8c268b" [[package]] name = "idna" @@ -722,44 +734,44 @@ files = [ [[package]] name = "mypy" -version = "1.10.0" +version = "1.11.1" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, - {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, - {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, - {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, - {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, - {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, - {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, - {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, - {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, - {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, - {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, - {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, - {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, - {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, - {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, - {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, - {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, - {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, - {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, - {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, - {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, - {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, - {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, - {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, - {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, - {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, - {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, + {file = "mypy-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a32fc80b63de4b5b3e65f4be82b4cfa362a46702672aa6a0f443b4689af7008c"}, + {file = "mypy-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1952f5ea8a5a959b05ed5f16452fddadbaae48b5d39235ab4c3fc444d5fd411"}, + {file = "mypy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1e30dc3bfa4e157e53c1d17a0dad20f89dc433393e7702b813c10e200843b03"}, + {file = "mypy-1.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c63350af88f43a66d3dfeeeb8d77af34a4f07d760b9eb3a8697f0386c7590b4"}, + {file = "mypy-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:a831671bad47186603872a3abc19634f3011d7f83b083762c942442d51c58d58"}, + {file = "mypy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7b6343d338390bb946d449677726edf60102a1c96079b4f002dedff375953fc5"}, + {file = "mypy-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4fe9f4e5e521b458d8feb52547f4bade7ef8c93238dfb5bbc790d9ff2d770ca"}, + {file = "mypy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:886c9dbecc87b9516eff294541bf7f3655722bf22bb898ee06985cd7269898de"}, + {file = "mypy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca4a60e1dd9fd0193ae0067eaeeb962f2d79e0d9f0f66223a0682f26ffcc809"}, + {file = "mypy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:0bd53faf56de9643336aeea1c925012837432b5faf1701ccca7fde70166ccf72"}, + {file = "mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8"}, + {file = "mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a"}, + {file = "mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417"}, + {file = "mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e"}, + {file = "mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525"}, + {file = "mypy-1.11.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:749fd3213916f1751fff995fccf20c6195cae941dc968f3aaadf9bb4e430e5a2"}, + {file = "mypy-1.11.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b639dce63a0b19085213ec5fdd8cffd1d81988f47a2dec7100e93564f3e8fb3b"}, + {file = "mypy-1.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c956b49c5d865394d62941b109728c5c596a415e9c5b2be663dd26a1ff07bc0"}, + {file = "mypy-1.11.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45df906e8b6804ef4b666af29a87ad9f5921aad091c79cc38e12198e220beabd"}, + {file = "mypy-1.11.1-cp38-cp38-win_amd64.whl", hash = "sha256:d44be7551689d9d47b7abc27c71257adfdb53f03880841a5db15ddb22dc63edb"}, + {file = "mypy-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2684d3f693073ab89d76da8e3921883019ea8a3ec20fa5d8ecca6a2db4c54bbe"}, + {file = "mypy-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79c07eb282cb457473add5052b63925e5cc97dfab9812ee65a7c7ab5e3cb551c"}, + {file = "mypy-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11965c2f571ded6239977b14deebd3f4c3abd9a92398712d6da3a772974fad69"}, + {file = "mypy-1.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a2b43895a0f8154df6519706d9bca8280cda52d3d9d1514b2d9c3e26792a0b74"}, + {file = "mypy-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:1a81cf05975fd61aec5ae16501a091cfb9f605dc3e3c878c0da32f250b74760b"}, + {file = "mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54"}, + {file = "mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08"}, ] [package.dependencies] mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.1.0" +typing-extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] @@ -780,13 +792,13 @@ files = [ [[package]] name = "packaging" -version = "24.0" +version = "24.1" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] [[package]] @@ -813,13 +825,13 @@ files = [ [[package]] name = "pip" -version = "24.0" +version = "24.2" description = "The PyPA recommended tool for installing Python packages." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pip-24.0-py3-none-any.whl", hash = "sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc"}, - {file = "pip-24.0.tar.gz", hash = "sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2"}, + {file = "pip-24.2-py3-none-any.whl", hash = "sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2"}, + {file = "pip-24.2.tar.gz", hash = "sha256:5b5e490b5e9cb275c879595064adce9ebd31b854e3e803740b72f9ccf34a45b8"}, ] [[package]] @@ -962,161 +974,163 @@ files = [ [[package]] name = "pyyaml" -version = "6.0.1" +version = "6.0.2" description = "YAML parser and emitter for Python" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, ] [[package]] name = "regex" -version = "2024.5.15" +version = "2024.7.24" description = "Alternative regular expression module, to replace re." optional = false python-versions = ">=3.8" files = [ - {file = "regex-2024.5.15-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a81e3cfbae20378d75185171587cbf756015ccb14840702944f014e0d93ea09f"}, - {file = "regex-2024.5.15-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7b59138b219ffa8979013be7bc85bb60c6f7b7575df3d56dc1e403a438c7a3f6"}, - {file = "regex-2024.5.15-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0bd000c6e266927cb7a1bc39d55be95c4b4f65c5be53e659537537e019232b1"}, - {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5eaa7ddaf517aa095fa8da0b5015c44d03da83f5bd49c87961e3c997daed0de7"}, - {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba68168daedb2c0bab7fd7e00ced5ba90aebf91024dea3c88ad5063c2a562cca"}, - {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6e8d717bca3a6e2064fc3a08df5cbe366369f4b052dcd21b7416e6d71620dca1"}, - {file = "regex-2024.5.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1337b7dbef9b2f71121cdbf1e97e40de33ff114801263b275aafd75303bd62b5"}, - {file = "regex-2024.5.15-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9ebd0a36102fcad2f03696e8af4ae682793a5d30b46c647eaf280d6cfb32796"}, - {file = "regex-2024.5.15-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9efa1a32ad3a3ea112224897cdaeb6aa00381627f567179c0314f7b65d354c62"}, - {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:1595f2d10dff3d805e054ebdc41c124753631b6a471b976963c7b28543cf13b0"}, - {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b802512f3e1f480f41ab5f2cfc0e2f761f08a1f41092d6718868082fc0d27143"}, - {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a0981022dccabca811e8171f913de05720590c915b033b7e601f35ce4ea7019f"}, - {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:19068a6a79cf99a19ccefa44610491e9ca02c2be3305c7760d3831d38a467a6f"}, - {file = "regex-2024.5.15-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1b5269484f6126eee5e687785e83c6b60aad7663dafe842b34691157e5083e53"}, - {file = "regex-2024.5.15-cp310-cp310-win32.whl", hash = "sha256:ada150c5adfa8fbcbf321c30c751dc67d2f12f15bd183ffe4ec7cde351d945b3"}, - {file = "regex-2024.5.15-cp310-cp310-win_amd64.whl", hash = "sha256:ac394ff680fc46b97487941f5e6ae49a9f30ea41c6c6804832063f14b2a5a145"}, - {file = "regex-2024.5.15-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f5b1dff3ad008dccf18e652283f5e5339d70bf8ba7c98bf848ac33db10f7bc7a"}, - {file = "regex-2024.5.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c6a2b494a76983df8e3d3feea9b9ffdd558b247e60b92f877f93a1ff43d26656"}, - {file = "regex-2024.5.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a32b96f15c8ab2e7d27655969a23895eb799de3665fa94349f3b2fbfd547236f"}, - {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10002e86e6068d9e1c91eae8295ef690f02f913c57db120b58fdd35a6bb1af35"}, - {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ec54d5afa89c19c6dd8541a133be51ee1017a38b412b1321ccb8d6ddbeb4cf7d"}, - {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10e4ce0dca9ae7a66e6089bb29355d4432caed736acae36fef0fdd7879f0b0cb"}, - {file = "regex-2024.5.15-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e507ff1e74373c4d3038195fdd2af30d297b4f0950eeda6f515ae3d84a1770f"}, - {file = "regex-2024.5.15-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1f059a4d795e646e1c37665b9d06062c62d0e8cc3c511fe01315973a6542e40"}, - {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0721931ad5fe0dda45d07f9820b90b2148ccdd8e45bb9e9b42a146cb4f695649"}, - {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:833616ddc75ad595dee848ad984d067f2f31be645d603e4d158bba656bbf516c"}, - {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:287eb7f54fc81546346207c533ad3c2c51a8d61075127d7f6d79aaf96cdee890"}, - {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:19dfb1c504781a136a80ecd1fff9f16dddf5bb43cec6871778c8a907a085bb3d"}, - {file = "regex-2024.5.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:119af6e56dce35e8dfb5222573b50c89e5508d94d55713c75126b753f834de68"}, - {file = "regex-2024.5.15-cp311-cp311-win32.whl", hash = "sha256:1c1c174d6ec38d6c8a7504087358ce9213d4332f6293a94fbf5249992ba54efa"}, - {file = "regex-2024.5.15-cp311-cp311-win_amd64.whl", hash = "sha256:9e717956dcfd656f5055cc70996ee2cc82ac5149517fc8e1b60261b907740201"}, - {file = "regex-2024.5.15-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:632b01153e5248c134007209b5c6348a544ce96c46005d8456de1d552455b014"}, - {file = "regex-2024.5.15-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e64198f6b856d48192bf921421fdd8ad8eb35e179086e99e99f711957ffedd6e"}, - {file = "regex-2024.5.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68811ab14087b2f6e0fc0c2bae9ad689ea3584cad6917fc57be6a48bbd012c49"}, - {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8ec0c2fea1e886a19c3bee0cd19d862b3aa75dcdfb42ebe8ed30708df64687a"}, - {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d0c0c0003c10f54a591d220997dd27d953cd9ccc1a7294b40a4be5312be8797b"}, - {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2431b9e263af1953c55abbd3e2efca67ca80a3de8a0437cb58e2421f8184717a"}, - {file = "regex-2024.5.15-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a605586358893b483976cffc1723fb0f83e526e8f14c6e6614e75919d9862cf"}, - {file = "regex-2024.5.15-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:391d7f7f1e409d192dba8bcd42d3e4cf9e598f3979cdaed6ab11288da88cb9f2"}, - {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9ff11639a8d98969c863d4617595eb5425fd12f7c5ef6621a4b74b71ed8726d5"}, - {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4eee78a04e6c67e8391edd4dad3279828dd66ac4b79570ec998e2155d2e59fd5"}, - {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8fe45aa3f4aa57faabbc9cb46a93363edd6197cbc43523daea044e9ff2fea83e"}, - {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:d0a3d8d6acf0c78a1fff0e210d224b821081330b8524e3e2bc5a68ef6ab5803d"}, - {file = "regex-2024.5.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c486b4106066d502495b3025a0a7251bf37ea9540433940a23419461ab9f2a80"}, - {file = "regex-2024.5.15-cp312-cp312-win32.whl", hash = "sha256:c49e15eac7c149f3670b3e27f1f28a2c1ddeccd3a2812cba953e01be2ab9b5fe"}, - {file = "regex-2024.5.15-cp312-cp312-win_amd64.whl", hash = "sha256:673b5a6da4557b975c6c90198588181029c60793835ce02f497ea817ff647cb2"}, - {file = "regex-2024.5.15-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:87e2a9c29e672fc65523fb47a90d429b70ef72b901b4e4b1bd42387caf0d6835"}, - {file = "regex-2024.5.15-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c3bea0ba8b73b71b37ac833a7f3fd53825924165da6a924aec78c13032f20850"}, - {file = "regex-2024.5.15-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bfc4f82cabe54f1e7f206fd3d30fda143f84a63fe7d64a81558d6e5f2e5aaba9"}, - {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5bb9425fe881d578aeca0b2b4b3d314ec88738706f66f219c194d67179337cb"}, - {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64c65783e96e563103d641760664125e91bd85d8e49566ee560ded4da0d3e704"}, - {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf2430df4148b08fb4324b848672514b1385ae3807651f3567871f130a728cc3"}, - {file = "regex-2024.5.15-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5397de3219a8b08ae9540c48f602996aa6b0b65d5a61683e233af8605c42b0f2"}, - {file = "regex-2024.5.15-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:455705d34b4154a80ead722f4f185b04c4237e8e8e33f265cd0798d0e44825fa"}, - {file = "regex-2024.5.15-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:b2b6f1b3bb6f640c1a92be3bbfbcb18657b125b99ecf141fb3310b5282c7d4ed"}, - {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:3ad070b823ca5890cab606c940522d05d3d22395d432f4aaaf9d5b1653e47ced"}, - {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:5b5467acbfc153847d5adb21e21e29847bcb5870e65c94c9206d20eb4e99a384"}, - {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:e6662686aeb633ad65be2a42b4cb00178b3fbf7b91878f9446075c404ada552f"}, - {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:2b4c884767504c0e2401babe8b5b7aea9148680d2e157fa28f01529d1f7fcf67"}, - {file = "regex-2024.5.15-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:3cd7874d57f13bf70078f1ff02b8b0aa48d5b9ed25fc48547516c6aba36f5741"}, - {file = "regex-2024.5.15-cp38-cp38-win32.whl", hash = "sha256:e4682f5ba31f475d58884045c1a97a860a007d44938c4c0895f41d64481edbc9"}, - {file = "regex-2024.5.15-cp38-cp38-win_amd64.whl", hash = "sha256:d99ceffa25ac45d150e30bd9ed14ec6039f2aad0ffa6bb87a5936f5782fc1569"}, - {file = "regex-2024.5.15-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13cdaf31bed30a1e1c2453ef6015aa0983e1366fad2667657dbcac7b02f67133"}, - {file = "regex-2024.5.15-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cac27dcaa821ca271855a32188aa61d12decb6fe45ffe3e722401fe61e323cd1"}, - {file = "regex-2024.5.15-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7dbe2467273b875ea2de38ded4eba86cbcbc9a1a6d0aa11dcf7bd2e67859c435"}, - {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f18a9a3513a99c4bef0e3efd4c4a5b11228b48aa80743be822b71e132ae4f5"}, - {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d347a741ea871c2e278fde6c48f85136c96b8659b632fb57a7d1ce1872547600"}, - {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1878b8301ed011704aea4c806a3cadbd76f84dece1ec09cc9e4dc934cfa5d4da"}, - {file = "regex-2024.5.15-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4babf07ad476aaf7830d77000874d7611704a7fcf68c9c2ad151f5d94ae4bfc4"}, - {file = "regex-2024.5.15-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35cb514e137cb3488bce23352af3e12fb0dbedd1ee6e60da053c69fb1b29cc6c"}, - {file = "regex-2024.5.15-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cdd09d47c0b2efee9378679f8510ee6955d329424c659ab3c5e3a6edea696294"}, - {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:72d7a99cd6b8f958e85fc6ca5b37c4303294954eac1376535b03c2a43eb72629"}, - {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:a094801d379ab20c2135529948cb84d417a2169b9bdceda2a36f5f10977ebc16"}, - {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:c0c18345010870e58238790a6779a1219b4d97bd2e77e1140e8ee5d14df071aa"}, - {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:16093f563098448ff6b1fa68170e4acbef94e6b6a4e25e10eae8598bb1694b5d"}, - {file = "regex-2024.5.15-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:e38a7d4e8f633a33b4c7350fbd8bad3b70bf81439ac67ac38916c4a86b465456"}, - {file = "regex-2024.5.15-cp39-cp39-win32.whl", hash = "sha256:71a455a3c584a88f654b64feccc1e25876066c4f5ef26cd6dd711308aa538694"}, - {file = "regex-2024.5.15-cp39-cp39-win_amd64.whl", hash = "sha256:cab12877a9bdafde5500206d1020a584355a97884dfd388af3699e9137bf7388"}, - {file = "regex-2024.5.15.tar.gz", hash = "sha256:d3ee02d9e5f482cc8309134a91eeaacbdd2261ba111b0fef3748eeb4913e6a2c"}, + {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:228b0d3f567fafa0633aee87f08b9276c7062da9616931382993c03808bb68ce"}, + {file = "regex-2024.7.24-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3426de3b91d1bc73249042742f45c2148803c111d1175b283270177fdf669024"}, + {file = "regex-2024.7.24-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f273674b445bcb6e4409bf8d1be67bc4b58e8b46fd0d560055d515b8830063cd"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23acc72f0f4e1a9e6e9843d6328177ae3074b4182167e34119ec7233dfeccf53"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65fd3d2e228cae024c411c5ccdffae4c315271eee4a8b839291f84f796b34eca"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c414cbda77dbf13c3bc88b073a1a9f375c7b0cb5e115e15d4b73ec3a2fbc6f59"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf7a89eef64b5455835f5ed30254ec19bf41f7541cd94f266ab7cbd463f00c41"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19c65b00d42804e3fbea9708f0937d157e53429a39b7c61253ff15670ff62cb5"}, + {file = "regex-2024.7.24-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:7a5486ca56c8869070a966321d5ab416ff0f83f30e0e2da1ab48815c8d165d46"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6f51f9556785e5a203713f5efd9c085b4a45aecd2a42573e2b5041881b588d1f"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:a4997716674d36a82eab3e86f8fa77080a5d8d96a389a61ea1d0e3a94a582cf7"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c0abb5e4e8ce71a61d9446040c1e86d4e6d23f9097275c5bd49ed978755ff0fe"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:18300a1d78cf1290fa583cd8b7cde26ecb73e9f5916690cf9d42de569c89b1ce"}, + {file = "regex-2024.7.24-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:416c0e4f56308f34cdb18c3f59849479dde5b19febdcd6e6fa4d04b6c31c9faa"}, + {file = "regex-2024.7.24-cp310-cp310-win32.whl", hash = "sha256:fb168b5924bef397b5ba13aabd8cf5df7d3d93f10218d7b925e360d436863f66"}, + {file = "regex-2024.7.24-cp310-cp310-win_amd64.whl", hash = "sha256:6b9fc7e9cc983e75e2518496ba1afc524227c163e43d706688a6bb9eca41617e"}, + {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:382281306e3adaaa7b8b9ebbb3ffb43358a7bbf585fa93821300a418bb975281"}, + {file = "regex-2024.7.24-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4fdd1384619f406ad9037fe6b6eaa3de2749e2e12084abc80169e8e075377d3b"}, + {file = "regex-2024.7.24-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3d974d24edb231446f708c455fd08f94c41c1ff4f04bcf06e5f36df5ef50b95a"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a2ec4419a3fe6cf8a4795752596dfe0adb4aea40d3683a132bae9c30b81e8d73"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb563dd3aea54c797adf513eeec819c4213d7dbfc311874eb4fd28d10f2ff0f2"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:45104baae8b9f67569f0f1dca5e1f1ed77a54ae1cd8b0b07aba89272710db61e"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:994448ee01864501912abf2bad9203bffc34158e80fe8bfb5b031f4f8e16da51"}, + {file = "regex-2024.7.24-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3fac296f99283ac232d8125be932c5cd7644084a30748fda013028c815ba3364"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7e37e809b9303ec3a179085415cb5f418ecf65ec98cdfe34f6a078b46ef823ee"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:01b689e887f612610c869421241e075c02f2e3d1ae93a037cb14f88ab6a8934c"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f6442f0f0ff81775eaa5b05af8a0ffa1dda36e9cf6ec1e0d3d245e8564b684ce"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:871e3ab2838fbcb4e0865a6e01233975df3a15e6fce93b6f99d75cacbd9862d1"}, + {file = "regex-2024.7.24-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c918b7a1e26b4ab40409820ddccc5d49871a82329640f5005f73572d5eaa9b5e"}, + {file = "regex-2024.7.24-cp311-cp311-win32.whl", hash = "sha256:2dfbb8baf8ba2c2b9aa2807f44ed272f0913eeeba002478c4577b8d29cde215c"}, + {file = "regex-2024.7.24-cp311-cp311-win_amd64.whl", hash = "sha256:538d30cd96ed7d1416d3956f94d54e426a8daf7c14527f6e0d6d425fcb4cca52"}, + {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:fe4ebef608553aff8deb845c7f4f1d0740ff76fa672c011cc0bacb2a00fbde86"}, + {file = "regex-2024.7.24-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:74007a5b25b7a678459f06559504f1eec2f0f17bca218c9d56f6a0a12bfffdad"}, + {file = "regex-2024.7.24-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7df9ea48641da022c2a3c9c641650cd09f0cd15e8908bf931ad538f5ca7919c9"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6a1141a1dcc32904c47f6846b040275c6e5de0bf73f17d7a409035d55b76f289"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80c811cfcb5c331237d9bad3bea2c391114588cf4131707e84d9493064d267f9"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7214477bf9bd195894cf24005b1e7b496f46833337b5dedb7b2a6e33f66d962c"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d55588cba7553f0b6ec33130bc3e114b355570b45785cebdc9daed8c637dd440"}, + {file = "regex-2024.7.24-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:558a57cfc32adcf19d3f791f62b5ff564922942e389e3cfdb538a23d65a6b610"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a512eed9dfd4117110b1881ba9a59b31433caed0c4101b361f768e7bcbaf93c5"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:86b17ba823ea76256b1885652e3a141a99a5c4422f4a869189db328321b73799"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:5eefee9bfe23f6df09ffb6dfb23809f4d74a78acef004aa904dc7c88b9944b05"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:731fcd76bbdbf225e2eb85b7c38da9633ad3073822f5ab32379381e8c3c12e94"}, + {file = "regex-2024.7.24-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaef80eac3b4cfbdd6de53c6e108b4c534c21ae055d1dbea2de6b3b8ff3def38"}, + {file = "regex-2024.7.24-cp312-cp312-win32.whl", hash = "sha256:185e029368d6f89f36e526764cf12bf8d6f0e3a2a7737da625a76f594bdfcbfc"}, + {file = "regex-2024.7.24-cp312-cp312-win_amd64.whl", hash = "sha256:2f1baff13cc2521bea83ab2528e7a80cbe0ebb2c6f0bfad15be7da3aed443908"}, + {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:66b4c0731a5c81921e938dcf1a88e978264e26e6ac4ec96a4d21ae0354581ae0"}, + {file = "regex-2024.7.24-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:88ecc3afd7e776967fa16c80f974cb79399ee8dc6c96423321d6f7d4b881c92b"}, + {file = "regex-2024.7.24-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64bd50cf16bcc54b274e20235bf8edbb64184a30e1e53873ff8d444e7ac656b2"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb462f0e346fcf41a901a126b50f8781e9a474d3927930f3490f38a6e73b6950"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a82465ebbc9b1c5c50738536fdfa7cab639a261a99b469c9d4c7dcbb2b3f1e57"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68a8f8c046c6466ac61a36b65bb2395c74451df2ffb8458492ef49900efed293"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dac8e84fff5d27420f3c1e879ce9929108e873667ec87e0c8eeb413a5311adfe"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba2537ef2163db9e6ccdbeb6f6424282ae4dea43177402152c67ef869cf3978b"}, + {file = "regex-2024.7.24-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:43affe33137fcd679bdae93fb25924979517e011f9dea99163f80b82eadc7e53"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:c9bb87fdf2ab2370f21e4d5636e5317775e5d51ff32ebff2cf389f71b9b13750"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:945352286a541406f99b2655c973852da7911b3f4264e010218bbc1cc73168f2"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:8bc593dcce679206b60a538c302d03c29b18e3d862609317cb560e18b66d10cf"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:3f3b6ca8eae6d6c75a6cff525c8530c60e909a71a15e1b731723233331de4169"}, + {file = "regex-2024.7.24-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:c51edc3541e11fbe83f0c4d9412ef6c79f664a3745fab261457e84465ec9d5a8"}, + {file = "regex-2024.7.24-cp38-cp38-win32.whl", hash = "sha256:d0a07763776188b4db4c9c7fb1b8c494049f84659bb387b71c73bbc07f189e96"}, + {file = "regex-2024.7.24-cp38-cp38-win_amd64.whl", hash = "sha256:8fd5afd101dcf86a270d254364e0e8dddedebe6bd1ab9d5f732f274fa00499a5"}, + {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0ffe3f9d430cd37d8fa5632ff6fb36d5b24818c5c986893063b4e5bdb84cdf24"}, + {file = "regex-2024.7.24-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:25419b70ba00a16abc90ee5fce061228206173231f004437730b67ac77323f0d"}, + {file = "regex-2024.7.24-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:33e2614a7ce627f0cdf2ad104797d1f68342d967de3695678c0cb84f530709f8"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d33a0021893ede5969876052796165bab6006559ab845fd7b515a30abdd990dc"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04ce29e2c5fedf296b1a1b0acc1724ba93a36fb14031f3abfb7abda2806c1535"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b16582783f44fbca6fcf46f61347340c787d7530d88b4d590a397a47583f31dd"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:836d3cc225b3e8a943d0b02633fb2f28a66e281290302a79df0e1eaa984ff7c1"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:438d9f0f4bc64e8dea78274caa5af971ceff0f8771e1a2333620969936ba10be"}, + {file = "regex-2024.7.24-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:973335b1624859cb0e52f96062a28aa18f3a5fc77a96e4a3d6d76e29811a0e6e"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c5e69fd3eb0b409432b537fe3c6f44ac089c458ab6b78dcec14478422879ec5f"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:fbf8c2f00904eaf63ff37718eb13acf8e178cb940520e47b2f05027f5bb34ce3"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ae2757ace61bc4061b69af19e4689fa4416e1a04840f33b441034202b5cd02d4"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:44fc61b99035fd9b3b9453f1713234e5a7c92a04f3577252b45feefe1b327759"}, + {file = "regex-2024.7.24-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:84c312cdf839e8b579f504afcd7b65f35d60b6285d892b19adea16355e8343c9"}, + {file = "regex-2024.7.24-cp39-cp39-win32.whl", hash = "sha256:ca5b2028c2f7af4e13fb9fc29b28d0ce767c38c7facdf64f6c2cd040413055f1"}, + {file = "regex-2024.7.24-cp39-cp39-win_amd64.whl", hash = "sha256:7c479f5ae937ec9985ecaf42e2e10631551d909f203e31308c12d703922742f9"}, + {file = "regex-2024.7.24.tar.gz", hash = "sha256:9cfd009eed1a46b27c14039ad5bbc5e71b6367c5b2e6d5f5da0ea91600817506"}, ] [[package]] name = "requests" -version = "2.32.2" +version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" files = [ - {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, - {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, ] [package.dependencies] @@ -1153,13 +1167,13 @@ files = [ [[package]] name = "transitions" -version = "0.9.1" +version = "0.9.2" description = "A lightweight, object-oriented Python state machine implementation with many extensions." optional = false python-versions = "*" files = [ - {file = "transitions-0.9.1-py2.py3-none-any.whl", hash = "sha256:e76ad7bf44d46bd03bc50f269cebfa5affef28534c92b87b43e60b03d8925864"}, - {file = "transitions-0.9.1.tar.gz", hash = "sha256:3542c37108e93e2ae5f215208ec5732c94a772937854a102cd7345b967fee61b"}, + {file = "transitions-0.9.2-py2.py3-none-any.whl", hash = "sha256:f7b40c9b4a93869f36c4d1c33809aeb18cdeeb065fd1adba018ee39c3db216f3"}, + {file = "transitions-0.9.2.tar.gz", hash = "sha256:2f8490dbdbd419366cef1516032ab06d07ccb5839ef54905e842a472692d4204"}, ] [package.dependencies] @@ -1171,24 +1185,24 @@ test = ["pytest"] [[package]] name = "typing-extensions" -version = "4.12.0" +version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, - {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] [[package]] name = "urllib3" -version = "2.2.1" +version = "2.2.2" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.8" files = [ - {file = "urllib3-2.2.1-py3-none-any.whl", hash = "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d"}, - {file = "urllib3-2.2.1.tar.gz", hash = "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"}, + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, ] [package.extras] @@ -1319,4 +1333,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10.1,<3.11" -content-hash = "723fb14f980b4ff8e8b32fa59a42f47f4c56d33d9e856412c0a67fa044812a37" +content-hash = "7872d626bf1bf9f8a086b4e81ec53003a7ac9af3c999fa1d562df50439d65746" diff --git a/goth_tests/pyproject.toml b/goth_tests/pyproject.toml index 94c4700976..17f7f08931 100644 --- a/goth_tests/pyproject.toml +++ b/goth_tests/pyproject.toml @@ -27,7 +27,7 @@ pytest-asyncio = "0.21" pytest-split = "^0.8.1" #goth = "0.18.1" # to use development goth version uncomment below -goth = { git = "https://github.com/golemfactory/goth.git", rev = "cda4aa83a72622cd97de45d8ce9066750c3b522e" } +goth = { git = "https://github.com/golemfactory/goth.git", rev = "3891382c4a18e0e82a735dfe2280a6287e8c268b" } [tool.poetry.dev-dependencies] black = "21.7b0" From d28beba718bcac47f016339545d57d7c6a931ecd Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 7 Aug 2024 14:31:28 +0200 Subject: [PATCH 079/125] Ability to bind identity modules on prefixed gsb path --- core/identity/src/service.rs | 18 +++++- core/identity/src/service/appkey.rs | 16 +++--- core/identity/src/service/identity.rs | 42 +++++++------- core/model/src/appkey.rs | 10 ++++ core/model/src/bus.rs | 55 +++++++++++++++++++ core/model/src/identity.rs | 11 ++++ core/model/src/lib.rs | 1 + core/payment/tests/test_allocation.rs | 4 +- core/payment/tests/test_debit_notes.rs | 2 +- core/payment/tests/test_invoices.rs | 2 +- core/payment/tests/test_payment_sync.rs | 30 +++++----- .../tests/tutorial_how_to_use_module_tests.rs | 2 +- .../framework-mocks/src/identity.rs | 7 ++- .../framework-mocks/src/node.rs | 20 +++++-- .../framework-mocks/src/payment.rs | 2 +- .../src/payment/fake_payment.rs | 8 ++- 16 files changed, 169 insertions(+), 61 deletions(-) create mode 100644 core/model/src/bus.rs diff --git a/core/identity/src/service.rs b/core/identity/src/service.rs index 15941cd3d3..4b51fc9e04 100644 --- a/core/identity/src/service.rs +++ b/core/identity/src/service.rs @@ -2,6 +2,8 @@ use futures::lock::Mutex; use std::sync::Arc; +use ya_core_model as model; +use ya_core_model::bus::GsbBindPoints; use ya_persistence::executor::DbExecutor; use ya_service_api_interfaces::{Provider, Service}; @@ -18,16 +20,26 @@ impl Service for Identity { impl Identity { pub async fn gsb>(context: &Context) -> anyhow::Result<()> { + Self::gsb_prefixed(context, None).await + } + + pub async fn gsb_prefixed>( + context: &Context, + base: Option, + ) -> anyhow::Result<()> { let db = context.component(); + let gsb = base.unwrap_or(GsbBindPoints::default()); + let gsb_ident = Arc::new(gsb.clone().service(model::identity::BUS_SERVICE_NAME)); let service = Arc::new(Mutex::new( identity::IdentityService::from_db(db.clone()).await?, )); - identity::IdentityService::bind_service(service); + identity::IdentityService::bind_service(service, gsb_ident.clone()); - identity::wait_for_default_account_unlock().await?; + identity::wait_for_default_account_unlock(gsb_ident.clone()).await?; - appkey::activate(&db).await?; + let gsb_appkey = Arc::new(gsb.service(model::appkey::BUS_SERVICE_NAME)); + appkey::activate(&db, gsb_appkey.clone()).await?; Ok(()) } } diff --git a/core/identity/src/service/appkey.rs b/core/identity/src/service/appkey.rs index 8ec44a6d87..8d98440054 100644 --- a/core/identity/src/service/appkey.rs +++ b/core/identity/src/service/appkey.rs @@ -1,6 +1,7 @@ use std::cell::{Ref, RefCell}; use std::collections::HashMap; use std::rc::Rc; +use std::sync::Arc; use chrono::{NaiveDateTime, Utc}; use futures::prelude::*; @@ -9,6 +10,7 @@ use ya_service_bus::{typed as bus, RpcEndpoint}; use ya_core_model::appkey as model; use ya_core_model::appkey::event::AppKeyEvent; +use ya_core_model::bus::GsbBindPoints; use ya_core_model::identity as idm; use ya_persistence::executor::DbExecutor; @@ -72,7 +74,7 @@ pub async fn preconfigured_to_appkey_model( }) } -pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { +pub async fn activate(db: &DbExecutor, gsb: Arc) -> anyhow::Result<()> { let (tx, rx) = futures::channel::mpsc::unbounded(); let subscription = Rc::new(RefCell::new(Subscription::default())); @@ -85,7 +87,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { }); } - let _ = bus::bind(model::BUS_ID, move |s: model::Subscribe| { + let _ = bus::bind(gsb.local_addr(), move |s: model::Subscribe| { let id = subscription.borrow_mut().subscribe(s.endpoint); future::ok(id) }); @@ -99,7 +101,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { // Create a new application key entry let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); - let _ = bus::bind(model::BUS_ID, move |create: model::Create| { + let _ = bus::bind(gsb.local_addr(), move |create: model::Create| { let key = Uuid::new_v4().to_simple().to_string(); let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); @@ -157,7 +159,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { { let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); - let _ = bus::bind(model::BUS_ID, move |get: model::Get| { + let _ = bus::bind(gsb.local_addr(), move |get: model::Get| { let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); async move { @@ -185,7 +187,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { { let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); - let _ = bus::bind(model::BUS_ID, move |get: model::GetByName| { + let _ = bus::bind(gsb.local_addr(), move |get: model::GetByName| { let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); async move { @@ -214,7 +216,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { { let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); - let _ = bus::bind(model::BUS_ID, move |list: model::List| { + let _ = bus::bind(gsb.local_addr(), move |list: model::List| { let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); async move { @@ -249,7 +251,7 @@ pub async fn activate(db: &DbExecutor) -> anyhow::Result<()> { { let create_tx = tx; let db = db.clone(); - let _ = bus::bind(model::BUS_ID, move |rm: model::Remove| { + let _ = bus::bind(gsb.local_addr(), move |rm: model::Remove| { let db = db.clone(); let preconfigured_appkey = preconfigured_appkey.clone(); let mut create_tx = create_tx.clone(); diff --git a/core/identity/src/service/identity.rs b/core/identity/src/service/identity.rs index f88bbd8e37..67c2f26080 100644 --- a/core/identity/src/service/identity.rs +++ b/core/identity/src/service/identity.rs @@ -11,6 +11,7 @@ use futures::lock::Mutex; use futures::prelude::*; use ya_client_model::NodeId; +use ya_core_model::bus::GsbBindPoints; use ya_service_bus::{typed as bus, RpcEndpoint, RpcMessage}; use ya_core_model::identity as model; @@ -465,14 +466,14 @@ impl IdentityService { key.to_key_file().map_err(model::Error::new_err_msg) } - pub fn bind_service(me: Arc>) { + pub fn bind_service(me: Arc>, gsb: Arc) { let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |_list: model::List| { + let _ = bus::bind(gsb.local_addr(), move |_list: model::List| { let this = this.clone(); async move { this.lock().await.list_ids() } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |get: model::Get| { + let _ = bus::bind(gsb.local_addr(), move |get: model::Get| { let this = this.clone(); async move { match get { @@ -484,7 +485,7 @@ impl IdentityService { } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |create: model::CreateGenerated| { + let _ = bus::bind(gsb.local_addr(), move |create: model::CreateGenerated| { let this = this.clone(); async move { @@ -512,12 +513,12 @@ impl IdentityService { }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |update: model::Update| { + let _ = bus::bind(gsb.local_addr(), move |update: model::Update| { let this = this.clone(); async move { this.lock().await.update_identity(update).await } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |lock: model::Lock| { + let _ = bus::bind(gsb.local_addr(), move |lock: model::Lock| { let this = this.clone(); async move { let mut lock_sender = this.lock().await.sender().clone(); @@ -540,7 +541,7 @@ impl IdentityService { } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |unlock: model::Unlock| { + let _ = bus::bind(gsb.local_addr(), move |unlock: model::Unlock| { let this = this.clone(); async move { let mut unlock_sender = this.lock().await.sender().clone(); @@ -560,22 +561,22 @@ impl IdentityService { } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |sign: model::Sign| { + let _ = bus::bind(gsb.local_addr(), move |sign: model::Sign| { let this = this.clone(); async move { this.lock().await.sign(sign.node_id, sign.payload).await } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |subscribe: model::Subscribe| { + let _ = bus::bind(gsb.local_addr(), move |subscribe: model::Subscribe| { let this = this.clone(); async move { this.lock().await.subscribe(subscribe).await } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |unsubscribe: model::Unsubscribe| { + let _ = bus::bind(gsb.local_addr(), move |unsubscribe: model::Unsubscribe| { let this = this.clone(); async move { this.lock().await.unsubscribe(unsubscribe).await } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |node_id: model::GetPubKey| { + let _ = bus::bind(gsb.local_addr(), move |node_id: model::GetPubKey| { let this = this.clone(); async move { this.lock() @@ -586,12 +587,12 @@ impl IdentityService { } }); let this = me.clone(); - let _ = bus::bind(model::BUS_ID, move |node_id: model::GetKeyFile| { + let _ = bus::bind(gsb.local_addr(), move |node_id: model::GetKeyFile| { let this = this.clone(); async move { this.lock().await.get_key_file(node_id).await } }); let this = me; - let _ = bus::bind(model::BUS_ID, move |drop_cmd: model::DropId| { + let _ = bus::bind(gsb.local_addr(), move |drop_cmd: model::DropId| { let this = this.clone(); async move { log::trace!("Dropping identity: {:?}", drop_cmd); @@ -605,8 +606,8 @@ impl IdentityService { } } -pub async fn wait_for_default_account_unlock() -> anyhow::Result<()> { - let identity_key = get_default_identity_key().await?; +pub async fn wait_for_default_account_unlock(gsb: Arc) -> anyhow::Result<()> { + let identity_key = get_default_identity_key(gsb.clone()).await?; if identity_key.is_locked { let locked_identity = identity_key.node_id; @@ -634,7 +635,7 @@ pub async fn wait_for_default_account_unlock() -> anyhow::Result<()> { "Daemon cannot start because default account is locked. Unlock it by running 'yagna id unlock'" )); - wait_for_unlock(rx).await?; + wait_for_unlock(gsb.clone(), rx).await?; unsubscribe(endpoint.clone()).await?; unbind(endpoint).await?; @@ -644,10 +645,11 @@ pub async fn wait_for_default_account_unlock() -> anyhow::Result<()> { } async fn wait_for_unlock( + gsb: Arc, mut rx: futures::channel::mpsc::UnboundedReceiver<()>, ) -> anyhow::Result<()> { - // Check lock second time because user could unlocked database before subscription - if get_default_identity_key().await?.is_locked { + // Check lock second time because user could unlock database before subscription + if get_default_identity_key(gsb).await?.is_locked { tokio::select! { _ = rx.next() => { log::info!("Default account unlocked"); @@ -683,8 +685,8 @@ async fn unbind(endpoint: String) -> anyhow::Result<()> { Ok(()) } -async fn get_default_identity_key() -> anyhow::Result { - bus::service(model::BUS_ID) +async fn get_default_identity_key(gsb: Arc) -> anyhow::Result { + gsb.local() .send(model::Get::ByDefault {}) .await?? .ok_or_else(|| anyhow::anyhow!("No default Identity found")) diff --git a/core/model/src/appkey.rs b/core/model/src/appkey.rs index f02e0da497..37e9102df3 100644 --- a/core/model/src/appkey.rs +++ b/core/model/src/appkey.rs @@ -1,9 +1,12 @@ use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; use thiserror::Error; + +use crate::bus::GsbBindPoints; use ya_client_model::NodeId; use ya_service_bus::RpcMessage; +pub const BUS_SERVICE_NAME: &str = "appkey"; pub const BUS_ID: &str = "/local/appkey"; pub const DEFAULT_ROLE: &str = "manager"; @@ -11,6 +14,13 @@ pub const AUTOCONFIGURED_KEY_NAME: &str = "autoconfigured"; const DEFAULT_PAGE_SIZE: u32 = 20; +pub fn bus_bindpoints(base: Option) -> GsbBindPoints { + match base { + Some(base) => base.service(BUS_SERVICE_NAME), + None => GsbBindPoints::default().service(BUS_SERVICE_NAME), + } +} + #[derive(Clone, Error, Debug, Serialize, Deserialize)] #[error("appkey error [{code}]: {message}")] pub struct Error { diff --git a/core/model/src/bus.rs b/core/model/src/bus.rs new file mode 100644 index 0000000000..5f03e0f6f6 --- /dev/null +++ b/core/model/src/bus.rs @@ -0,0 +1,55 @@ +use ya_service_bus::typed as bus; + +pub const BUS_LOCAL: &str = "/local"; +pub const BUS_PUBLIC: &str = "/public"; + +#[derive(Clone)] +pub struct GsbBindPoints { + public: bus::Endpoint, + local: bus::Endpoint, +} + +impl GsbBindPoints { + pub fn new(public: &str, local: &str) -> Self { + Self { + public: bus::service(public), + local: bus::service(local), + } + } + + pub fn service(&self, name: &str) -> GsbBindPoints { + Self { + public: bus::service(format!("{}/{name}", self.public.addr())), + local: bus::service(format!("{}/{name}", self.local.addr())), + } + } + + pub fn prefix(&self, prefix: &str) -> GsbBindPoints { + Self { + public: bus::service(format!("{prefix}{}", self.public.addr())), + local: bus::service(format!("{prefix}{}", self.local.addr())), + } + } + + pub fn public(&self) -> bus::Endpoint { + self.public.clone() + } + + pub fn local(&self) -> bus::Endpoint { + self.local.clone() + } + + pub fn local_addr(&self) -> &str { + self.local.addr() + } + + pub fn public_addr(&self) -> &str { + self.public.addr() + } +} + +impl Default for GsbBindPoints { + fn default() -> Self { + Self::new(BUS_PUBLIC, BUS_LOCAL) + } +} diff --git a/core/model/src/identity.rs b/core/model/src/identity.rs index f981d32721..d35fb841ce 100644 --- a/core/model/src/identity.rs +++ b/core/model/src/identity.rs @@ -1,10 +1,21 @@ use serde::{Deserialize, Serialize}; use thiserror::Error; + use ya_client_model::NodeId; use ya_service_bus::RpcMessage; +use crate::bus::GsbBindPoints; + +pub const BUS_SERVICE_NAME: &str = "identity"; pub const BUS_ID: &str = "/local/identity"; +pub fn bus_bindpoints(base: Option) -> GsbBindPoints { + match base { + Some(base) => base.service(BUS_SERVICE_NAME), + None => GsbBindPoints::default().service(BUS_SERVICE_NAME), + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Ack {} diff --git a/core/model/src/lib.rs b/core/model/src/lib.rs index 48d782597d..6ab4826f41 100644 --- a/core/model/src/lib.rs +++ b/core/model/src/lib.rs @@ -28,6 +28,7 @@ pub mod gftp; #[cfg(feature = "sgx")] pub mod sgx; +pub mod bus; #[cfg(feature = "version")] pub mod version; diff --git a/core/payment/tests/test_allocation.rs b/core/payment/tests/test_allocation.rs index ac3f59c83f..9cf842a6ba 100644 --- a/core/payment/tests/test_allocation.rs +++ b/core/payment/tests/test_allocation.rs @@ -27,7 +27,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu .with_identity() .with_payment() .with_fake_market(); - node.bind_gsb().await?; + node.bind_gsb(false).await?; node.start_server(ctx).await?; let requestor_appkey = node @@ -175,7 +175,7 @@ async fn test_validate_allocation(ctx: &mut DroppableTestContext) -> anyhow::Res .with_identity() .with_payment() .with_fake_market(); - node.bind_gsb().await?; + node.bind_gsb(false).await?; node.start_server(ctx).await?; let appkey_req = node diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs index 79e22db446..ed45c06ba2 100644 --- a/core/payment/tests/test_debit_notes.rs +++ b/core/payment/tests/test_debit_notes.rs @@ -29,7 +29,7 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< .with_payment() .with_fake_market() .with_fake_activity(); - node.bind_gsb().await?; + node.bind_gsb(false).await?; node.start_server(ctx).await?; let appkey_prov = node.get_identity()?.create_identity_key("provider").await?; diff --git a/core/payment/tests/test_invoices.rs b/core/payment/tests/test_invoices.rs index b530b3a979..7bf79839f8 100644 --- a/core/payment/tests/test_invoices.rs +++ b/core/payment/tests/test_invoices.rs @@ -29,7 +29,7 @@ async fn test_invoice_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .with_identity() .with_payment() .with_fake_market(); - node.bind_gsb().await?; + node.bind_gsb(false).await?; node.start_server(ctx).await?; let appkey_prov = node.get_identity()?.create_identity_key("provider").await?; diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index d681330e91..41c7557eb4 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -12,7 +12,7 @@ use ya_framework_mocks::payment::Driver; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(false); + enable_logs(true); let dir = temp_dir!("test_payment_sync")?; let dir = dir.path(); @@ -23,31 +23,33 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .with_identity() .with_payment() .with_fake_market(); - node1.bind_gsb().await?; + node1.bind_gsb(false).await?; node1.start_server(ctx).await?; - let node2 = MockNode::new(net, "node-2", dir).with_fake_payment(); - - let appkey_prov = node1 - .get_identity()? - .create_identity_key("provider") - .await?; let appkey_req = node1 .get_identity()? .create_from_private_key(&resource!("ci-requestor-1.key.priv")) .await?; - let mut agreement = - FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); - node1.get_market()?.add_agreement(agreement.clone()).await; - node1 .get_payment()? .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) .await?; - let requestor = node1.rest_payments(&appkey_req.key)?; - let provider = node1.rest_payments(&appkey_prov.key)?; + let node2 = MockNode::new(net, "node-2", dir) + .with_identity() + .with_fake_payment(); + node2.bind_gsb(true).await?; + + let appkey_prov = node2 + .get_identity()? + .create_identity_key("provider") + .await?; + + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + let _requestor = node1.rest_payments(&appkey_req.key)?; Ok(()) } diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index b3b1ee279e..7c0d96b1fd 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -51,7 +51,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Bind GSB and start server like yagna node would do in full setup. // Those functions will bind only modules chosen for MockNode. - node.bind_gsb().await?; + node.bind_gsb(false).await?; node.start_server(ctx).await?; // Creating identities is essential to use REST API and create Agreements and Payments. diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index e5d1426c5a..7e2567f2c4 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -6,6 +6,7 @@ use std::path::Path; use ya_client_model::NodeId; use ya_core_model::appkey::AppKey; +use ya_core_model::bus::GsbBindPoints; use ya_core_model::identity::IdentityInfo; use ya_identity::cli::{AppKeyCommand, IdentityCommand}; use ya_identity::service::Identity; @@ -38,9 +39,9 @@ impl RealIdentity { Ok(db) } - pub async fn bind_gsb(&self) -> anyhow::Result<()> { - log::info!("MockIdentity ({}) - binding GSB", self.name); - Identity::gsb(&self.db).await?; + pub async fn bind_gsb(&self, gsb: Option) -> anyhow::Result<()> { + log::info!("RealIdentity ({}) - binding GSB", self.name); + Identity::gsb_prefixed(&self.db, gsb).await?; Ok(()) } diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index a6b3d88e39..6b872a5ad4 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -8,6 +8,7 @@ use url::Url; use ya_client::payment::PaymentApi; use ya_client::web::WebClient; +use ya_core_model::bus::GsbBindPoints; use ya_framework_basic::async_drop::DroppableTestContext; use ya_service_api_web::middleware::auth; use ya_service_api_web::middleware::cors::{AppKeyCors, CorsConfig}; @@ -121,18 +122,27 @@ impl MockNode { /// Binds GSB router and all initialized modules to GSB. /// If you want to bind only chosen modules, you should bind them manually. - pub async fn bind_gsb(&self) -> anyhow::Result<()> { + /// + /// `use_prefix` parameter decides if GSB will be bound without prefix like normally + /// yagna does, or if GSB paths will be prefixed by Node name. + /// The second options gives you possibility to run multiple nodes with GSB bound. + pub async fn bind_gsb(&self, use_prefix: bool) -> anyhow::Result<()> { + let gsb = match use_prefix { + true => Some(GsbBindPoints::default().prefix(&format!("/{}", self.name))), + false => None, + }; + self.bind_gsb_router().await?; if let Some(identity) = &self.identity { - identity.bind_gsb().await?; + identity.bind_gsb(gsb.clone()).await?; } - if let Some(payment) = &self.payment { - payment.bind_gsb().await?; + if let Some(payment) = &self.fake_payment { + payment.bind_gsb(gsb).await?; } - if let Some(payment) = &self.fake_payment { + if let Some(payment) = &self.payment { payment.bind_gsb().await?; } diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index bda34650c1..f5883139a3 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -71,7 +71,7 @@ impl RealPayment { } pub async fn bind_gsb(&self) -> anyhow::Result<()> { - log::info!("MockPayment ({}) - binding GSB", self.name); + log::info!("RealPayment ({}) - binding GSB", self.name); ya_payment::service::bind_service( &self.db, diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs index ee433868f1..9c0752f647 100644 --- a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -4,6 +4,7 @@ use anyhow::anyhow; use std::path::{Path, PathBuf}; use std::sync::Arc; +use ya_core_model::bus::GsbBindPoints; use ya_core_model::payment::public::{ AcceptDebitNote, AcceptInvoice, AcceptRejectError, Ack, CancelDebitNote, CancelError, CancelInvoice, PaymentSync, PaymentSyncError, PaymentSyncRequest, PaymentSyncWithBytes, @@ -25,7 +26,7 @@ pub struct FakePayment { impl FakePayment { pub fn new(name: &str, testdir: &Path) -> Self { - let db = Self::create_db(testdir, "payment").unwrap(); + let db = Self::create_db(testdir, name).unwrap(); FakePayment { name: name.to_string(), testdir: testdir.to_path_buf(), @@ -40,10 +41,11 @@ impl FakePayment { Ok(db) } - pub async fn bind_gsb(&self) -> anyhow::Result<()> { + pub async fn bind_gsb(&self, gsb: Option) -> anyhow::Result<()> { log::info!("FakePayment ({}) - binding GSB", self.name); - ServiceBinder::new(BUS_ID, &self.db, ()) + let gsb = gsb.unwrap_or_default().service("payment"); + ServiceBinder::new(gsb.public_addr(), &self.db, ()) .bind(send_debit_note) .bind(accept_debit_note) .bind(reject_debit_note) From f7c049dd6efc7509ad88bdbf83093328f6b7b5af Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 7 Aug 2024 19:36:22 +0200 Subject: [PATCH 080/125] Indentity module is able to be instantiated with base GSB address --- .github/workflows/system-test.yml | 2 +- Cargo.lock | 7 +- Cargo.toml | 2 +- core/identity/Cargo.toml | 8 ++ core/identity/src/cli.rs | 2 +- core/identity/src/cli/appkey.rs | 30 ++++--- core/identity/src/cli/identity.rs | 26 +++--- core/identity/src/cli/identity/drop_id.rs | 14 +++- core/identity/src/cli/identity/list.rs | 6 +- core/identity/src/service/identity.rs | 18 ++--- core/identity/tests/test_identity_unlock.rs | 60 ++++++++++++++ core/model/src/bus.rs | 14 ++++ core/model/src/payment.rs | 1 + core/payment/Cargo.toml | 1 - core/payment/tests/test_allocation.rs | 4 +- core/payment/tests/test_debit_notes.rs | 2 +- core/payment/tests/test_invoices.rs | 2 +- core/payment/tests/test_payment_sync.rs | 7 +- .../tests/tutorial_how_to_use_module_tests.rs | 2 +- core/serv-api/Cargo.toml | 1 + core/serv-api/src/lib.rs | 9 +++ .../test-framework/framework-mocks/Cargo.toml | 1 + .../framework-mocks/src/identity.rs | 79 ++++++++++++++++++- .../framework-mocks/src/node.rs | 45 +++++++---- .../src/payment/fake_payment.rs | 14 +++- 25 files changed, 286 insertions(+), 71 deletions(-) create mode 100644 core/identity/tests/test_identity_unlock.rs diff --git a/.github/workflows/system-test.yml b/.github/workflows/system-test.yml index 6f9a1523ff..3c6b70f467 100644 --- a/.github/workflows/system-test.yml +++ b/.github/workflows/system-test.yml @@ -71,4 +71,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --test '*' -p yagna -p ya-exe-unit -p ya-transfer -p ya-payment --features framework-test + args: --test '*' -p yagna -p ya-exe-unit -p ya-transfer -p ya-payment -p ya-identity --features framework-test diff --git a/Cargo.lock b/Cargo.lock index 598cfc92ba..69fe9a0d9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9115,6 +9115,7 @@ dependencies = [ "ya-service-api", "ya-service-api-web", "ya-service-bus", + "ya-std-utils", "ya-utils-path", ] @@ -9217,13 +9218,17 @@ dependencies = [ "rpassword", "rustc-hex", "serde_json", + "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", "sha2 0.9.9", "structopt", + "test-context", "thiserror", "tokio", "uuid 0.8.2", "ya-client-model", "ya-core-model", + "ya-framework-basic", + "ya-framework-mocks", "ya-persistence", "ya-sb-router", "ya-service-api", @@ -9473,7 +9478,6 @@ dependencies = [ "metrics 0.12.1", "num-bigint 0.3.3", "open", - "portpicker", "problem_details", "r2d2", "rand 0.8.5", @@ -9867,6 +9871,7 @@ dependencies = [ "lazy_static", "serde", "url", + "ya-core-model", "ya-utils-cli", ] diff --git a/Cargo.toml b/Cargo.toml index b5b3c6f713..d0a44fd20f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ static-openssl = ["openssl/vendored", "openssl-probe"] dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] tos = [] -framework-test = ['ya-exe-unit/framework-test', 'ya-payment/framework-test'] +framework-test = ['ya-exe-unit/framework-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index d695413827..5d927802f8 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -5,6 +5,10 @@ description = "Yagna identity management" authors = ["Golem Factory "] edition = "2018" +[features] +framework-test = [] + + [dependencies] ya-client-model = { version = "0.6", features = ["with-diesel"] } ya-core-model = { version = "^0.9", features = ["identity", "appkey"] } @@ -39,6 +43,8 @@ yansi = "0.5.0" [dev-dependencies] ya-service-api-derive = "0.2" ya-sb-router = { workspace = true } +ya-framework-basic = { version = "0.1" } +ya-framework-mocks = { version = "0.1" } actix-rt = "2.7" actix-service = "2" @@ -47,6 +53,8 @@ awc = "3" base64 = "0.12" dotenv = "0.15" env_logger = "0.7.1" +serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"] } +test-context = "0.1.4" [lints] workspace = true \ No newline at end of file diff --git a/core/identity/src/cli.rs b/core/identity/src/cli.rs index 40ea03daee..b38004f732 100644 --- a/core/identity/src/cli.rs +++ b/core/identity/src/cli.rs @@ -2,7 +2,7 @@ mod appkey; mod identity; pub use appkey::AppKeyCommand; -pub use identity::IdentityCommand; +pub use identity::{IdentityCommand, NodeOrAlias}; use structopt::StructOpt; use ya_service_api::{CliCtx, CommandOutput}; diff --git a/core/identity/src/cli/appkey.rs b/core/identity/src/cli/appkey.rs index 4f3653158b..8d3fdf79ba 100644 --- a/core/identity/src/cli/appkey.rs +++ b/core/identity/src/cli/appkey.rs @@ -2,10 +2,11 @@ use anyhow::Result; use structopt::*; use ya_core_model::appkey as model; +use ya_core_model::bus::GsbBindPoints; use ya_core_model::identity as idm; use ya_core_model::identity::IdentityInfo; use ya_service_api::{CliCtx, CommandOutput, ResponseTable}; -use ya_service_bus::{typed as bus, RpcEndpoint}; +use ya_service_bus::RpcEndpoint; #[derive(StructOpt, Debug)] #[structopt(setting = clap::AppSettings::DeriveDisplayOrder)] @@ -41,8 +42,8 @@ pub enum AppKeyCommand { } impl AppKeyCommand { - async fn get_identity(get_by: idm::Get) -> anyhow::Result { - bus::service(idm::BUS_ID) + async fn get_identity(gsb: GsbBindPoints, get_by: idm::Get) -> anyhow::Result { + gsb.local() .send(get_by) .await .map_err(anyhow::Error::msg)? @@ -50,7 +51,10 @@ impl AppKeyCommand { .ok_or_else(|| anyhow::Error::msg("Identity not found")) } - pub async fn run_command(&self, _ctx: &CliCtx) -> Result { + pub async fn run_command(&self, ctx: &CliCtx) -> Result { + let gsb = ctx.gsb.service(model::BUS_SERVICE_NAME); + let ident_gsb = ctx.gsb.service(idm::BUS_SERVICE_NAME); + match &self { AppKeyCommand::Create { name, @@ -63,12 +67,16 @@ impl AppKeyCommand { if id.starts_with("0x") { id.parse()? } else { - Self::get_identity(idm::Get::ByAlias(id.into())) + Self::get_identity(ident_gsb.clone(), idm::Get::ByAlias(id.into())) .await? .node_id } } - None => Self::get_identity(idm::Get::ByDefault).await?.node_id, + None => { + Self::get_identity(ident_gsb.clone(), idm::Get::ByDefault) + .await? + .node_id + } }; let create = model::Create { name: name.clone(), @@ -76,7 +84,7 @@ impl AppKeyCommand { identity, allow_origins: allow_origin.clone(), }; - let key = bus::service(model::BUS_ID).send(create).await??; + let key = gsb.local().send(create).await??; Ok(CommandOutput::Object(serde_json::to_value(key)?)) } AppKeyCommand::Drop { name, id } => { @@ -84,7 +92,7 @@ impl AppKeyCommand { name: name.clone(), identity: id.clone(), }; - bus::service(model::BUS_ID) + gsb.local() .send(remove) .await .map_err(anyhow::Error::msg)? @@ -92,7 +100,8 @@ impl AppKeyCommand { Ok(CommandOutput::NoOutput) } AppKeyCommand::Show { name } => { - let appkey = bus::service(model::BUS_ID) + let appkey = gsb + .local() .send(model::GetByName { name: name.clone() }) .await .map_err(anyhow::Error::msg)? @@ -105,7 +114,8 @@ impl AppKeyCommand { page: *page, per_page: *per_page, }; - let result: (Vec, u32) = bus::service(model::BUS_ID) + let result: (Vec, u32) = gsb + .local() .send(list) .await .map_err(anyhow::Error::msg)? diff --git a/core/identity/src/cli/identity.rs b/core/identity/src/cli/identity.rs index 1a0ae872ff..320875693a 100644 --- a/core/identity/src/cli/identity.rs +++ b/core/identity/src/cli/identity.rs @@ -224,13 +224,14 @@ fn to_private_key(key_file_json: &str) -> Result<[u8; 32], anyhow::Error> { } impl IdentityCommand { - pub async fn run_command(&self, _ctx: &CliCtx) -> Result { + pub async fn run_command(&self, ctx: &CliCtx) -> Result { + let gsb = ctx.gsb.service(identity::BUS_SERVICE_NAME); match self { - IdentityCommand::List { .. } => list::list().await, + IdentityCommand::List { .. } => list::list(&gsb).await, IdentityCommand::Show { node_or_alias } => { let command: identity::Get = node_or_alias.clone().unwrap_or_default().into(); CommandOutput::object( - bus::service(identity::BUS_ID) + gsb.local() .send(command) .await .map_err(anyhow::Error::msg)?, @@ -239,7 +240,7 @@ impl IdentityCommand { IdentityCommand::PubKey { node_or_alias } => { let node_id = node_or_alias.clone().unwrap_or_default().resolve().await?; CommandOutput::object( - bus::service(identity::BUS_ID) + gsb.local() .send(identity::GetPubKey(node_id)) .await .map_err(anyhow::Error::msg)? @@ -282,7 +283,7 @@ impl IdentityCommand { let payload = sha256.finalize().to_vec(); CommandOutput::object( - bus::service(identity::BUS_ID) + gsb.local() .send(identity::Sign { node_id, payload }) .await .map_err(anyhow::Error::msg)? @@ -298,7 +299,8 @@ impl IdentityCommand { set_default, } => { let node_id = alias_or_id.resolve().await?; - let id = bus::service(identity::BUS_ID) + let id = gsb + .local() .send( identity::Update::with_id(node_id) .with_alias(alias.clone()) @@ -358,7 +360,8 @@ impl IdentityCommand { crate::id_key::generate_new_keyfile(password, from_private_key_slice)? }; - let id = bus::service(identity::BUS_ID) + let id = gsb + .local() .send(identity::CreateGenerated { alias: alias.clone(), from_keystore: Some(key_file), @@ -384,7 +387,7 @@ impl IdentityCommand { None }; CommandOutput::object( - bus::service(identity::BUS_ID) + gsb.local() .send(identity::Lock::with_id(node_id).with_set_password(password)) .await .map_err(anyhow::Error::msg)?, @@ -401,7 +404,7 @@ impl IdentityCommand { rpassword::read_password_from_tty(Some("Password: "))? }; CommandOutput::object( - bus::service(identity::BUS_ID) + gsb.local() .send(identity::Unlock::with_id(node_id, password)) .await .map_err(anyhow::Error::msg)?, @@ -410,14 +413,15 @@ impl IdentityCommand { IdentityCommand::Drop { node_or_alias, force, - } => drop_id::drop_id(node_or_alias, *force).await, + } => drop_id::drop_id(&gsb, node_or_alias, *force).await, IdentityCommand::Export { node_or_alias, file_path, plain, } => { let node_id = node_or_alias.clone().unwrap_or_default().resolve().await?; - let mut key_file = bus::service(identity::BUS_ID) + let mut key_file = gsb + .local() .send(identity::GetKeyFile(node_id)) .await? .map_err(anyhow::Error::msg)?; diff --git a/core/identity/src/cli/identity/drop_id.rs b/core/identity/src/cli/identity/drop_id.rs index ba46754efc..2a9c0e835b 100644 --- a/core/identity/src/cli/identity/drop_id.rs +++ b/core/identity/src/cli/identity/drop_id.rs @@ -1,4 +1,5 @@ -use super::{bus, identity, CommandOutput, NodeOrAlias, Result, RpcEndpoint}; +use super::{identity, CommandOutput, NodeOrAlias, Result, RpcEndpoint}; +use ya_core_model::bus::GsbBindPoints; async fn prompt(message: &str, question: &str) -> anyhow::Result { use tokio::io::{self, AsyncWriteExt}; @@ -19,9 +20,14 @@ async fn prompt(message: &str, question: &str) -> anyhow::Result { Ok(v) } -pub async fn drop_id(node_or_alias: &NodeOrAlias, force: bool) -> Result { +pub async fn drop_id( + gsb: &GsbBindPoints, + node_or_alias: &NodeOrAlias, + force: bool, +) -> Result { let command: identity::Get = node_or_alias.clone().into(); - let id = bus::service(identity::BUS_ID) + let id = gsb + .local() .send(command) .await .map_err(anyhow::Error::msg)?; @@ -63,7 +69,7 @@ pub async fn drop_id(node_or_alias: &NodeOrAlias, force: bool) -> Result Result { - let mut identities: Vec = bus::service(identity::BUS_ID) +use ya_core_model::bus::GsbBindPoints; +pub async fn list(gsb: &GsbBindPoints) -> Result { + let mut identities: Vec = gsb + .local() .send(identity::List::default()) .await .map_err(anyhow::Error::msg) diff --git a/core/identity/src/service/identity.rs b/core/identity/src/service/identity.rs index 67c2f26080..4b1d940913 100644 --- a/core/identity/src/service/identity.rs +++ b/core/identity/src/service/identity.rs @@ -612,7 +612,7 @@ pub async fn wait_for_default_account_unlock(gsb: Arc) -> anyhow: if identity_key.is_locked { let locked_identity = identity_key.node_id; let (tx, rx) = futures::channel::mpsc::unbounded(); - let endpoint = format!("{}/await_unlock", model::BUS_ID); + let endpoint = gsb.endpoint("await_unlock").addr().to_string(); let _ = bus::bind(&endpoint, move |e: IdentityEvent| { let mut tx_clone = tx.clone(); @@ -629,7 +629,7 @@ pub async fn wait_for_default_account_unlock(gsb: Arc) -> anyhow: Ok(()) } }); - subscribe(endpoint.clone()).await?; + subscribe(gsb.clone(), endpoint.clone()).await?; log::info!("{}", yansi::Color::RGB(0xFF, 0xA5, 0x00).paint( "Daemon cannot start because default account is locked. Unlock it by running 'yagna id unlock'" @@ -637,7 +637,7 @@ pub async fn wait_for_default_account_unlock(gsb: Arc) -> anyhow: wait_for_unlock(gsb.clone(), rx).await?; - unsubscribe(endpoint.clone()).await?; + unsubscribe(gsb.clone(), endpoint.clone()).await?; unbind(endpoint).await?; } @@ -663,18 +663,14 @@ async fn wait_for_unlock( Ok(()) } -async fn subscribe(endpoint: String) -> anyhow::Result<()> { - bus::service(model::BUS_ID) - .send(model::Subscribe { endpoint }) - .await??; +async fn subscribe(gsb: Arc, endpoint: String) -> anyhow::Result<()> { + gsb.local().send(model::Subscribe { endpoint }).await??; Ok(()) } -async fn unsubscribe(endpoint: String) -> anyhow::Result<()> { - bus::service(model::BUS_ID) - .send(model::Unsubscribe { endpoint }) - .await??; +async fn unsubscribe(gsb: Arc, endpoint: String) -> anyhow::Result<()> { + gsb.local().send(model::Unsubscribe { endpoint }).await??; Ok(()) } diff --git a/core/identity/tests/test_identity_unlock.rs b/core/identity/tests/test_identity_unlock.rs new file mode 100644 index 0000000000..f7da23df5b --- /dev/null +++ b/core/identity/tests/test_identity_unlock.rs @@ -0,0 +1,60 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use test_context::test_context; + +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::temp_dir; +use ya_framework_mocks::net::MockNet; +use ya_framework_mocks::node::MockNode; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_identity_unlock(_ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_identity_unlock")?; + let dir = dir.path(); + + let net = MockNet::new().bind(); + + let node1 = MockNode::new(net.clone(), "node-1", dir) + .with_prefixed_gsb() + .with_identity(); + node1.bind_gsb().await?; + + // Create new identity + let identity = node1.get_identity()?; + let appkey = identity.create_identity_key("locked-id").await?; + + // Lock identity and change it to default. After restart this identity needs to be unlocked + // for identity modules to start. + let password = "password1234"; + identity.set_default_identity(appkey.identity).await?; + identity.lock_identity(appkey.identity, password).await?; + + node1.stop().await; + + let started = Arc::new(AtomicBool::new(false)); + let started_ = started.clone(); + + tokio::task::spawn_local(async move { + node1.bind_gsb().await.unwrap(); + log::info!("Finished starting node."); + started_.store(true, Ordering::SeqCst); + }); + + assert!(!started.load(Ordering::SeqCst)); + + // Make sure that Identity::bind is still waiting for the identity to be unlocked. + tokio::time::sleep(Duration::from_secs(3)).await; + + identity.unlock_identity(appkey.identity, password).await?; + + // Now Identity module should be able to start. + tokio::time::sleep(Duration::from_secs(1)).await; + assert!(started.load(Ordering::SeqCst)); + Ok(()) +} diff --git a/core/model/src/bus.rs b/core/model/src/bus.rs index 5f03e0f6f6..b369d03675 100644 --- a/core/model/src/bus.rs +++ b/core/model/src/bus.rs @@ -24,6 +24,11 @@ impl GsbBindPoints { } } + // Returns local endpoint relative to give GSB binding point. + pub fn endpoint(&self, name: &str) -> bus::Endpoint { + bus::service(format!("{}/{name}", self.local.addr())) + } + pub fn prefix(&self, prefix: &str) -> GsbBindPoints { Self { public: bus::service(format!("{prefix}{}", self.public.addr())), @@ -53,3 +58,12 @@ impl Default for GsbBindPoints { Self::new(BUS_PUBLIC, BUS_LOCAL) } } + +impl std::fmt::Debug for GsbBindPoints { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GsbBindPoints") + .field("public", &self.public.addr()) + .field("local", &self.local.addr()) + .finish() + } +} diff --git a/core/model/src/payment.rs b/core/model/src/payment.rs index 8885180457..b9832fc1a7 100644 --- a/core/model/src/payment.rs +++ b/core/model/src/payment.rs @@ -30,6 +30,7 @@ pub mod local { use ya_client_model::{payment::allocation::Deposit, NodeId}; + pub const BUS_SERVICE_NAME: &str = "payment"; pub const BUS_ID: &str = "/local/payment"; pub const DEFAULT_PAYMENT_DRIVER: &str = "erc20"; diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index d13480d79e..ca6354f8a6 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -79,7 +79,6 @@ ya-framework-mocks = { version = "0.1" } actix-rt = "2.7" rand = "0.8" ethsign = "0.8" -portpicker = "0.1" serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"] } test-context = "0.1.4" url = "2.5" diff --git a/core/payment/tests/test_allocation.rs b/core/payment/tests/test_allocation.rs index 9cf842a6ba..ac3f59c83f 100644 --- a/core/payment/tests/test_allocation.rs +++ b/core/payment/tests/test_allocation.rs @@ -27,7 +27,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu .with_identity() .with_payment() .with_fake_market(); - node.bind_gsb(false).await?; + node.bind_gsb().await?; node.start_server(ctx).await?; let requestor_appkey = node @@ -175,7 +175,7 @@ async fn test_validate_allocation(ctx: &mut DroppableTestContext) -> anyhow::Res .with_identity() .with_payment() .with_fake_market(); - node.bind_gsb(false).await?; + node.bind_gsb().await?; node.start_server(ctx).await?; let appkey_req = node diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs index ed45c06ba2..79e22db446 100644 --- a/core/payment/tests/test_debit_notes.rs +++ b/core/payment/tests/test_debit_notes.rs @@ -29,7 +29,7 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< .with_payment() .with_fake_market() .with_fake_activity(); - node.bind_gsb(false).await?; + node.bind_gsb().await?; node.start_server(ctx).await?; let appkey_prov = node.get_identity()?.create_identity_key("provider").await?; diff --git a/core/payment/tests/test_invoices.rs b/core/payment/tests/test_invoices.rs index 7bf79839f8..b530b3a979 100644 --- a/core/payment/tests/test_invoices.rs +++ b/core/payment/tests/test_invoices.rs @@ -29,7 +29,7 @@ async fn test_invoice_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .with_identity() .with_payment() .with_fake_market(); - node.bind_gsb(false).await?; + node.bind_gsb().await?; node.start_server(ctx).await?; let appkey_prov = node.get_identity()?.create_identity_key("provider").await?; diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index 41c7557eb4..e13a489af7 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -23,7 +23,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .with_identity() .with_payment() .with_fake_market(); - node1.bind_gsb(false).await?; + node1.bind_gsb().await?; node1.start_server(ctx).await?; let appkey_req = node1 @@ -38,8 +38,9 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let node2 = MockNode::new(net, "node-2", dir) .with_identity() - .with_fake_payment(); - node2.bind_gsb(true).await?; + .with_fake_payment() + .with_prefixed_gsb(); + node2.bind_gsb().await?; let appkey_prov = node2 .get_identity()? diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 7c0d96b1fd..b3b1ee279e 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -51,7 +51,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Bind GSB and start server like yagna node would do in full setup. // Those functions will bind only modules chosen for MockNode. - node.bind_gsb(false).await?; + node.bind_gsb().await?; node.start_server(ctx).await?; // Creating identities is essential to use REST API and create Agreements and Payments. diff --git a/core/serv-api/Cargo.toml b/core/serv-api/Cargo.toml index 1c3b6184f5..efd64ef953 100644 --- a/core/serv-api/Cargo.toml +++ b/core/serv-api/Cargo.toml @@ -12,3 +12,4 @@ serde = { version = "1.0", features = ["derive"] } url = "2.1.1" ya-utils-cli = "0.1" +ya-core-model = "0.9" diff --git a/core/serv-api/src/lib.rs b/core/serv-api/src/lib.rs index dad1146da5..73415423a0 100644 --- a/core/serv-api/src/lib.rs +++ b/core/serv-api/src/lib.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::path::PathBuf; +use ya_core_model::bus::GsbBindPoints; pub use ya_utils_cli::{CommandOutput, ResponseTable}; #[derive(Clone, Debug, Default)] @@ -19,10 +20,18 @@ pub struct CliCtx { pub accept_terms: bool, pub quiet: bool, pub metrics_ctx: Option, + pub gsb: GsbBindPoints, } impl CliCtx { pub fn output(&self, output: CommandOutput) -> Result<(), anyhow::Error> { output.print(self.json_output) } + + pub fn with_prefixed_gsb(mut self, gsb: Option) -> Self { + if let Some(gsb) = gsb { + self.gsb = gsb; + } + self + } } diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index 73186003a3..9fd64a3089 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -21,6 +21,7 @@ ya-service-api-web = "0.2" ya-service-bus = { workspace = true } ya-sb-router = { workspace = true } ya-utils-path = { version = "0.1", path = "../../../utils/path" } +ya-std-utils = "0.1" ya-framework-basic = { version = "0.1" } diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index 7e2567f2c4..a720188384 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -5,13 +5,16 @@ use std::fs; use std::path::Path; use ya_client_model::NodeId; +use ya_core_model as model; use ya_core_model::appkey::AppKey; use ya_core_model::bus::GsbBindPoints; +use ya_core_model::identity; use ya_core_model::identity::IdentityInfo; use ya_identity::cli::{AppKeyCommand, IdentityCommand}; use ya_identity::service::Identity; use ya_persistence::executor::DbExecutor; use ya_service_api::{CliCtx, CommandOutput}; +use ya_service_bus::{typed as bus, RpcEndpoint}; use crate::net::{IMockNet, MockNet}; @@ -20,6 +23,8 @@ pub struct RealIdentity { net: MockNet, name: String, db: DbExecutor, + + gsb: Option, } impl RealIdentity { @@ -30,21 +35,45 @@ impl RealIdentity { net, name: name.to_string(), db, + gsb: None, } } + pub fn with_prefixed_gsb(mut self, gsb: Option) -> Self { + self.gsb = gsb; + self + } + fn create_db(testdir: &Path, name: &str) -> anyhow::Result { let db = DbExecutor::from_data_dir(testdir, name) .map_err(|e| anyhow!("Failed to create db [{name:?}]. Error: {e}"))?; Ok(db) } - pub async fn bind_gsb(&self, gsb: Option) -> anyhow::Result<()> { + pub async fn bind_gsb(&self) -> anyhow::Result<()> { log::info!("RealIdentity ({}) - binding GSB", self.name); - Identity::gsb_prefixed(&self.db, gsb).await?; + Identity::gsb_prefixed(&self.db, self.gsb.clone()).await?; Ok(()) } + pub async fn unbind(&self) { + log::info!("RealIdentity ({}) - unbinding", self.name); + + let gsb = self.gsb.clone().unwrap_or_default(); + bus::unbind(gsb.service(model::identity::BUS_SERVICE_NAME).public_addr()) + .await + .unwrap(); + bus::unbind(gsb.service(model::identity::BUS_SERVICE_NAME).local_addr()) + .await + .unwrap(); + bus::unbind(gsb.service(model::appkey::BUS_SERVICE_NAME).public_addr()) + .await + .unwrap(); + bus::unbind(gsb.service(model::appkey::BUS_SERVICE_NAME).local_addr()) + .await + .unwrap(); + } + pub async fn create_identity_key(&self, name: &str) -> anyhow::Result { let identity: IdentityInfo = self .create_identity(name) @@ -118,7 +147,7 @@ impl RealIdentity { } async fn run_create_identity(&self, command: IdentityCommand) -> anyhow::Result { - let ctx = CliCtx::default(); + let ctx = CliCtx::default().with_prefixed_gsb(self.gsb.clone()); let identity = parse_output_result::(command.run_command(&ctx).boxed_local().await?)?; @@ -127,7 +156,7 @@ impl RealIdentity { } pub async fn create_appkey(&self, name: &str, id: NodeId) -> anyhow::Result { - let ctx = CliCtx::default(); + let ctx = CliCtx::default().with_prefixed_gsb(self.gsb.clone()); let command = AppKeyCommand::Create { name: name.to_string(), role: "manager".to_string(), @@ -145,6 +174,48 @@ impl RealIdentity { parse_output::(output) } + + pub async fn lock_identity(&self, id: NodeId, password: &str) -> anyhow::Result<()> { + self.gsb_identity() + .local() + .send(identity::Lock::with_id(id).with_set_password(Some(password.to_string()))) + .await + .map_err(anyhow::Error::msg)??; + Ok(()) + } + + pub async fn unlock_identity(&self, id: NodeId, password: &str) -> anyhow::Result<()> { + self.gsb_identity() + .local() + .send(identity::Unlock::with_id(id, password.to_string())) + .await + .map_err(anyhow::Error::msg)??; + Ok(()) + } + + pub async fn set_default_identity(&self, id: NodeId) -> anyhow::Result<()> { + self.gsb_identity() + .local() + .send(identity::Update::with_id(id).with_default(true)) + .await + .map_err(anyhow::Error::msg)??; + Ok(()) + } + + fn gsb_identity(&self) -> GsbBindPoints { + self.gsb + .clone() + .unwrap_or_default() + .service(model::identity::BUS_SERVICE_NAME) + } + + #[allow(unused)] + fn gsb_appkey(&self) -> GsbBindPoints { + self.gsb + .clone() + .unwrap_or_default() + .service(model::appkey::BUS_SERVICE_NAME) + } } fn parse_output_result(output: CommandOutput) -> anyhow::Result { diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index 6b872a5ad4..1ca26ad9c0 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -33,6 +33,7 @@ pub struct MockNode { name: String, testdir: PathBuf, + use_prefix: bool, rest_url: Url, @@ -53,6 +54,7 @@ impl MockNode { net, name: name.to_string(), testdir, + use_prefix: false, rest_url: Self::generate_rest_url(), identity: None, payment: None, @@ -62,13 +64,18 @@ impl MockNode { } } + pub fn with_prefixed_gsb(mut self) -> Self { + self.use_prefix = true; + self + } + /// Use full wrapped Identity module for this node. pub fn with_identity(mut self) -> Self { - self.identity = Some(RealIdentity::new( - self.net.clone(), - &self.testdir, - &self.name, - )); + let gsb = self.binding_points(); + + self.identity = Some( + RealIdentity::new(self.net.clone(), &self.testdir, &self.name).with_prefixed_gsb(gsb), + ); self } @@ -80,7 +87,9 @@ impl MockNode { /// Use fake Market module for this node. pub fn with_fake_payment(mut self) -> Self { - self.fake_payment = Some(FakePayment::new(&self.name, &self.testdir)); + let gsb = self.binding_points(); + self.fake_payment = + Some(FakePayment::new(&self.name, &self.testdir).with_prefixed_gsb(gsb)); self } @@ -126,20 +135,15 @@ impl MockNode { /// `use_prefix` parameter decides if GSB will be bound without prefix like normally /// yagna does, or if GSB paths will be prefixed by Node name. /// The second options gives you possibility to run multiple nodes with GSB bound. - pub async fn bind_gsb(&self, use_prefix: bool) -> anyhow::Result<()> { - let gsb = match use_prefix { - true => Some(GsbBindPoints::default().prefix(&format!("/{}", self.name))), - false => None, - }; - + pub async fn bind_gsb(&self) -> anyhow::Result<()> { self.bind_gsb_router().await?; if let Some(identity) = &self.identity { - identity.bind_gsb(gsb.clone()).await?; + identity.bind_gsb().await?; } if let Some(payment) = &self.fake_payment { - payment.bind_gsb(gsb).await?; + payment.bind_gsb().await?; } if let Some(payment) = &self.payment { @@ -156,6 +160,19 @@ impl MockNode { Ok(()) } + pub async fn stop(&self) { + if let Some(identity) = &self.identity { + identity.unbind().await; + } + } + + fn binding_points(&self) -> Option { + match self.use_prefix { + true => Some(GsbBindPoints::default().prefix(&format!("/{}", self.name))), + false => None, + } + } + /// Query REST API client for payment module. /// /// You need to provider access token, which can be generated together with identity diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs index 9c0752f647..2f6008ce32 100644 --- a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -4,6 +4,7 @@ use anyhow::anyhow; use std::path::{Path, PathBuf}; use std::sync::Arc; +use ya_core_model as model; use ya_core_model::bus::GsbBindPoints; use ya_core_model::payment::public::{ AcceptDebitNote, AcceptInvoice, AcceptRejectError, Ack, CancelDebitNote, CancelError, @@ -22,6 +23,7 @@ pub struct FakePayment { testdir: PathBuf, db: DbExecutor, + gsb: GsbBindPoints, } impl FakePayment { @@ -31,6 +33,7 @@ impl FakePayment { name: name.to_string(), testdir: testdir.to_path_buf(), db, + gsb: GsbBindPoints::default().service(model::payment::local::BUS_SERVICE_NAME), } } @@ -41,10 +44,17 @@ impl FakePayment { Ok(db) } - pub async fn bind_gsb(&self, gsb: Option) -> anyhow::Result<()> { + pub fn with_prefixed_gsb(mut self, gsb: Option) -> Self { + self.gsb = gsb + .unwrap_or_default() + .service(model::payment::local::BUS_SERVICE_NAME); + self + } + + pub async fn bind_gsb(&self) -> anyhow::Result<()> { log::info!("FakePayment ({}) - binding GSB", self.name); - let gsb = gsb.unwrap_or_default().service("payment"); + let gsb = self.gsb.clone(); ServiceBinder::new(gsb.public_addr(), &self.db, ()) .bind(send_debit_note) .bind(accept_debit_note) From eff1d224da345bb77d42643ac13120403dd5fbcd Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 7 Aug 2024 19:43:29 +0200 Subject: [PATCH 081/125] Remove test_payment_sync (Will be added in next PRs) --- core/identity/tests/test_identity_unlock.rs | 2 +- core/payment/tests/test_payment_sync.rs | 56 --------------------- 2 files changed, 1 insertion(+), 57 deletions(-) delete mode 100644 core/payment/tests/test_payment_sync.rs diff --git a/core/identity/tests/test_identity_unlock.rs b/core/identity/tests/test_identity_unlock.rs index f7da23df5b..9fc9c5d1f0 100644 --- a/core/identity/tests/test_identity_unlock.rs +++ b/core/identity/tests/test_identity_unlock.rs @@ -13,7 +13,7 @@ use ya_framework_mocks::node::MockNode; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_identity_unlock(_ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("test_identity_unlock")?; let dir = dir.path(); diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs deleted file mode 100644 index e13a489af7..0000000000 --- a/core/payment/tests/test_payment_sync.rs +++ /dev/null @@ -1,56 +0,0 @@ -use test_context::test_context; - -use ya_framework_basic::async_drop::DroppableTestContext; -use ya_framework_basic::log::enable_logs; -use ya_framework_basic::{resource, temp_dir}; -use ya_framework_mocks::market::FakeMarket; -use ya_framework_mocks::net::MockNet; -use ya_framework_mocks::node::MockNode; -use ya_framework_mocks::payment::Driver; - -#[cfg_attr(not(feature = "framework-test"), ignore)] -#[test_context(DroppableTestContext)] -#[serial_test::serial] -async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); - - let dir = temp_dir!("test_payment_sync")?; - let dir = dir.path(); - - let net = MockNet::new().bind(); - - let node1 = MockNode::new(net.clone(), "node-1", dir) - .with_identity() - .with_payment() - .with_fake_market(); - node1.bind_gsb().await?; - node1.start_server(ctx).await?; - - let appkey_req = node1 - .get_identity()? - .create_from_private_key(&resource!("ci-requestor-1.key.priv")) - .await?; - - node1 - .get_payment()? - .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) - .await?; - - let node2 = MockNode::new(net, "node-2", dir) - .with_identity() - .with_fake_payment() - .with_prefixed_gsb(); - node2.bind_gsb().await?; - - let appkey_prov = node2 - .get_identity()? - .create_identity_key("provider") - .await?; - - let agreement = - FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); - node1.get_market()?.add_agreement(agreement.clone()).await; - - let _requestor = node1.rest_payments(&appkey_req.key)?; - Ok(()) -} From f62064fa13fc367ea56aae1fd7cc5fbc47cc259b Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 7 Aug 2024 19:45:57 +0200 Subject: [PATCH 082/125] Restore test_payment_sync --- core/payment/tests/test_payment_sync.rs | 56 +++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 core/payment/tests/test_payment_sync.rs diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs new file mode 100644 index 0000000000..76b2f506db --- /dev/null +++ b/core/payment/tests/test_payment_sync.rs @@ -0,0 +1,56 @@ +use test_context::test_context; + +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::{resource, temp_dir}; +use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::net::MockNet; +use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::Driver; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_payment_sync")?; + let dir = dir.path(); + + let net = MockNet::new().bind(); + + let node1 = MockNode::new(net.clone(), "node-1", dir) + .with_identity() + .with_payment() + .with_fake_market(); + node1.bind_gsb().await?; + node1.start_server(ctx).await?; + + let appkey_req = node1 + .get_identity()? + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; + + node1 + .get_payment()? + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) + .await?; + + let node2 = MockNode::new(net, "node-2", dir) + .with_prefixed_gsb() + .with_identity() + .with_fake_payment(); + node2.bind_gsb().await?; + + let appkey_prov = node2 + .get_identity()? + .create_identity_key("provider") + .await?; + + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + let _requestor = node1.rest_payments(&appkey_req.key)?; + Ok(()) +} From 9af3e261cc20193bfa3c5232d299cd7c4069ecbc Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 7 Aug 2024 19:47:17 +0200 Subject: [PATCH 083/125] Fix clippy --- core/identity/src/service.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/identity/src/service.rs b/core/identity/src/service.rs index 4b51fc9e04..d7d526d83d 100644 --- a/core/identity/src/service.rs +++ b/core/identity/src/service.rs @@ -28,7 +28,7 @@ impl Identity { base: Option, ) -> anyhow::Result<()> { let db = context.component(); - let gsb = base.unwrap_or(GsbBindPoints::default()); + let gsb = base.unwrap_or_default(); let gsb_ident = Arc::new(gsb.clone().service(model::identity::BUS_SERVICE_NAME)); let service = Arc::new(Mutex::new( From a8fe9a3fd7e828f2844ca641fc967e09ea6295bd Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 7 Aug 2024 19:54:13 +0200 Subject: [PATCH 084/125] Fix compilation --- core/serv/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/serv/src/main.rs b/core/serv/src/main.rs index 16e5a91fce..9960bed5c5 100644 --- a/core/serv/src/main.rs +++ b/core/serv/src/main.rs @@ -142,7 +142,7 @@ impl TryFrom<&CliArgs> for CliCtx { } else { true }, - metrics_ctx: None, + ..CliCtx::default() }) } } From ce39e82e8c88b4060b5f6c5b58a32faa695c3621 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Aug 2024 16:47:43 +0200 Subject: [PATCH 085/125] Send Accept Invoice to FakePayment --- Cargo.lock | 1 + core/payment/tests/test_payment_sync.rs | 39 +++++++++++- .../test-framework/framework-mocks/Cargo.toml | 1 + .../framework-mocks/src/identity.rs | 20 +++---- .../framework-mocks/src/node.rs | 10 +++- .../src/payment/fake_payment.rs | 59 ++++++++++++++++++- 6 files changed, 114 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69fe9a0d9c..8cad8ba1c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9084,6 +9084,7 @@ dependencies = [ "actix-web", "anyhow", "async-trait", + "bigdecimal 0.2.2", "chrono", "derive_more", "ethsign", diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index 76b2f506db..caf18300de 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -1,12 +1,18 @@ +use bigdecimal::BigDecimal; +use std::str::FromStr; use test_context::test_context; +use ya_client_model::payment::Acceptance; +use ya_core_model::payment::public::SendInvoice; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::fake_payment::FakePayment; use ya_framework_mocks::payment::Driver; +use ya_service_bus::RpcEndpoint; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] @@ -15,11 +21,11 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> enable_logs(true); let dir = temp_dir!("test_payment_sync")?; - let dir = dir.path(); + let dir = dir.into_path(); let net = MockNet::new().bind(); - let node1 = MockNode::new(net.clone(), "node-1", dir) + let node1 = MockNode::new(net.clone(), "node-1", &dir) .with_identity() .with_payment() .with_fake_market(); @@ -51,6 +57,33 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); node1.get_market()?.add_agreement(agreement.clone()).await; - let _requestor = node1.rest_payments(&appkey_req.key)?; + let payment = node1.get_payment()?; + let requestor = node1.rest_payments(&appkey_req.key)?; + + log::info!("Creating allocation..."); + let new_allocation = FakePayment::default_allocation(&agreement, BigDecimal::from(10u64))?; + let allocation = requestor.create_allocation(&new_allocation).await?; + log::info!("Allocation created. ({})", allocation.allocation_id); + + log::info!("Issuing invoice..."); + let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; + payment + .gsb_public_endpoint() + .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) + .await??; + + log::info!("Accepting Invoice ({})...", invoice.invoice_id); + requestor.get_invoice(&invoice.invoice_id).await.unwrap(); + requestor + .accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id.to_string(), + }, + ) + .await + .unwrap(); + Ok(()) } diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index 9fd64a3089..726c1435fb 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -32,6 +32,7 @@ actix-service = "2" anyhow = "1.0" async-trait = "0.1.80" chrono = "0.4" +bigdecimal = "0.2" derive_more = { workspace = true } ethsign = "0.8" futures = "0.3" diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index a720188384..bc985d4e12 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -106,16 +106,16 @@ impl RealIdentity { } fn register_identity_in_net(&self, id: NodeId) { - // This line is temporary, until we will be able to rebind all modules to non-fixed prefix. - // Currently, all modules must be bound under `/local/{module}` and `/public/{module}`. - // Not doing so would break most of them. - // For example Payment module uses fixed prefix to call market and identity modules. - // When we will work around this problem, we will be able to instantiate many nodes in tests. - self.net.register_node(&id, "/public"); - - // Should be instead in the future: - // self.net - // .register_node(&id, &format!("/{}/public/{id}", self.name)); + if let Some(gsb) = &self.gsb { + self.net.register_node(&id, &gsb.public_addr()); + } else { + // This line is temporary, until we will be able to rebind all modules to non-fixed prefix. + // Currently, all modules must be bound under `/local/{module}` and `/public/{module}`. + // Not doing so would break most of them. + // For example Payment module uses fixed prefix to call market and identity modules. + // When we will work around this problem, we will be able to instantiate many nodes in tests. + self.net.register_node(&id, "/public"); + } } pub async fn create_identity(&self, name: &str) -> anyhow::Result { diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index 1ca26ad9c0..9ac6f8b956 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -46,8 +46,8 @@ pub struct MockNode { } impl MockNode { - pub fn new(net: MockNet, name: &str, testdir: &Path) -> Self { - let testdir = testdir.join(name); + pub fn new(net: MockNet, name: &str, testdir: impl AsRef) -> Self { + let testdir = testdir.as_ref().join(name); fs::create_dir_all(&testdir).expect("Failed to create test directory"); MockNode { @@ -117,6 +117,12 @@ impl MockNode { .ok_or_else(|| anyhow!("Payment ({}) is not initialized", self.name)) } + pub fn get_fake_payment(&self) -> anyhow::Result { + self.fake_payment + .clone() + .ok_or_else(|| anyhow!("Payment ({}) is not initialized", self.name)) + } + pub fn get_market(&self) -> anyhow::Result { self.market .clone() diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs index 2f6008ce32..026ae8ec81 100644 --- a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -1,11 +1,19 @@ #![allow(unused)] use anyhow::anyhow; +use bigdecimal::BigDecimal; +use chrono::{Duration, Utc}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use uuid::Uuid; +use ya_client_model::market::Agreement; +use ya_agreement_utils::AgreementView; +use ya_client_model::payment::allocation::PaymentPlatformEnum; +use ya_client_model::payment::{DocumentStatus, Invoice, NewAllocation}; use ya_core_model as model; use ya_core_model::bus::GsbBindPoints; +use ya_core_model::payment::public; use ya_core_model::payment::public::{ AcceptDebitNote, AcceptInvoice, AcceptRejectError, Ack, CancelDebitNote, CancelError, CancelInvoice, PaymentSync, PaymentSyncError, PaymentSyncRequest, PaymentSyncWithBytes, @@ -15,7 +23,7 @@ use ya_core_model::payment::public::{ use ya_payment::migrations; use ya_payment::processor::PaymentProcessor; use ya_persistence::executor::DbExecutor; -use ya_service_bus::typed::ServiceBinder; +use ya_service_bus::typed::{Endpoint, ServiceBinder}; #[derive(Clone)] pub struct FakePayment { @@ -72,6 +80,55 @@ impl FakePayment { Ok(()) } + + pub fn gsb_local_endpoint(&self) -> Endpoint { + self.gsb.local() + } + + pub fn gsb_public_endpoint(&self) -> Endpoint { + self.gsb.public() + } + + fn platform_from(agreement: &Agreement) -> anyhow::Result { + let view = AgreementView::try_from(agreement)?; + Ok(view.pointer_typed("/demand/properties/golem/com/payment/chosen-platform")?) + } + + pub fn fake_invoice(agreement: &Agreement, amount: BigDecimal) -> anyhow::Result { + let platform = Self::platform_from(agreement)?; + Ok(Invoice { + invoice_id: Uuid::new_v4().to_string(), + issuer_id: agreement.offer.provider_id, + recipient_id: agreement.demand.requestor_id, + payee_addr: agreement.offer.provider_id.to_string(), + payer_addr: agreement.demand.requestor_id.to_string(), + payment_platform: platform, + timestamp: Utc::now(), + agreement_id: agreement.agreement_id.to_string(), + activity_ids: vec![], + amount, + payment_due_date: Utc::now() + Duration::seconds(10), + status: DocumentStatus::Issued, + }) + } + + pub fn default_allocation( + agreement: &Agreement, + amount: BigDecimal, + ) -> anyhow::Result { + let platform = Self::platform_from(agreement)?; + let payment_platform = PaymentPlatformEnum::PaymentPlatformName(platform); + + Ok(NewAllocation { + address: None, // Use default address (i.e. identity) + payment_platform: Some(payment_platform.clone()), + total_amount: amount, + timeout: None, + make_deposit: false, + deposit: None, + extend_timeout: None, + }) + } } async fn send_debit_note( From 83106c2e925f7ea830a33d720fd1f3eee5ae642a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Aug 2024 18:15:36 +0200 Subject: [PATCH 086/125] FakePayment - override GSB bindings and message channel --- core/payment/tests/test_payment_sync.rs | 9 +++- .../test-framework/framework-mocks/Cargo.toml | 2 +- .../framework-mocks/src/identity.rs | 2 +- .../src/payment/fake_payment.rs | 44 ++++++++++++++++++- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index caf18300de..ae3ea0f3c6 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use test_context::test_context; use ya_client_model::payment::Acceptance; -use ya_core_model::payment::public::SendInvoice; +use ya_core_model::payment::public::{AcceptInvoice, Ack, SendInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; @@ -72,6 +72,10 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) .await??; + let mut channel = node2 + .get_fake_payment()? + .message_channel::(Ok(Ack {})); + log::info!("Accepting Invoice ({})...", invoice.invoice_id); requestor.get_invoice(&invoice.invoice_id).await.unwrap(); requestor @@ -85,5 +89,8 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .await .unwrap(); + let accept = channel.recv().await.unwrap(); + assert_eq!(accept.invoice_id, invoice.invoice_id); + assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); Ok(()) } diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index 726c1435fb..4d4948c39a 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -44,7 +44,7 @@ regex = "1.5" serde = "1.0" serde_json = "1.0" test-context = "0.1.4" -tokio = "1" +tokio = { version = "1", features = ["sync"] } uuid = { version = "0.8", features = ["v4", "serde"] } url = "2.5" diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index bc985d4e12..43ed358fdd 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -107,7 +107,7 @@ impl RealIdentity { fn register_identity_in_net(&self, id: NodeId) { if let Some(gsb) = &self.gsb { - self.net.register_node(&id, &gsb.public_addr()); + self.net.register_node(&id, gsb.public_addr()); } else { // This line is temporary, until we will be able to rebind all modules to non-fixed prefix. // Currently, all modules must be bound under `/local/{module}` and `/public/{module}`. diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs index 026ae8ec81..697fcc9b93 100644 --- a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -3,12 +3,14 @@ use anyhow::anyhow; use bigdecimal::BigDecimal; use chrono::{Duration, Utc}; +use std::fmt::Display; use std::path::{Path, PathBuf}; use std::sync::Arc; +use tokio::sync::mpsc; use uuid::Uuid; -use ya_client_model::market::Agreement; use ya_agreement_utils::AgreementView; +use ya_client_model::market::Agreement; use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::{DocumentStatus, Invoice, NewAllocation}; use ya_core_model as model; @@ -24,6 +26,7 @@ use ya_payment::migrations; use ya_payment::processor::PaymentProcessor; use ya_persistence::executor::DbExecutor; use ya_service_bus::typed::{Endpoint, ServiceBinder}; +use ya_service_bus::RpcMessage; #[derive(Clone)] pub struct FakePayment { @@ -81,6 +84,45 @@ impl FakePayment { Ok(()) } + /// Function binds new GSB handler to the given message. + /// It returns Receiver that can be used to inspect the messages and make assertions. + /// GSB will always return `result` passed in parameter back to the caller. + /// Function overrides previous handler, so only one Receiver at the same time can be used. + pub fn message_channel( + &self, + result: Result, + ) -> mpsc::UnboundedReceiver + where + T: RpcMessage, + T::Item: Clone, + T::Error: Clone + Display, + { + let (sender, receiver) = mpsc::unbounded_channel(); + self.override_gsb_public() + .bind(move |_db: DbExecutor, _sender_id: String, msg: T| { + let result = result.clone(); + let sender = sender.clone(); + async move { + let _ = sender.send(msg).map_err(|_e| { + log::error!( + "[FakePayment] Unable to send message '{}' to channel.", + T::ID + ); + }); + result + } + }); + receiver + } + + pub fn override_gsb_public(&self) -> ServiceBinder { + ServiceBinder::new(self.gsb.public_addr(), &self.db, ()) + } + + pub fn override_gsb_local(&self) -> ServiceBinder { + ServiceBinder::new(self.gsb.local_addr(), &self.db, ()) + } + pub fn gsb_local_endpoint(&self) -> Endpoint { self.gsb.local() } From c4239c323d06429d496d13683eaf91a2af0c0354 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 9 Aug 2024 18:52:29 +0200 Subject: [PATCH 087/125] Document GSB message capture channels in tutorial --- .../tests/tutorial_how_to_use_module_tests.rs | 64 ++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index b3b1ee279e..04d3b878e5 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -1,9 +1,12 @@ use bigdecimal::BigDecimal; +use std::str::FromStr; use test_context::test_context; use ya_client_model::payment::allocation::{PaymentPlatform, PaymentPlatformEnum}; -use ya_client_model::payment::NewAllocation; +use ya_client_model::payment::{Acceptance, NewAllocation}; use ya_core_model::payment::local::GetStatus; +use ya_core_model::payment::public::{AcceptInvoice, Ack, SendInvoice}; +use ya_service_bus::RpcEndpoint; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; @@ -12,6 +15,7 @@ use ya_framework_basic::{resource, temp_dir}; use ya_framework_mocks::market::FakeMarket; use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::fake_payment::FakePayment; use ya_framework_mocks::payment::Driver; // Tests should be always wrapped in these macros. @@ -41,7 +45,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Create MockNode which is container for all Golem modules and represents // single node in tests. - let node = MockNode::new(net, "node-1", dir) + let node = MockNode::new(net.clone(), "node-1", dir) // Request instantiating wrappers around real Golem modules. .with_identity() .with_payment() @@ -123,5 +127,61 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any status.amount ); + // Instead of using real yagna node, we can use FakePayment instead. + // It allows us to capture GSB traffic to test if real payment module sends them correctly. + let node3 = MockNode::new(net, "node-3", dir) + // All GSB handler will be bound on addresses prefixed by node name. + // After creating identity, it will be registered in MockNet and all GSB messages + // will be routed to those prefixes. + .with_prefixed_gsb() + .with_identity() + .with_fake_payment(); + node3.bind_gsb().await?; + + let identity = node3.get_identity()?.create_identity_key("node-3").await?; + + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, identity.identity).unwrap(); + node.get_market()?.add_agreement(agreement.clone()).await; + + // Sending Invoice from FakePayment node to real payment module. + // This can be done by directly sending GSB message directly, simulating, what would happen + // after calling REST API. + // `gsb_public_endpoint` returns GSB address on which `node` payment module was bound. + // Note that you should never build GSB addresses directly, because `MockNode` implementation + // decides about them, and they can be different from real yagna. + let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; + node.get_payment()? + .gsb_public_endpoint() + .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) + .await??; + + // FakePayment responds always with correct answer to any GSB message. + // This behavior can be overridden by querying `FakePayment::message_channel` and setting + // expected response. + // Function overrides default message handler and returns channel which yields all messages + // received. + let mut channel = node3 + .get_fake_payment()? + .message_channel::(Ok(Ack {})); + + // Sending accept Invoice in real Payment module. + // Internally GSB message will be sent to node3 and will be captured by channel. + let new_allocation = FakePayment::default_allocation(&agreement, BigDecimal::from(10u64))?; + let allocation = api.create_allocation(&new_allocation).await?; + api.accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id.to_string(), + }, + ) + .await + .unwrap(); + + // Waiting for any message sent to `node3`. + // We expect that `AcceptInvoice` message will be received after previous REST api call was made. + let accept = channel.recv().await.unwrap(); + assert_eq!(accept.invoice_id, invoice.invoice_id); Ok(()) } From fcfc1d5e8145993a4500c26a597254dac59e49a9 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 27 Aug 2024 20:47:49 +0200 Subject: [PATCH 088/125] Fix timeout parsing error in payment REST API. --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- core/payment/src/api/payments.rs | 11 ++++------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69fe9a0d9c..b3af8f008f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8757,7 +8757,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=a2fa0d54fe47a013add90b7ae687afe4f08ce7a2#a2fa0d54fe47a013add90b7ae687afe4f08ce7a2" +source = "git+https://github.com/golemfactory/ya-client.git?rev=fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1#fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" dependencies = [ "actix-codec", "awc", @@ -8781,7 +8781,7 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=a2fa0d54fe47a013add90b7ae687afe4f08ce7a2#a2fa0d54fe47a013add90b7ae687afe4f08ce7a2" +source = "git+https://github.com/golemfactory/ya-client.git?rev=fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1#fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" dependencies = [ "bigdecimal 0.2.2", "chrono", diff --git a/Cargo.toml b/Cargo.toml index d0a44fd20f..5139af7233 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -287,9 +287,9 @@ ya-service-api-interfaces = { path = "core/serv-api/interfaces" } ya-service-api-web = { path = "core/serv-api/web" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "a2fa0d54fe47a013add90b7ae687afe4f08ce7a2" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" } #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "a2fa0d54fe47a013add90b7ae687afe4f08ce7a2" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" } #ya-client-model = { path = "../ya-client/model" } golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } diff --git a/core/payment/src/api/payments.rs b/core/payment/src/api/payments.rs index 511311421e..38597413cb 100644 --- a/core/payment/src/api/payments.rs +++ b/core/payment/src/api/payments.rs @@ -30,11 +30,8 @@ async fn get_payments( id: Identity, ) -> HttpResponse { let node_id = id.identity; - let timeout_secs = query - .event_params - .timeout - .unwrap_or(params::DEFAULT_EVENT_TIMEOUT); - let after_timestamp = query.event_params.after_timestamp.map(|d| d.naive_utc()); + let timeout_secs = query.timeout.unwrap_or(params::DEFAULT_EVENT_TIMEOUT); + let after_timestamp = query.after_timestamp.map(|d| d.naive_utc()); let network = match query .network .as_ref() @@ -53,8 +50,8 @@ async fn get_payments( Ok(driver) => driver, Err(e) => return response::server_error(&e), }; - let max_events = query.event_params.max_events; - let app_session_id = &query.event_params.app_session_id; + let max_events = query.max_events; + let app_session_id = &query.app_session_id; let dao: PaymentDao = db.as_dao(); let getter = || async { From f3c64c1cfe8579eac2e73f5f7746d894be7a3a6e Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Mon, 2 Sep 2024 18:28:59 +0200 Subject: [PATCH 089/125] Payments test env: breaking network; Test payment sync --- core/payment/src/api/invoices.rs | 4 +- core/payment/src/config.rs | 12 +++ core/payment/src/lib.rs | 5 +- core/payment/src/service.rs | 41 +--------- core/payment/tests/test_allocation.rs | 4 +- core/payment/tests/test_debit_notes.rs | 2 +- core/payment/tests/test_invoices.rs | 2 +- core/payment/tests/test_payment_sync.rs | 75 ++++++++++++++++++- .../tests/tutorial_how_to_use_module_tests.rs | 2 +- .../test-framework/framework-mocks/src/net.rs | 61 +++++++++++---- .../framework-mocks/src/node.rs | 5 +- .../framework-mocks/src/payment.rs | 17 +++-- 12 files changed, 157 insertions(+), 73 deletions(-) diff --git a/core/payment/src/api/invoices.rs b/core/payment/src/api/invoices.rs index af7755133f..81f8df64f8 100644 --- a/core/payment/src/api/invoices.rs +++ b/core/payment/src/api/invoices.rs @@ -528,11 +528,11 @@ async fn accept_invoice( .await; if let Ok(response) = send_result { - log::debug!("AcceptInvoice delivered"); + log::debug!("AcceptInvoice delivered for [{invoice_id}]"); dao.mark_accept_sent(invoice_id.clone(), node_id).await?; response?; } else { - log::debug!("AcceptInvoice not delivered"); + log::debug!("AcceptInvoice not delivered for [{invoice_id}]"); sync_dao.upsert(issuer_id).await?; SYNC_NOTIFS_NOTIFY.notify_one(); } diff --git a/core/payment/src/config.rs b/core/payment/src/config.rs index 9e8fab3574..afdfa38470 100644 --- a/core/payment/src/config.rs +++ b/core/payment/src/config.rs @@ -8,6 +8,13 @@ pub struct Config { #[derive(StructOpt, Clone)] pub struct SyncNotifBackoffConfig { + /// Enables background job for synchronizing invoice / debit note document status. + /// + /// This depends on the identity service being enabled to work. If you're working with a limited + /// subsets of services (e.g. in payment_api.rs example) you might wish to disable that. + #[structopt(parse(try_from_str), default_value = "true")] + pub run_sync_job: bool, + #[structopt(long, env = "YA_PAYMENT_SYNC_NOTIF_BACKOFF_INITIAL_DELAY", parse(try_from_str = humantime::parse_duration), default_value = "30s")] pub initial_delay: std::time::Duration, @@ -38,4 +45,9 @@ impl Config { // or default values if ENV variables are not set. Config::from_iter_safe(&[""]) } + + pub fn run_sync_job(mut self, value: bool) -> Self { + self.sync_notif_backoff.run_sync_job = value; + self + } } diff --git a/core/payment/src/lib.rs b/core/payment/src/lib.rs index 41b3d1e39d..1e28cd8811 100644 --- a/core/payment/src/lib.rs +++ b/core/payment/src/lib.rs @@ -2,9 +2,10 @@ #![allow(unused_variables)] // Crate under development pub use crate::config::Config; use crate::processor::PaymentProcessor; + use futures::FutureExt; -use service::BindOptions; use std::{sync::Arc, time::Duration}; + use ya_core_model::payment::local as pay_local; use ya_persistence::executor::DbExecutor; use ya_service_api_interfaces::*; @@ -58,7 +59,7 @@ impl PaymentService { let config = Arc::new(Config::from_env()?); let processor = Arc::new(PaymentProcessor::new(db.clone())); - self::service::bind_service(&db, processor.clone(), BindOptions::default(), config); + self::service::bind_service(&db, processor.clone(), config); tokio::task::spawn(async move { processor.release_allocations(false).await; diff --git a/core/payment/src/service.rs b/core/payment/src/service.rs index a4c63c7915..282a8e9310 100644 --- a/core/payment/src/service.rs +++ b/core/payment/src/service.rs @@ -13,39 +13,11 @@ use ya_core_model::payment::public::{AcceptDebitNote, AcceptInvoice, PaymentSync use ya_persistence::executor::DbExecutor; use ya_service_bus::typed::{service, ServiceBinder}; -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BindOptions { - /// Enables background job for synchronizing invoice / debit note document status. - /// - /// This depends on the identity service being enabled to work. If you're working with a limited - /// subsets of services (e.g. in payment_api.rs example) you might wish to disable that. - pub run_sync_job: bool, -} - -impl BindOptions { - /// Configure the `run_async_job` option. - pub fn run_sync_job(mut self, value: bool) -> Self { - self.run_sync_job = value; - self - } -} - -impl Default for BindOptions { - fn default() -> Self { - BindOptions { run_sync_job: true } - } -} - -pub fn bind_service( - db: &DbExecutor, - processor: Arc, - opts: BindOptions, - config: Arc, -) { +pub fn bind_service(db: &DbExecutor, processor: Arc, config: Arc) { log::debug!("Binding payment service to service bus"); local::bind_service(db, processor.clone()); - public::bind_service(db, processor, opts, config); + public::bind_service(db, processor, config); log::debug!("Successfully bound payment service to service bus"); } @@ -834,12 +806,7 @@ mod public { use ya_persistence::types::Role; use ya_std_utils::LogErr; - pub fn bind_service( - db: &DbExecutor, - processor: Arc, - opts: BindOptions, - config: Arc, - ) { + pub fn bind_service(db: &DbExecutor, processor: Arc, config: Arc) { log::debug!("Binding payment public service to service bus"); ServiceBinder::new(BUS_ID, db, processor) @@ -857,7 +824,7 @@ mod public { .bind_with_processor(sync_payment) .bind_with_processor(sync_payment_with_bytes); - if opts.run_sync_job { + if config.sync_notif_backoff.run_sync_job { send_sync_notifs_job(db.clone(), config); send_sync_requests(db.clone()); } diff --git a/core/payment/tests/test_allocation.rs b/core/payment/tests/test_allocation.rs index ac3f59c83f..9585ba22a2 100644 --- a/core/payment/tests/test_allocation.rs +++ b/core/payment/tests/test_allocation.rs @@ -25,7 +25,7 @@ async fn test_release_allocation(ctx: &mut DroppableTestContext) -> anyhow::Resu let node = MockNode::new(net, "node-1", dir.path()) .with_identity() - .with_payment() + .with_payment(None) .with_fake_market(); node.bind_gsb().await?; node.start_server(ctx).await?; @@ -173,7 +173,7 @@ async fn test_validate_allocation(ctx: &mut DroppableTestContext) -> anyhow::Res let node = MockNode::new(net, "node-1", dir.path()) .with_identity() - .with_payment() + .with_payment(None) .with_fake_market(); node.bind_gsb().await?; node.start_server(ctx).await?; diff --git a/core/payment/tests/test_debit_notes.rs b/core/payment/tests/test_debit_notes.rs index 79e22db446..c30f2c477a 100644 --- a/core/payment/tests/test_debit_notes.rs +++ b/core/payment/tests/test_debit_notes.rs @@ -26,7 +26,7 @@ async fn test_debit_note_flow(ctx: &mut DroppableTestContext) -> anyhow::Result< let node = MockNode::new(net, "node-1", dir) .with_identity() - .with_payment() + .with_payment(None) .with_fake_market() .with_fake_activity(); node.bind_gsb().await?; diff --git a/core/payment/tests/test_invoices.rs b/core/payment/tests/test_invoices.rs index b530b3a979..32fec6a6ea 100644 --- a/core/payment/tests/test_invoices.rs +++ b/core/payment/tests/test_invoices.rs @@ -27,7 +27,7 @@ async fn test_invoice_flow(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let node = MockNode::new(net, "node-1", dir) .with_identity() - .with_payment() + .with_payment(None) .with_fake_market(); node.bind_gsb().await?; node.start_server(ctx).await?; diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index ae3ea0f3c6..84c7fb962d 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -1,9 +1,11 @@ use bigdecimal::BigDecimal; use std::str::FromStr; +use std::time::Duration; use test_context::test_context; +use tokio::sync::mpsc::error::TryRecvError; use ya_client_model::payment::Acceptance; -use ya_core_model::payment::public::{AcceptInvoice, Ack, SendInvoice}; +use ya_core_model::payment::public::{AcceptInvoice, Ack, PaymentSyncWithBytes, SendInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; @@ -12,6 +14,7 @@ use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::fake_payment::FakePayment; use ya_framework_mocks::payment::Driver; +use ya_service_bus::timeout::IntoTimeoutFuture; use ya_service_bus::RpcEndpoint; #[cfg_attr(not(feature = "framework-test"), ignore)] @@ -25,9 +28,16 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let net = MockNet::new().bind(); + // Notifies will be sent in regular intervals, with values more appropriate for tests + // taking less time to trigger. + let mut config = ya_payment::config::Config::from_env()?; + config.sync_notif_backoff.run_sync_job = true; + config.sync_notif_backoff.initial_delay = Duration::from_secs(2); + config.sync_notif_backoff.cap = Some(Duration::from_secs(2)); + let node1 = MockNode::new(net.clone(), "node-1", &dir) .with_identity() - .with_payment() + .with_payment(Some(config)) .with_fake_market(); node1.bind_gsb().await?; node1.start_server(ctx).await?; @@ -42,7 +52,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) .await?; - let node2 = MockNode::new(net, "node-2", dir) + let node2 = MockNode::new(net.clone(), "node-2", dir) .with_prefixed_gsb() .with_identity() .with_fake_payment(); @@ -65,6 +75,9 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let allocation = requestor.create_allocation(&new_allocation).await?; log::info!("Allocation created. ({})", allocation.allocation_id); + // Scenario 1: + // Send Invoice to Requestor node. + // Requestor is able to immediately accept. log::info!("Issuing invoice..."); let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; payment @@ -92,5 +105,61 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let accept = channel.recv().await.unwrap(); assert_eq!(accept.invoice_id, invoice.invoice_id); assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); + + // Scenario 2: + // Send Invoice to Requestor node. + // Requestor attempt to accept Invoice but network is broken. + // Acceptance will be sent later as payment sync. + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + log::info!("Issuing invoice..."); + let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; + payment + .gsb_public_endpoint() + .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) + .await??; + + log::info!( + "Accepting Invoice ({})... with broken network", + invoice.invoice_id + ); + net.break_network_for(appkey_prov.identity); + + requestor.get_invoice(&invoice.invoice_id).await.unwrap(); + requestor + .accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id.to_string(), + }, + ) + .await + .unwrap(); + + // We expect that AcceptInvoice wasn't delivered. + matches!(channel.try_recv().unwrap_err(), TryRecvError::Empty); + + net.enable_network_for(appkey_prov.identity); + let mut sync_channel = node2 + .get_fake_payment()? + .message_channel::(Ok(Ack {})); + + // We expect that AcceptInvoice will be delivered within 4s. + let sync = sync_channel + .recv() + .timeout(Some(Duration::from_secs(4))) + .await + .unwrap() + .unwrap(); + assert_eq!(sync.invoice_accepts.len(), 1); + assert_eq!(sync.invoice_accepts[0].invoice_id, invoice.invoice_id); + assert_eq!( + sync.invoice_accepts[0].acceptance.total_amount_accepted, + invoice.amount + ); + Ok(()) } diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 04d3b878e5..02d4b9e38f 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -48,7 +48,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any let node = MockNode::new(net.clone(), "node-1", dir) // Request instantiating wrappers around real Golem modules. .with_identity() - .with_payment() + .with_payment(None) // Mock market module with very basic implementation, which will allow to manually // create fake Agreements without need for Offers broadcasting and negotiation process. .with_fake_market(); diff --git a/test-utils/test-framework/framework-mocks/src/net.rs b/test-utils/test-framework/framework-mocks/src/net.rs index 1ed1127f4f..6ce1581253 100644 --- a/test-utils/test-framework/framework-mocks/src/net.rs +++ b/test-utils/test-framework/framework-mocks/src/net.rs @@ -1,4 +1,5 @@ -use anyhow::Result; +use anyhow::{bail, Result}; +use std::collections::hash_map::Entry; use std::collections::HashMap; use std::rc::Rc; use std::sync::{Arc, Mutex}; @@ -21,10 +22,18 @@ pub struct MockNet { broadcast: BCastService, } +#[derive(Clone)] +struct NodeInfo { + pub prefix: String, + // Indicates that connection is broken. Sending message to this node + // will result with following error. + pub break_error: Option, +} + #[derive(Default)] struct MockNetInner { /// Maps NodeIds to gsb prefixes of other nodes. - pub nodes: HashMap, + pub nodes: HashMap, } impl Default for MockNet { @@ -56,7 +65,11 @@ impl IMockNet for MockNet { log::info!("[MockNet] Registering node {node_id} at prefix: {prefix}"); let mut inner = self.inner.lock().unwrap(); - if inner.nodes.insert(*node_id, prefix.to_string()).is_some() { + let info = NodeInfo { + prefix: prefix.to_string(), + break_error: None, + }; + if inner.nodes.insert(*node_id, info).is_some() { panic!("[MockNet] Node [{}] already existed.", node_id); } } @@ -88,6 +101,22 @@ impl MockNet { self } + pub fn break_network_for(&self, node_id: NodeId) { + let mut inner = self.inner.lock().unwrap(); + inner.nodes.entry(node_id).and_modify(|info| { + log::info!("Disabling networking for: {node_id}"); + info.break_error = Some("Unreachable".to_string()); + }); + } + + pub fn enable_network_for(&self, node_id: NodeId) { + let mut inner = self.inner.lock().unwrap(); + inner.nodes.entry(node_id).and_modify(|info| { + log::info!("Enabling networking for: {node_id}"); + info.break_error = None; + }); + } + fn translate_to(&self, id: NodeId, addr: &str) -> Result { let prefix = self.node_prefix(id)?; let net_prefix = format!("/net/{}", id); @@ -97,19 +126,21 @@ impl MockNet { } fn node_prefix(&self, id: NodeId) -> Result { - let inner = self.inner.lock().unwrap(); - inner - .nodes - .get(&id) - .cloned() - .ok_or_else(|| anyhow::anyhow!("Node not registered: {id}")) + let mut inner = self.inner.lock().unwrap(); + match inner.nodes.entry(id) { + Entry::Occupied(info) => match info.get().break_error { + Some(ref err) => bail!("{err}"), + None => Ok(info.get().prefix.clone()), + }, + Entry::Vacant(_) => bail!("Node not registered: {id}"), + } } - pub fn node_by_prefix(&self, address: &str) -> Option { + fn node_by_prefix(&self, address: &str) -> Option { let inner = self.inner.lock().unwrap(); - for (id, prefix) in inner.nodes.iter() { - if address.contains(prefix) { - return Some(*id); + for (_id, info) in inner.nodes.iter() { + if address.contains(&info.prefix) { + return Some(info.clone()); } } None @@ -162,8 +193,8 @@ impl MockNet { let addr_local = addr.replacen("local", "public", 1); let node_id = match mock_net.node_by_prefix(&addr_local) { - Some(node_id) => node_id, - None => { + Some(info) if info.break_error.is_none() => info.prefix, + _ => { log::debug!( "Not broadcasting on topic {topic} to {addr}. Node not found on list. \ Probably networking was disabled for this Node." diff --git a/test-utils/test-framework/framework-mocks/src/node.rs b/test-utils/test-framework/framework-mocks/src/node.rs index 9ac6f8b956..ec5bc7ac49 100644 --- a/test-utils/test-framework/framework-mocks/src/node.rs +++ b/test-utils/test-framework/framework-mocks/src/node.rs @@ -10,6 +10,7 @@ use ya_client::payment::PaymentApi; use ya_client::web::WebClient; use ya_core_model::bus::GsbBindPoints; use ya_framework_basic::async_drop::DroppableTestContext; +use ya_payment::Config; use ya_service_api_web::middleware::auth; use ya_service_api_web::middleware::cors::{AppKeyCors, CorsConfig}; use ya_service_api_web::rest_api_host_port; @@ -80,8 +81,8 @@ impl MockNode { } /// Use full wrapped Payment module for this node. - pub fn with_payment(mut self) -> Self { - self.payment = Some(RealPayment::new(&self.name, &self.testdir)); + pub fn with_payment(mut self, config: Option) -> Self { + self.payment = Some(RealPayment::new(&self.name, &self.testdir).with_config(config)); self } diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index f5883139a3..78f4539cff 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -14,7 +14,6 @@ use ya_payment::api::web_scope; use ya_payment::config::Config; use ya_payment::migrations; use ya_payment::processor::PaymentProcessor; -use ya_payment::service::BindOptions; use ya_persistence::executor::DbExecutor; use ya_service_bus::typed as bus; use ya_service_bus::typed::Endpoint; @@ -48,21 +47,30 @@ pub struct RealPayment { db: DbExecutor, processor: Arc, + + config: Arc, } impl RealPayment { pub fn new(name: &str, testdir: &Path) -> Self { let db = Self::create_db(testdir, "payment.db").unwrap(); let processor = Arc::new(PaymentProcessor::new(db.clone())); + let config = Config::from_env().unwrap().run_sync_job(false); RealPayment { name: name.to_string(), testdir: testdir.to_path_buf(), db, processor, + config: Arc::new(config), } } + pub fn with_config(mut self, config: Option) -> Self { + self.config = Arc::new(config.unwrap_or(Config::from_env().unwrap())); + self + } + fn create_db(testdir: &Path, name: &str) -> anyhow::Result { let db = DbExecutor::from_data_dir(testdir, name) .map_err(|e| anyhow!("Failed to create db [{name:?}]. Error: {e}"))?; @@ -73,12 +81,7 @@ impl RealPayment { pub async fn bind_gsb(&self) -> anyhow::Result<()> { log::info!("RealPayment ({}) - binding GSB", self.name); - ya_payment::service::bind_service( - &self.db, - self.processor.clone(), - BindOptions::default().run_sync_job(false), - Arc::new(Config::from_env()?), - ); + ya_payment::service::bind_service(&self.db, self.processor.clone(), self.config.clone()); self.start_dummy_driver().await?; self.start_erc20_driver().await?; From 934a0e63adf239f7853dffc1c72873c724571270 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 3 Sep 2024 10:29:00 +0200 Subject: [PATCH 090/125] Fix unit tests --- core/payment/examples/payment_api.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/payment/examples/payment_api.rs b/core/payment/examples/payment_api.rs index 93dd3f764c..10adc28b4f 100644 --- a/core/payment/examples/payment_api.rs +++ b/core/payment/examples/payment_api.rs @@ -7,7 +7,6 @@ use ethsign::keyfile::Bytes; use ethsign::{KeyFile, Protected, SecretKey}; use futures::Future; use rand::Rng; -use ya_payment::service::BindOptions; use std::convert::TryInto; use std::io::Write; @@ -234,8 +233,7 @@ async fn main() -> anyhow::Result<()> { ya_payment::service::bind_service( &db, processor, - BindOptions::default().run_sync_job(false), - Arc::new(PaymentConfig::from_env()?), + Arc::new(PaymentConfig::from_env()?.run_sync_job(false)), ); log::debug!("bind_service()"); From 03992b33a692bfa4f79f999a303e2993a83892ce Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 3 Sep 2024 13:50:11 +0200 Subject: [PATCH 091/125] Test Payment confirmation sync delivery --- core/payment/src/payment_sync.rs | 31 ++++++++++-- core/payment/tests/test_payment_sync.rs | 49 +++++++++++++++++-- .../test-framework/framework-mocks/src/net.rs | 4 +- .../framework-mocks/src/payment.rs | 34 +++++++++++++ 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/core/payment/src/payment_sync.rs b/core/payment/src/payment_sync.rs index 6e5a8f1357..a4af2b4a37 100644 --- a/core/payment/src/payment_sync.rs +++ b/core/payment/src/payment_sync.rs @@ -33,7 +33,7 @@ const REMOTE_CALL_TIMEOUT: Duration = Duration::from_secs(30); async fn payment_sync( db: &DbExecutor, - current: NodeId, + owner: NodeId, peer_id: NodeId, ) -> anyhow::Result<(PaymentSync, PaymentSyncWithBytes)> { let payment_dao: PaymentDao = db.as_dao(); @@ -62,7 +62,7 @@ async fn payment_sync( } let mut invoice_accepts = Vec::default(); - for invoice in invoice_dao.unsent_accepted(current, peer_id).await? { + for invoice in invoice_dao.unsent_accepted(owner, peer_id).await? { invoice_accepts.push(AcceptInvoice::new( invoice.invoice_id, Acceptance { @@ -74,7 +74,7 @@ async fn payment_sync( } let mut invoice_rejects = Vec::default(); - for invoice in invoice_dao.unsent_rejected(current, peer_id).await? { + for invoice in invoice_dao.unsent_rejected(owner, peer_id).await? { let events = invoice_event_dao .get_for_invoice_id( invoice.invoice_id.clone(), @@ -98,7 +98,7 @@ async fn payment_sync( } let mut debit_note_accepts = Vec::default(); - for debit_note in debit_note_dao.unsent_accepted(current, peer_id).await? { + for debit_note in debit_note_dao.unsent_accepted(owner, peer_id).await? { debit_note_accepts.push(AcceptDebitNote::new( debit_note.debit_note_id, Acceptance { @@ -131,24 +131,44 @@ async fn mark_all_sent(db: &DbExecutor, owner_id: NodeId, msg: PaymentSync) -> a let debit_note_dao: DebitNoteDao = db.as_dao(); for payment_send in msg.payments { + log::info!( + "Delivered payment [{}] to [{}]", + payment_send.payment.payment_id, + payment_send.payment.payee_id + ); payment_dao .mark_sent(payment_send.payment.payment_id) .await?; } for invoice_accept in msg.invoice_accepts { + log::info!( + "Delivered Invoice [{}] acceptance to [{}]", + invoice_accept.invoice_id, + invoice_accept.issuer_id + ); invoice_dao .mark_accept_sent(invoice_accept.invoice_id, owner_id) .await?; } for invoice_reject in msg.invoice_rejects { + log::info!( + "Delivered Invoice [{}] rejection to [{}]", + invoice_reject.invoice_id, + invoice_reject.issuer_id + ); invoice_dao .mark_reject_sent(invoice_reject.invoice_id, owner_id) .await?; } for debit_note_accept in msg.debit_note_accepts { + log::info!( + "Delivered DebitNote [{}] acceptance to [{}]", + debit_note_accept.debit_note_id, + debit_note_accept.issuer_id + ); debit_note_dao .mark_accept_sent(debit_note_accept.debit_note_id, owner_id) .await?; @@ -163,6 +183,7 @@ async fn send_sync_notifs_for_identity( config: &Config, cutoff: &DateTime, ) -> anyhow::Result> { + log::debug!("Processing PaymentSync from identity [{identity}]."); let dao: SyncNotifsDao = db.as_dao(); let backoff_config = &config.sync_notif_backoff; @@ -202,7 +223,7 @@ async fn send_sync_notifs_for_identity( // Centralnet and hybridnet return different errors when the endpoint is not supported, so // we have to resort to checking error message. // This message will be sent even if the node can handle PaymentSyncWithBytes but is not - // connected at all, but there is no standard way to differenciate between these cases. + // connected at all, but there is no standard way to differentiate between these cases. if matches!(&result, Err(e) if e.to_string().contains("endpoint address not found")) { log::debug!("Sending PaymentSync as [{identity}] to [{peer}]: PaymentSyncWithBytes endpoint not found, falling back to PaymentSync."); result = ya_net::from(identity) diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index 84c7fb962d..f776fae5f5 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -1,4 +1,5 @@ use bigdecimal::BigDecimal; +use chrono::Utc; use std::str::FromStr; use std::time::Duration; use test_context::test_context; @@ -13,7 +14,7 @@ use ya_framework_mocks::market::FakeMarket; use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::fake_payment::FakePayment; -use ya_framework_mocks::payment::Driver; +use ya_framework_mocks::payment::{Driver, PaymentRestExt}; use ya_service_bus::timeout::IntoTimeoutFuture; use ya_service_bus::RpcEndpoint; @@ -75,7 +76,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let allocation = requestor.create_allocation(&new_allocation).await?; log::info!("Allocation created. ({})", allocation.allocation_id); - // Scenario 1: + log::info!("================== Scenario 1 =================="); // Send Invoice to Requestor node. // Requestor is able to immediately accept. log::info!("Issuing invoice..."); @@ -106,7 +107,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> assert_eq!(accept.invoice_id, invoice.invoice_id); assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); - // Scenario 2: + log::info!("================== Scenario 2 =================="); // Send Invoice to Requestor node. // Requestor attempt to accept Invoice but network is broken. // Acceptance will be sent later as payment sync. @@ -142,10 +143,10 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> // We expect that AcceptInvoice wasn't delivered. matches!(channel.try_recv().unwrap_err(), TryRecvError::Empty); - net.enable_network_for(appkey_prov.identity); let mut sync_channel = node2 .get_fake_payment()? .message_channel::(Ok(Ack {})); + net.enable_network_for(appkey_prov.identity); // We expect that AcceptInvoice will be delivered within 4s. let sync = sync_channel @@ -154,6 +155,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .await .unwrap() .unwrap(); + println!("{:?}", sync); assert_eq!(sync.invoice_accepts.len(), 1); assert_eq!(sync.invoice_accepts[0].invoice_id, invoice.invoice_id); assert_eq!( @@ -161,5 +163,44 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> invoice.amount ); + log::info!("================== Scenario 3 =================="); + // Send Invoice to Requestor node. + // Requestor accepts Invoice, but network is broken before he sends payment confirmation. + // Payment confirmation will be sent later as payment sync. + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + log::info!("Issuing invoice..."); + let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; + payment + .gsb_public_endpoint() + .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) + .await??; + + log::info!( + "Accepting Invoice ({})... with broken network", + invoice.invoice_id + ); + + requestor.get_invoice(&invoice.invoice_id).await.unwrap(); + requestor + .accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id.to_string(), + }, + ) + .await + .unwrap(); + net.break_network_for(appkey_prov.identity); + + // Wait until Invoice will be paid on Requestor side. + let payments = requestor + .wait_for_invoice_payment::(&invoice.invoice_id, Duration::from_secs(5 * 60), None) + .await?; + assert_eq!(payments.len(), 1); + Ok(()) } diff --git a/test-utils/test-framework/framework-mocks/src/net.rs b/test-utils/test-framework/framework-mocks/src/net.rs index 6ce1581253..3c99717652 100644 --- a/test-utils/test-framework/framework-mocks/src/net.rs +++ b/test-utils/test-framework/framework-mocks/src/net.rs @@ -1,4 +1,5 @@ use anyhow::{bail, Result}; +use futures::TryFutureExt; use std::collections::hash_map::Entry; use std::collections::HashMap; use std::rc::Rc; @@ -238,6 +239,7 @@ impl MockNet { let data = Vec::from(msg); let caller = caller.to_string(); let addr = addr.to_string(); + let addr_ = addr.to_string(); let resolver_ = resolver.clone(); async move { @@ -253,7 +255,7 @@ impl MockNet { "[MockNet] Sending message from [{from}], to: [{to}], address [{translated}]." ); local_bus::send(&translated, &from.to_string(), &data).await - } + }.inspect_err(move |e| log::warn!("[MockNet] Sending message [{addr_}]: {e}")) }, // TODO: Implement stream handler (), diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index 78f4539cff..dc5552a8f9 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -135,6 +135,16 @@ pub trait PaymentRestExt { where Tz: TimeZone, Tz::Offset: Display; + + async fn wait_for_invoice_payment( + &self, + invoice_id: &str, + timeout: Duration, + after_timestamp: Option>, + ) -> anyhow::Result> + where + Tz: TimeZone, + Tz::Offset: Display; } #[async_trait::async_trait(?Send)] @@ -164,4 +174,28 @@ impl PaymentRestExt for PaymentApi { } Err(anyhow!("Timeout {timeout:?} waiting for payments.")) } + + async fn wait_for_invoice_payment( + &self, + invoice_id: &str, + timeout: Duration, + after_timestamp: Option>, + ) -> anyhow::Result> + where + Tz: TimeZone, + Tz::Offset: Display, + { + let start = Utc::now(); + while start + timeout > Utc::now() { + let payments = self + .get_payments_for_invoice(invoice_id, after_timestamp.clone(), None) + .await?; + + if !payments.is_empty() { + return Ok(payments); + } + tokio::time::sleep(Duration::from_millis(300)).await; + } + Err(anyhow!("Timeout {timeout:?} waiting for payments.")) + } } From 9a3cef297150ce9f1e021e73ababb9b907c6eee3 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Tue, 3 Sep 2024 18:21:00 +0200 Subject: [PATCH 092/125] Fix payment sync identites loop. Fix test --- core/payment/src/dao/sync_notifs.rs | 1 + core/payment/src/payment_sync.rs | 128 +++++++++--------- core/payment/tests/test_payment_sync.rs | 38 ++++-- .../tests/tutorial_how_to_use_module_tests.rs | 2 +- .../src/payment/fake_payment.rs | 9 +- 5 files changed, 94 insertions(+), 84 deletions(-) diff --git a/core/payment/src/dao/sync_notifs.rs b/core/payment/src/dao/sync_notifs.rs index eb356045ba..5a8805e94d 100644 --- a/core/payment/src/dao/sync_notifs.rs +++ b/core/payment/src/dao/sync_notifs.rs @@ -1,6 +1,7 @@ use crate::error::DbResult; use crate::models::sync_notifs::{ReadObj, WriteObj}; use crate::schema::pay_sync_needed_notifs::dsl; + use chrono::NaiveDateTime; use diesel::{self, QueryDsl, RunQueryDsl}; diff --git a/core/payment/src/payment_sync.rs b/core/payment/src/payment_sync.rs index a4af2b4a37..788a93a38a 100644 --- a/core/payment/src/payment_sync.rs +++ b/core/payment/src/payment_sync.rs @@ -177,56 +177,46 @@ async fn mark_all_sent(db: &DbExecutor, owner_id: NodeId, msg: PaymentSync) -> a Ok(()) } -async fn send_sync_notifs_for_identity( - identity: NodeId, +async fn send_sync_notifs_for_peer( + peer: NodeId, db: &DbExecutor, config: &Config, cutoff: &DateTime, -) -> anyhow::Result> { - log::debug!("Processing PaymentSync from identity [{identity}]."); +) -> anyhow::Result<()> { + log::debug!("Processing PaymentSync for peer [{peer}]."); + let dao: SyncNotifsDao = db.as_dao(); - let backoff_config = &config.sync_notif_backoff; + let mut all_delivered = true; + let identities = typed::service(identity::BUS_ID) + .call(ya_core_model::identity::List {}) + .await??; - let exp_backoff = |n| { - let secs = backoff_config.initial_delay * backoff_config.exponent.powi(n) as u32; - let capped: Duration = if let Some(cap) = backoff_config.cap { - ::std::cmp::min(cap, secs) - } else { - secs - }; - capped - }; - let peers_to_notify = dao - .list() - .await? - .into_iter() - .filter(|entry| { - let next_deadline = entry.last_ping + exp_backoff(entry.retries as _); - next_deadline.and_utc() <= *cutoff && entry.retries <= backoff_config.max_retries as i32 - }) - .map(|entry| entry.id) - .collect::>(); + for identity in identities { + let owner = identity.node_id; + if identity.is_locked { + log::info!("Skipping PaymentSync for [{owner}] since identity is locked."); + continue; + } - for peer in peers_to_notify { - let (msg, msg_with_bytes) = payment_sync(db, identity, peer).await?; + let (msg, msg_with_bytes) = payment_sync(db, owner, peer).await?; - log::debug!("Sending PaymentSync as [{identity}] to [{peer}]."); - let mut result = ya_net::from(identity) + log::debug!("Sending PaymentSync as [{owner}] to [{peer}]."); + let mut result = ya_net::from(owner) .to(peer) .service(ya_core_model::payment::public::BUS_ID) .call(msg_with_bytes.clone()) .await; - log::debug!("Sending PaymentSync as [{identity}] to [{peer}] result: {result:?}"); + log::debug!("Sending PaymentSync as [{owner}] to [{peer}] result: {result:?}"); // Centralnet and hybridnet return different errors when the endpoint is not supported, so // we have to resort to checking error message. // This message will be sent even if the node can handle PaymentSyncWithBytes but is not // connected at all, but there is no standard way to differentiate between these cases. if matches!(&result, Err(e) if e.to_string().contains("endpoint address not found")) { - log::debug!("Sending PaymentSync as [{identity}] to [{peer}]: PaymentSyncWithBytes endpoint not found, falling back to PaymentSync."); - result = ya_net::from(identity) + log::debug!("Sending PaymentSync as [{owner}] to [{peer}]: PaymentSyncWithBytes endpoint not found, falling back to PaymentSync."); + result = ya_net::from(owner) .to(peer) .service(ya_core_model::payment::public::BUS_ID) .call(msg.clone()) @@ -234,20 +224,57 @@ async fn send_sync_notifs_for_identity( } if matches!(&result, Ok(Ok(_))) { - log::debug!("Delivered PaymentSync to [{peer}] as [{identity}]."); - mark_all_sent(db, identity, msg).await?; - dao.drop(peer).await?; + log::debug!("Delivered PaymentSync to [{peer}] as [{owner}]."); + mark_all_sent(db, owner, msg).await?; } else { + all_delivered = false; let err = match result { Err(x) => x.to_string(), Ok(Err(x)) => x.to_string(), Ok(Ok(_)) => unreachable!(), }; - log::debug!("Couldn't deliver PaymentSync to [{peer}] as [{identity}]: {err}"); + log::debug!("Couldn't deliver PaymentSync to [{peer}] as [{owner}]: {err}"); dao.increment_retry(peer, cutoff.naive_utc()).await?; } } + if all_delivered { + dao.drop(peer).await?; + } + + Ok(()) +} + +async fn send_sync_notifs(db: &DbExecutor, config: &Config) -> anyhow::Result> { + let cutoff = Utc::now(); + let dao: SyncNotifsDao = db.as_dao(); + + let backoff_config = &config.sync_notif_backoff; + + let exp_backoff = |n| { + let secs = backoff_config.initial_delay * backoff_config.exponent.powi(n) as u32; + let capped: Duration = if let Some(cap) = backoff_config.cap { + ::std::cmp::min(cap, secs) + } else { + secs + }; + capped + }; + let peers_to_notify = dao + .list() + .await? + .into_iter() + .filter(|entry| { + let next_deadline = entry.last_ping + exp_backoff(entry.retries as _); + next_deadline.and_utc() <= cutoff && entry.retries <= backoff_config.max_retries as i32 + }) + .map(|entry| entry.id) + .collect::>(); + + for peer in peers_to_notify { + send_sync_notifs_for_peer(peer, db, config, &cutoff).await?; + } + // Next sleep duration is calculated after all events were updated to pick up entries // that were not delivered in current run. let next_sleep_duration = dao @@ -258,7 +285,7 @@ async fn send_sync_notifs_for_identity( let next_deadline = entry.last_ping + exp_backoff(entry.retries as _); next_deadline.and_utc() }) - .filter(|deadline| deadline > cutoff) + .filter(|deadline| deadline > &cutoff) .min() .map(|ts| ts - cutoff) .and_then(|dur| dur.to_std().ok()); @@ -266,35 +293,6 @@ async fn send_sync_notifs_for_identity( Ok(next_sleep_duration) } -async fn send_sync_notifs(db: &DbExecutor, config: &Config) -> anyhow::Result> { - let cutoff = Utc::now(); - - let identities = typed::service(identity::BUS_ID) - .call(ya_core_model::identity::List {}) - .await??; - - let mut next_sleep_duration: Option = None; - for identity in identities { - if identity.is_locked { - continue; - } - let sleep_duration = - send_sync_notifs_for_identity(identity.node_id, db, config, &cutoff).await?; - next_sleep_duration = match sleep_duration { - None => next_sleep_duration, - Some(duration) => { - let result_duration = match next_sleep_duration { - None => duration, - Some(last_duration) => ::std::cmp::min(duration, last_duration), - }; - Some(result_duration) - } - }; - } - - Ok(next_sleep_duration) -} - lazy_static::lazy_static! { pub static ref SYNC_NOTIFS_NOTIFY: Notify = Notify::new(); } diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index f776fae5f5..822c6ab9b8 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -103,7 +103,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .await .unwrap(); - let accept = channel.recv().await.unwrap(); + let (_from, accept) = channel.recv().await.unwrap(); assert_eq!(accept.invoice_id, invoice.invoice_id); assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); @@ -148,20 +148,28 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .message_channel::(Ok(Ack {})); net.enable_network_for(appkey_prov.identity); - // We expect that AcceptInvoice will be delivered within 4s. - let sync = sync_channel - .recv() - .timeout(Some(Duration::from_secs(4))) - .await - .unwrap() - .unwrap(); - println!("{:?}", sync); - assert_eq!(sync.invoice_accepts.len(), 1); - assert_eq!(sync.invoice_accepts[0].invoice_id, invoice.invoice_id); - assert_eq!( - sync.invoice_accepts[0].acceptance.total_amount_accepted, - invoice.amount - ); + // We expect that PaymentSync will be delivered within 4s. + // Looping because sync is sent from multiple identities on Requestor Node. + loop { + let (from, sync) = sync_channel + .recv() + .timeout(Some(Duration::from_secs(4))) + .await + .unwrap() + .unwrap(); + + if from != appkey_req.identity { + continue; + } + + assert_eq!(sync.invoice_accepts.len(), 1); + assert_eq!(sync.invoice_accepts[0].invoice_id, invoice.invoice_id); + assert_eq!( + sync.invoice_accepts[0].acceptance.total_amount_accepted, + invoice.amount + ); + break; + } log::info!("================== Scenario 3 =================="); // Send Invoice to Requestor node. diff --git a/core/payment/tests/tutorial_how_to_use_module_tests.rs b/core/payment/tests/tutorial_how_to_use_module_tests.rs index 02d4b9e38f..66ead162a1 100644 --- a/core/payment/tests/tutorial_how_to_use_module_tests.rs +++ b/core/payment/tests/tutorial_how_to_use_module_tests.rs @@ -181,7 +181,7 @@ async fn tutorial_how_to_use_module_tests(ctx: &mut DroppableTestContext) -> any // Waiting for any message sent to `node3`. // We expect that `AcceptInvoice` message will be received after previous REST api call was made. - let accept = channel.recv().await.unwrap(); + let (_from, accept) = channel.recv().await.unwrap(); assert_eq!(accept.invoice_id, invoice.invoice_id); Ok(()) } diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs index 697fcc9b93..86a0f29a5f 100644 --- a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -5,6 +5,7 @@ use bigdecimal::BigDecimal; use chrono::{Duration, Utc}; use std::fmt::Display; use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::mpsc; use uuid::Uuid; @@ -13,6 +14,7 @@ use ya_agreement_utils::AgreementView; use ya_client_model::market::Agreement; use ya_client_model::payment::allocation::PaymentPlatformEnum; use ya_client_model::payment::{DocumentStatus, Invoice, NewAllocation}; +use ya_client_model::NodeId; use ya_core_model as model; use ya_core_model::bus::GsbBindPoints; use ya_core_model::payment::public; @@ -91,7 +93,7 @@ impl FakePayment { pub fn message_channel( &self, result: Result, - ) -> mpsc::UnboundedReceiver + ) -> mpsc::UnboundedReceiver<(NodeId, T)> where T: RpcMessage, T::Item: Clone, @@ -99,11 +101,12 @@ impl FakePayment { { let (sender, receiver) = mpsc::unbounded_channel(); self.override_gsb_public() - .bind(move |_db: DbExecutor, _sender_id: String, msg: T| { + .bind(move |_db: DbExecutor, sender_id: String, msg: T| { let result = result.clone(); let sender = sender.clone(); async move { - let _ = sender.send(msg).map_err(|_e| { + let id = NodeId::from_str(&sender_id).unwrap(); + let _ = sender.send((id, msg)).map_err(|_e| { log::error!( "[FakePayment] Unable to send message '{}' to channel.", T::ID From fba48bc0137d163aac5c413e9cc3d34df119485d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 4 Sep 2024 12:25:07 +0200 Subject: [PATCH 093/125] Send payment confirmation per owner id --- core/payment/src/dao/payment.rs | 7 ++- core/payment/src/payment_sync.rs | 4 +- core/payment/tests/test_payment_sync.rs | 56 +++++++++++-------- .../framework-mocks/src/payment.rs | 24 +++++++- 4 files changed, 65 insertions(+), 26 deletions(-) diff --git a/core/payment/src/dao/payment.rs b/core/payment/src/dao/payment.rs index 9c336d1d40..7fffc9c786 100644 --- a/core/payment/src/dao/payment.rs +++ b/core/payment/src/dao/payment.rs @@ -351,10 +351,15 @@ impl<'c> PaymentDao<'c> { .await } - pub async fn list_unsent(&self, peer_id: Option) -> DbResult> { + pub async fn list_unsent( + &self, + owner: NodeId, + peer_id: Option, + ) -> DbResult> { readonly_transaction(self.pool, "payment_dao_list_unsent", move |conn| { let mut query = dsl::pay_payment .filter(dsl::send_payment.eq(true)) + .filter(dsl::owner_id.eq(&owner)) .into_boxed(); if let Some(peer_id) = peer_id { query = query.filter(dsl::peer_id.eq(&peer_id)); diff --git a/core/payment/src/payment_sync.rs b/core/payment/src/payment_sync.rs index 788a93a38a..4100a4bd63 100644 --- a/core/payment/src/payment_sync.rs +++ b/core/payment/src/payment_sync.rs @@ -43,7 +43,7 @@ async fn payment_sync( let mut payments = Vec::default(); let mut payments_canonicalized = Vec::default(); - for payment in payment_dao.list_unsent(Some(peer_id)).await? { + for payment in payment_dao.list_unsent(owner, Some(peer_id)).await? { let platform_components = payment.payment_platform.split('-').collect::>(); let driver = &platform_components[0]; let bus_id = driver_bus_id(driver); @@ -132,7 +132,7 @@ async fn mark_all_sent(db: &DbExecutor, owner_id: NodeId, msg: PaymentSync) -> a for payment_send in msg.payments { log::info!( - "Delivered payment [{}] to [{}]", + "Delivered Payment confirmation [{}] to [{}]", payment_send.payment.payment_id, payment_send.payment.payee_id ); diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index 822c6ab9b8..f862cd0dda 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -5,7 +5,6 @@ use std::time::Duration; use test_context::test_context; use tokio::sync::mpsc::error::TryRecvError; -use ya_client_model::payment::Acceptance; use ya_core_model::payment::public::{AcceptInvoice, Ack, PaymentSyncWithBytes, SendInvoice}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; @@ -93,13 +92,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> log::info!("Accepting Invoice ({})...", invoice.invoice_id); requestor.get_invoice(&invoice.invoice_id).await.unwrap(); requestor - .accept_invoice( - &invoice.invoice_id, - &Acceptance { - total_amount_accepted: invoice.amount.clone(), - allocation_id: allocation.allocation_id.to_string(), - }, - ) + .simple_accept_invoice(&invoice, &allocation) .await .unwrap(); @@ -130,13 +123,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> requestor.get_invoice(&invoice.invoice_id).await.unwrap(); requestor - .accept_invoice( - &invoice.invoice_id, - &Acceptance { - total_amount_accepted: invoice.amount.clone(), - allocation_id: allocation.allocation_id.to_string(), - }, - ) + .simple_accept_invoice(&invoice, &allocation) .await .unwrap(); @@ -193,13 +180,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> requestor.get_invoice(&invoice.invoice_id).await.unwrap(); requestor - .accept_invoice( - &invoice.invoice_id, - &Acceptance { - total_amount_accepted: invoice.amount.clone(), - allocation_id: allocation.allocation_id.to_string(), - }, - ) + .simple_accept_invoice(&invoice, &allocation) .await .unwrap(); net.break_network_for(appkey_prov.identity); @@ -210,5 +191,36 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .await?; assert_eq!(payments.len(), 1); + net.enable_network_for(appkey_prov.identity); + // We expect that PaymentSync will be delivered within 4s. + // Looping because sync is sent from multiple identities on Requestor Node. + loop { + let (from, sync) = sync_channel + .recv() + .timeout(Some(Duration::from_secs(4))) + .await + .unwrap() + .unwrap(); + + if from != appkey_req.identity { + continue; + } + + assert!(!sync.payments.is_empty()); + let payment = sync + .payments + .iter() + .find(|p| { + p.payment + .agreement_payments + .iter() + .any(|a| a.agreement_id == agreement.agreement_id) + }) + .unwrap(); + + assert_eq!(payment.payment.amount, invoice.amount); + break; + } + Ok(()) } diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index dc5552a8f9..88c8c40cb7 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::time::Duration; use ya_client::payment::PaymentApi; -use ya_client_model::payment::Payment; +use ya_client_model::payment::{Acceptance, Allocation, Invoice, Payment}; use ya_core_model::driver::{driver_bus_id, Fund}; use ya_core_model::payment::local::BUS_ID; use ya_core_model::payment::public; @@ -145,6 +145,12 @@ pub trait PaymentRestExt { where Tz: TimeZone, Tz::Offset: Display; + + async fn simple_accept_invoice( + &self, + invoice: &Invoice, + allocation: &Allocation, + ) -> anyhow::Result<()>; } #[async_trait::async_trait(?Send)] @@ -198,4 +204,20 @@ impl PaymentRestExt for PaymentApi { } Err(anyhow!("Timeout {timeout:?} waiting for payments.")) } + + async fn simple_accept_invoice( + &self, + invoice: &Invoice, + allocation: &Allocation, + ) -> anyhow::Result<()> { + Ok(self + .accept_invoice( + &invoice.invoice_id, + &Acceptance { + total_amount_accepted: invoice.amount.clone(), + allocation_id: allocation.allocation_id.to_string(), + }, + ) + .await?) + } } From d938e7f2c9985df9e92f9886ab5c33ae548f09be Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 4 Sep 2024 15:05:07 +0200 Subject: [PATCH 094/125] Test payment sync with fallback mechanism --- core/payment/tests/test_payment_sync.rs | 215 +++++++++++++++++- .../src/payment/fake_payment.rs | 12 + 2 files changed, 216 insertions(+), 11 deletions(-) diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index f862cd0dda..4ece408908 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -5,7 +5,9 @@ use std::time::Duration; use test_context::test_context; use tokio::sync::mpsc::error::TryRecvError; -use ya_core_model::payment::public::{AcceptInvoice, Ack, PaymentSyncWithBytes, SendInvoice}; +use ya_core_model::payment::public::{ + AcceptInvoice, Ack, PaymentSync, PaymentSyncWithBytes, SendInvoice, SendPayment, +}; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; @@ -21,10 +23,10 @@ use ya_service_bus::RpcEndpoint; #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("test_payment_sync")?; - let dir = dir.into_path(); + let dir = dir.path(); let net = MockNet::new().bind(); @@ -76,6 +78,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> log::info!("Allocation created. ({})", allocation.allocation_id); log::info!("================== Scenario 1 =================="); + log::info!("Scenario: Invoice immediate acceptance"); // Send Invoice to Requestor node. // Requestor is able to immediately accept. log::info!("Issuing invoice..."); @@ -101,6 +104,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); log::info!("================== Scenario 2 =================="); + log::info!("Scenario: Invoice delayed acceptance"); // Send Invoice to Requestor node. // Requestor attempt to accept Invoice but network is broken. // Acceptance will be sent later as payment sync. @@ -115,10 +119,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) .await??; - log::info!( - "Accepting Invoice ({})... with broken network", - invoice.invoice_id - ); + log::info!("Accepting Invoice ({})", invoice.invoice_id); net.break_network_for(appkey_prov.identity); requestor.get_invoice(&invoice.invoice_id).await.unwrap(); @@ -159,6 +160,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> } log::info!("================== Scenario 3 =================="); + log::info!("Scenario: Payment delayed delivery"); // Send Invoice to Requestor node. // Requestor accepts Invoice, but network is broken before he sends payment confirmation. // Payment confirmation will be sent later as payment sync. @@ -173,16 +175,207 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) .await??; - log::info!( - "Accepting Invoice ({})... with broken network", - invoice.invoice_id - ); + log::info!("Accepting Invoice ({})", invoice.invoice_id); + + requestor.get_invoice(&invoice.invoice_id).await.unwrap(); + requestor + .simple_accept_invoice(&invoice, &allocation) + .await + .unwrap(); + net.break_network_for(appkey_prov.identity); + + // Wait until Invoice will be paid on Requestor side. + let payments = requestor + .wait_for_invoice_payment::(&invoice.invoice_id, Duration::from_secs(5 * 60), None) + .await?; + assert_eq!(payments.len(), 1); + + net.enable_network_for(appkey_prov.identity); + // We expect that PaymentSync will be delivered within 4s. + // Looping because sync is sent from multiple identities on Requestor Node. + loop { + let (from, sync) = sync_channel + .recv() + .timeout(Some(Duration::from_secs(4))) + .await + .unwrap() + .unwrap(); + + if from != appkey_req.identity { + continue; + } + + assert!(!sync.payments.is_empty()); + let payment = sync + .payments + .iter() + .find(|p| { + p.payment + .agreement_payments + .iter() + .any(|a| a.agreement_id == agreement.agreement_id) + }) + .unwrap(); + + assert_eq!(payment.payment.amount, invoice.amount); + break; + } + + Ok(()) +} + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_payment_sync_fallback")?; + let dir = dir.into_path(); + + let net = MockNet::new().bind(); + + // Notifies will be sent in regular intervals, with values more appropriate for tests + // taking less time to trigger. + let mut config = ya_payment::config::Config::from_env()?; + config.sync_notif_backoff.run_sync_job = true; + config.sync_notif_backoff.initial_delay = Duration::from_secs(2); + config.sync_notif_backoff.cap = Some(Duration::from_secs(2)); + + let node1 = MockNode::new(net.clone(), "node-1", &dir) + .with_identity() + .with_payment(Some(config)) + .with_fake_market(); + node1.bind_gsb().await?; + node1.start_server(ctx).await?; + + let appkey_req = node1 + .get_identity()? + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; + + node1 + .get_payment()? + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) + .await?; + + let node2 = MockNode::new(net.clone(), "node-2", dir) + .with_prefixed_gsb() + .with_identity() + .with_fake_payment(); + node2.bind_gsb().await?; + + let appkey_prov = node2 + .get_identity()? + .create_identity_key("provider") + .await?; + + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + let payment = node1.get_payment()?; + let requestor = node1.rest_payments(&appkey_req.key)?; + + log::info!("Creating allocation..."); + let new_allocation = FakePayment::default_allocation(&agreement, BigDecimal::from(10u64))?; + let allocation = requestor.create_allocation(&new_allocation).await?; + log::info!("Allocation created. ({})", allocation.allocation_id); + + let fake_payment = node2.get_fake_payment()?; + fake_payment.unbind_public().await; + let mut channel = fake_payment.message_channel::(Ok(Ack {})); + let mut sync_channel = fake_payment.message_channel::(Ok(Ack {})); + let mut payment_channel = fake_payment.message_channel::(Ok(Ack {})); + + log::info!("================== Scenario 1 =================="); + log::info!("Scenario: Payment immediate delivery with fallback"); + // Send Invoice to Requestor node. + // Requestor accepts Invoice and waits for payment confirmation. + // Payment will be sent immediately, but PaymentSyncWithBytes has no handler on Provider, so fallback + // mechanism should be used. + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + log::info!("Issuing invoice..."); + let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; + payment + .gsb_public_endpoint() + .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) + .await??; + + log::info!("Accepting Invoice ({})", invoice.invoice_id); + requestor.get_invoice(&invoice.invoice_id).await.unwrap(); + requestor + .simple_accept_invoice(&invoice, &allocation) + .await + .unwrap(); + // Check if AcceptInvoice was delivered. + let (_from, accept) = channel.recv().await.unwrap(); + assert_eq!(accept.invoice_id, invoice.invoice_id); + assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); + + // Wait until Invoice will be paid on Requestor side. + let payments = requestor + .wait_for_invoice_payment::(&invoice.invoice_id, Duration::from_secs(5 * 60), None) + .await?; + assert_eq!(payments.len(), 1); + + loop { + let (from, confirmation) = payment_channel + .recv() + .timeout(Some(Duration::from_secs(4))) + .await + .unwrap() + .unwrap(); + + if from != appkey_req.identity { + continue; + } + + assert!(confirmation + .payment + .agreement_payments + .iter() + .any(|a| a.agreement_id == agreement.agreement_id)); + + assert_eq!(confirmation.payment.amount, invoice.amount); + break; + } + + log::info!("================== Scenario 2 =================="); + log::info!("Scenario: Payment delayed delivery with fallback"); + // Send Invoice to Requestor node. + // Requestor accepts Invoice and waits for payment confirmation. + // In the meantime network is broken, so confirmation can't be immediately delivered + // Payment will be sent later, but PaymentSyncWithBytes has no handler on Provider, so fallback + // mechanism should be used. + + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + log::info!("Issuing invoice..."); + let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; + payment + .gsb_public_endpoint() + .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) + .await??; + + log::info!("Accepting Invoice ({})", invoice.invoice_id); requestor.get_invoice(&invoice.invoice_id).await.unwrap(); requestor .simple_accept_invoice(&invoice, &allocation) .await .unwrap(); + + // Check if AcceptInvoice was delivered. + let (_from, accept) = channel.recv().await.unwrap(); + assert_eq!(accept.invoice_id, invoice.invoice_id); + assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); + net.break_network_for(appkey_prov.identity); // Wait until Invoice will be paid on Requestor side. diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs index 86a0f29a5f..b13b246013 100644 --- a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -27,6 +27,7 @@ use ya_core_model::payment::public::{ use ya_payment::migrations; use ya_payment::processor::PaymentProcessor; use ya_persistence::executor::DbExecutor; +use ya_service_bus::typed as bus; use ya_service_bus::typed::{Endpoint, ServiceBinder}; use ya_service_bus::RpcMessage; @@ -126,6 +127,17 @@ impl FakePayment { ServiceBinder::new(self.gsb.local_addr(), &self.db, ()) } + /// Unbinds GSB public endpoint. + /// TODO: it would be nice to be able to unbind each message separately, + /// but GSB doesn't allow this; it can only unbind whole GSB prefix. + pub async fn unbind_public(&self) { + bus::unbind(self.gsb.public_addr()).await; + } + + pub async fn unbind_local(&self) { + bus::unbind(self.gsb.local_addr()).await; + } + pub fn gsb_local_endpoint(&self) -> Endpoint { self.gsb.local() } From 88a39f8b749cfb821ac52d03a4433ec63a605d92 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 4 Sep 2024 15:51:53 +0200 Subject: [PATCH 095/125] Test delayed invoice acceptance; Send acceptance event if status is Settled --- core/payment/src/dao/invoice.rs | 5 ++++- core/payment/tests/test_payment_sync.rs | 29 ++++++++++++++++--------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/core/payment/src/dao/invoice.rs b/core/payment/src/dao/invoice.rs index a02750ba7c..adc2be6225 100644 --- a/core/payment/src/dao/invoice.rs +++ b/core/payment/src/dao/invoice.rs @@ -373,7 +373,10 @@ impl<'c> InvoiceDao<'c> { let invoices: Vec = query!() .filter(dsl::owner_id.eq(owner_id)) .filter(dsl::send_accept.eq(true)) - .filter(dsl::status.eq(DocumentStatus::Accepted.to_string())) + .filter(dsl::status.eq_any([ + DocumentStatus::Accepted.to_string(), + DocumentStatus::Settled.to_string(), + ])) .filter(agreement_dsl::peer_id.eq(peer_id)) .load(conn)?; diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index 4ece408908..64a60d16a9 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -37,7 +37,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> config.sync_notif_backoff.initial_delay = Duration::from_secs(2); config.sync_notif_backoff.cap = Some(Duration::from_secs(2)); - let node1 = MockNode::new(net.clone(), "node-1", &dir) + let node1 = MockNode::new(net.clone(), "node-1", dir) .with_identity() .with_payment(Some(config)) .with_fake_market(); @@ -347,6 +347,7 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R log::info!("================== Scenario 2 =================="); log::info!("Scenario: Payment delayed delivery with fallback"); + log::info!("Scenario: Invoice acceptance delayed delivery"); // Send Invoice to Requestor node. // Requestor accepts Invoice and waits for payment confirmation. // In the meantime network is broken, so confirmation can't be immediately delivered @@ -364,20 +365,19 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) .await??; - log::info!("Accepting Invoice ({})", invoice.invoice_id); + // Break network to enforce Invoice acceptance delayed delivery. + net.break_network_for(appkey_prov.identity); + + log::info!( + "Accepting Invoice ({})... with broken network", + invoice.invoice_id + ); requestor.get_invoice(&invoice.invoice_id).await.unwrap(); requestor .simple_accept_invoice(&invoice, &allocation) .await .unwrap(); - // Check if AcceptInvoice was delivered. - let (_from, accept) = channel.recv().await.unwrap(); - assert_eq!(accept.invoice_id, invoice.invoice_id); - assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); - - net.break_network_for(appkey_prov.identity); - // Wait until Invoice will be paid on Requestor side. let payments = requestor .wait_for_invoice_payment::(&invoice.invoice_id, Duration::from_secs(5 * 60), None) @@ -395,6 +395,7 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R .unwrap() .unwrap(); + println!("{:?}", sync); if from != appkey_req.identity { continue; } @@ -410,8 +411,16 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R .any(|a| a.agreement_id == agreement.agreement_id) }) .unwrap(); - assert_eq!(payment.payment.amount, invoice.amount); + + // Check if AcceptInvoice was delivered. + assert!(!sync.invoice_accepts.is_empty()); + assert_eq!(sync.invoice_accepts[0].invoice_id, invoice.invoice_id); + assert_eq!( + sync.invoice_accepts[0].acceptance.total_amount_accepted, + invoice.amount + ); + break; } From 0392764a96ef78a561f63d57394116464f7f3a1d Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 4 Sep 2024 16:00:03 +0200 Subject: [PATCH 096/125] Payment confirmation (not sync) checks if endpoint is avaiable the same way as payment sync --- core/payment/src/processor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/payment/src/processor.rs b/core/payment/src/processor.rs index 48c7072347..aa753446ec 100644 --- a/core/payment/src/processor.rs +++ b/core/payment/src/processor.rs @@ -569,7 +569,7 @@ impl PaymentProcessor { .call(msg) .map(|res| match res { Ok(Ok(_)) => Ok(()), - Err(ya_service_bus::Error::GsbBadRequest(_)) => { + Err(e) if e.to_string().contains("endpoint address not found") => { Err(PaymentSendToGsbError::NotSupported) } Err(err) => { From d66b31e801d1d26616b616450554850b2c95bcef Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Wed, 4 Sep 2024 17:11:26 +0200 Subject: [PATCH 097/125] Test delayed DebitNotes acceptance --- core/payment/tests/test_payment_sync.rs | 62 ++++++++++++++----- .../framework-mocks/src/payment.rs | 24 ++++++- .../src/payment/fake_payment.rs | 29 ++++++++- 3 files changed, 98 insertions(+), 17 deletions(-) diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index 64a60d16a9..d35ca8599c 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -6,7 +6,8 @@ use test_context::test_context; use tokio::sync::mpsc::error::TryRecvError; use ya_core_model::payment::public::{ - AcceptInvoice, Ack, PaymentSync, PaymentSyncWithBytes, SendInvoice, SendPayment, + AcceptDebitNote, AcceptInvoice, Ack, PaymentSync, PaymentSyncWithBytes, SendDebitNote, + SendInvoice, SendPayment, }; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; @@ -40,7 +41,8 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> let node1 = MockNode::new(net.clone(), "node-1", dir) .with_identity() .with_payment(Some(config)) - .with_fake_market(); + .with_fake_market() + .with_fake_activity(); node1.bind_gsb().await?; node1.start_server(ctx).await?; @@ -88,9 +90,9 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) .await??; - let mut channel = node2 - .get_fake_payment()? - .message_channel::(Ok(Ack {})); + let fake_payment = node2.get_fake_payment()?; + let mut channel = fake_payment.message_channel::(Ok(Ack {})); + let mut debit_channel = fake_payment.message_channel::(Ok(Ack {})); log::info!("Accepting Invoice ({})...", invoice.invoice_id); requestor.get_invoice(&invoice.invoice_id).await.unwrap(); @@ -104,6 +106,7 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> assert_eq!(accept.acceptance.total_amount_accepted, invoice.amount); log::info!("================== Scenario 2 =================="); + log::info!("Scenario: DebitNote delayed acceptance"); log::info!("Scenario: Invoice delayed acceptance"); // Send Invoice to Requestor node. // Requestor attempt to accept Invoice but network is broken. @@ -112,6 +115,19 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); node1.get_market()?.add_agreement(agreement.clone()).await; + let activity_id = node1 + .get_activity()? + .create_activity(&agreement.agreement_id) + .await; + + log::info!("Sending DebitNote..."); + let debit_note = + FakePayment::fake_debit_note(&agreement, &activity_id, BigDecimal::from_str("0.1")?, None)?; + payment + .gsb_public_endpoint() + .send_as(debit_note.issuer_id, SendDebitNote(debit_note.clone())) + .await??; + log::info!("Issuing invoice..."); let invoice = FakePayment::fake_invoice(&agreement, BigDecimal::from_str("0.2")?)?; payment @@ -119,9 +135,19 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> .send_as(invoice.issuer_id, SendInvoice(invoice.clone())) .await??; - log::info!("Accepting Invoice ({})", invoice.invoice_id); net.break_network_for(appkey_prov.identity); + log::info!("Accepting DebitNote ({})", debit_note.debit_note_id); + requestor + .get_debit_note(&debit_note.debit_note_id) + .await + .unwrap(); + requestor + .simple_accept_debit_note(&debit_note, &allocation) + .await + .unwrap(); + + log::info!("Accepting Invoice ({})", invoice.invoice_id); requestor.get_invoice(&invoice.invoice_id).await.unwrap(); requestor .simple_accept_invoice(&invoice, &allocation) @@ -130,10 +156,9 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> // We expect that AcceptInvoice wasn't delivered. matches!(channel.try_recv().unwrap_err(), TryRecvError::Empty); + matches!(debit_channel.try_recv().unwrap_err(), TryRecvError::Empty); - let mut sync_channel = node2 - .get_fake_payment()? - .message_channel::(Ok(Ack {})); + let mut sync_channel = fake_payment.message_channel::(Ok(Ack {})); net.enable_network_for(appkey_prov.identity); // We expect that PaymentSync will be delivered within 4s. @@ -156,6 +181,16 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> sync.invoice_accepts[0].acceptance.total_amount_accepted, invoice.amount ); + + assert_eq!(sync.debit_note_accepts.len(), 1); + assert_eq!( + sync.debit_note_accepts[0].debit_note_id, + debit_note.debit_note_id + ); + assert_eq!( + sync.debit_note_accepts[0].acceptance.total_amount_accepted, + debit_note.total_amount_due + ); break; } @@ -228,10 +263,10 @@ async fn test_payment_sync(ctx: &mut DroppableTestContext) -> anyhow::Result<()> #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { - enable_logs(true); + enable_logs(false); let dir = temp_dir!("test_payment_sync_fallback")?; - let dir = dir.into_path(); + let dir = dir.path(); let net = MockNet::new().bind(); @@ -242,7 +277,7 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R config.sync_notif_backoff.initial_delay = Duration::from_secs(2); config.sync_notif_backoff.cap = Some(Duration::from_secs(2)); - let node1 = MockNode::new(net.clone(), "node-1", &dir) + let node1 = MockNode::new(net.clone(), "node-1", dir) .with_identity() .with_payment(Some(config)) .with_fake_market(); @@ -350,10 +385,9 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R log::info!("Scenario: Invoice acceptance delayed delivery"); // Send Invoice to Requestor node. // Requestor accepts Invoice and waits for payment confirmation. - // In the meantime network is broken, so confirmation can't be immediately delivered + // In the meantime network is broken, so both confirmations can't be immediately delivered. // Payment will be sent later, but PaymentSyncWithBytes has no handler on Provider, so fallback // mechanism should be used. - let agreement = FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); node1.get_market()?.add_agreement(agreement.clone()).await; diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index 88c8c40cb7..c35d7105a8 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::time::Duration; use ya_client::payment::PaymentApi; -use ya_client_model::payment::{Acceptance, Allocation, Invoice, Payment}; +use ya_client_model::payment::{Acceptance, Allocation, DebitNote, Invoice, Payment}; use ya_core_model::driver::{driver_bus_id, Fund}; use ya_core_model::payment::local::BUS_ID; use ya_core_model::payment::public; @@ -151,6 +151,12 @@ pub trait PaymentRestExt { invoice: &Invoice, allocation: &Allocation, ) -> anyhow::Result<()>; + + async fn simple_accept_debit_note( + &self, + debit_note: &DebitNote, + allocation: &Allocation, + ) -> anyhow::Result<()>; } #[async_trait::async_trait(?Send)] @@ -220,4 +226,20 @@ impl PaymentRestExt for PaymentApi { ) .await?) } + + async fn simple_accept_debit_note( + &self, + debit_note: &DebitNote, + allocation: &Allocation, + ) -> anyhow::Result<()> { + Ok(self + .accept_debit_note( + &debit_note.debit_note_id, + &Acceptance { + total_amount_accepted: debit_note.total_amount_due.clone(), + allocation_id: allocation.allocation_id.to_string(), + }, + ) + .await?) + } } diff --git a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs index b13b246013..0d11508141 100644 --- a/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment/fake_payment.rs @@ -2,7 +2,7 @@ use anyhow::anyhow; use bigdecimal::BigDecimal; -use chrono::{Duration, Utc}; +use chrono::{DateTime, Duration, Utc}; use std::fmt::Display; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -13,7 +13,7 @@ use uuid::Uuid; use ya_agreement_utils::AgreementView; use ya_client_model::market::Agreement; use ya_client_model::payment::allocation::PaymentPlatformEnum; -use ya_client_model::payment::{DocumentStatus, Invoice, NewAllocation}; +use ya_client_model::payment::{DebitNote, DocumentStatus, Invoice, NewAllocation}; use ya_client_model::NodeId; use ya_core_model as model; use ya_core_model::bus::GsbBindPoints; @@ -169,6 +169,31 @@ impl FakePayment { }) } + pub fn fake_debit_note( + agreement: &Agreement, + activity_id: &str, + amount: BigDecimal, + due_date: Option>, + ) -> anyhow::Result { + let platform = Self::platform_from(agreement)?; + Ok(DebitNote { + debit_note_id: Uuid::new_v4().to_string(), + issuer_id: agreement.offer.provider_id, + recipient_id: agreement.demand.requestor_id, + payee_addr: agreement.offer.provider_id.to_string(), + payer_addr: agreement.demand.requestor_id.to_string(), + payment_platform: platform, + previous_debit_note_id: None, + timestamp: Utc::now(), + agreement_id: agreement.agreement_id.to_string(), + activity_id: activity_id.to_string(), + total_amount_due: amount, + usage_counter_vector: None, + payment_due_date: due_date, + status: DocumentStatus::Issued, + }) + } + pub fn default_allocation( agreement: &Agreement, amount: BigDecimal, From d8dff86f41c3dece7fbaed90445c884029dd31cd Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 5 Sep 2024 14:23:11 +0200 Subject: [PATCH 098/125] Fix inability to update ya-client{-model} to newer version dues to conflicting dependencies --- Cargo.lock | 100 ++++++++++-------- Cargo.toml | 8 +- agent/provider/Cargo.toml | 2 +- core/activity/Cargo.toml | 2 +- core/gsb-api/Cargo.toml | 2 +- core/identity/Cargo.toml | 2 +- core/model/Cargo.toml | 2 +- core/net/Cargo.toml | 4 +- core/payment-driver/base/Cargo.toml | 2 +- core/payment-driver/dummy/Cargo.toml | 2 +- core/payment-driver/erc20/Cargo.toml | 2 +- core/payment/Cargo.toml | 2 +- core/persistence/Cargo.toml | 2 +- core/serv-api/web/Cargo.toml | 2 +- core/sgx/Cargo.toml | 4 +- core/version/Cargo.toml | 2 +- core/vpn/Cargo.toml | 2 +- exe-unit/Cargo.toml | 2 +- exe-unit/components/gsb-http-proxy/Cargo.toml | 2 +- exe-unit/components/transfer/Cargo.toml | 2 +- golem_cli/Cargo.toml | 16 +-- .../test-framework/framework-basic/Cargo.toml | 2 +- .../test-framework/framework-mocks/Cargo.toml | 2 +- utils/agreement-utils/Cargo.toml | 2 +- utils/manifest-utils/Cargo.toml | 2 +- 25 files changed, 91 insertions(+), 81 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b49b9084de..38c0556e7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3053,7 +3053,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.6.0", ] [[package]] @@ -3450,7 +3450,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.6", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -4343,12 +4343,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "multimap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" - [[package]] name = "names" version = "0.10.0" @@ -5308,7 +5302,7 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "log", - "multimap 0.8.3", + "multimap", "petgraph", "prost 0.10.4", "prost-types 0.10.1", @@ -5325,9 +5319,9 @@ checksum = "80b776a1b2dc779f5ee0641f8ade0125bc1298dd41a9a0c16d8bd57b42d222b1" dependencies = [ "bytes 1.6.0", "heck 0.5.0", - "itertools 0.12.1", + "itertools 0.10.5", "log", - "multimap 0.10.0", + "multimap", "once_cell", "petgraph", "prettyplease", @@ -5358,7 +5352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19de2de2a00075bf566bee3bd4db014b11587e84184d3f7a791bc17f1a8e9e48" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.60", @@ -8726,7 +8720,7 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-gsb-http-proxy", "ya-net", @@ -8751,13 +8745,13 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1#fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" +source = "git+https://github.com/golemfactory/ya-client.git?rev=258d5e085e591cab33858e8d1b19eff01b64687d#258d5e085e591cab33858e8d1b19eff01b64687d" dependencies = [ "actix-codec", "awc", @@ -8775,13 +8769,30 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1#fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe85a762be0297b9848ad0b7f1c73cb2afe778bb8eb3a952efd032792631a3e0" +dependencies = [ + "bigdecimal 0.2.2", + "chrono", + "derive_more", + "rand 0.8.5", + "serde", + "serde_json", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", +] + +[[package]] +name = "ya-client-model" +version = "0.7.0" +source = "git+https://github.com/golemfactory/ya-client.git?rev=258d5e085e591cab33858e8d1b19eff01b64687d#258d5e085e591cab33858e8d1b19eff01b64687d" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -8828,7 +8839,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", "ya-service-bus", ] @@ -8880,7 +8891,7 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-payment-driver", "ya-persistence", @@ -8925,7 +8936,7 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-payment-driver", "ya-service-api-interfaces", @@ -8979,7 +8990,7 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", "ya-core-model", "ya-counters", @@ -9056,7 +9067,7 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-net", "ya-service-bus", @@ -9102,7 +9113,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", @@ -9146,7 +9157,7 @@ dependencies = [ "thiserror", "tokio", "uuid 1.8.0", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-persistence", "ya-service-api", @@ -9184,7 +9195,7 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-counters", "ya-framework-basic", @@ -9226,7 +9237,7 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-framework-basic", "ya-framework-mocks", @@ -9281,7 +9292,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9418,7 +9429,7 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", @@ -9496,7 +9507,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", @@ -9539,7 +9550,7 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-persistence", "ya-service-bus", @@ -9563,7 +9574,7 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-service-api", "ya-service-api-interfaces", @@ -9631,7 +9642,7 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", "ya-core-model", "ya-file-logging", @@ -9649,7 +9660,7 @@ dependencies = [ [[package]] name = "ya-relay-client" version = "0.6.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "anyhow", "async-trait", @@ -9664,6 +9675,7 @@ dependencies = [ "metrics 0.21.1", "num_cpus", "parking_lot 0.12.3", + "rand 0.8.5", "strum 0.25.0", "strum_macros 0.25.3", "thiserror", @@ -9679,7 +9691,7 @@ dependencies = [ [[package]] name = "ya-relay-core" version = "0.4.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "anyhow", "chrono", @@ -9701,7 +9713,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.7.0", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -9709,7 +9721,7 @@ dependencies = [ [[package]] name = "ya-relay-proto" version = "0.4.3" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "anyhow", "bytes 1.6.0", @@ -9723,7 +9735,7 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489)", ] [[package]] @@ -9749,7 +9761,7 @@ dependencies = [ [[package]] name = "ya-relay-stack" version = "0.5.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "derive_more", "futures 0.3.30", @@ -9763,7 +9775,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489)", ] [[package]] @@ -9779,7 +9791,7 @@ dependencies = [ [[package]] name = "ya-relay-util" version = "0.1.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "bytes 1.6.0", "derive_more", @@ -9966,7 +9978,7 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-service-bus", ] @@ -10055,7 +10067,7 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-exe-unit", "ya-framework-basic", @@ -10218,7 +10230,7 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-core-model", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", @@ -10261,7 +10273,7 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", "ya-core-model", "ya-dummy-driver", diff --git a/Cargo.toml b/Cargo.toml index 5139af7233..056576af79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ ya-fd-metrics = { path = "utils/fd-metrics" } ya-version = "0.2" ya-vpn = "0.2" ya-client = "0.8" -ya-client-model = "0.6" +ya-client-model = "0.7" gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests @@ -251,7 +251,7 @@ url = "2.3.1" regex = "1.10.4" ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "0588dd1af311ae19c621b04cc2a4cfd9c0483252" } +ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489" } ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } ya-utils-futures = { path = "utils/futures" } @@ -287,9 +287,9 @@ ya-service-api-interfaces = { path = "core/serv-api/interfaces" } ya-service-api-web = { path = "core/serv-api/web" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "258d5e085e591cab33858e8d1b19eff01b64687d" } #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "258d5e085e591cab33858e8d1b19eff01b64687d" } #ya-client-model = { path = "../ya-client/model" } golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index 25563c2cc6..24e62796ca 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -16,7 +16,7 @@ path = "src/main.rs" ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } ya-client = { version = "0.8", features = ['cli'] } -ya-client-model = "0.6" +ya-client-model = "0.7" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ['activity', 'payment'] } ya-file-logging = "0.1" diff --git a/core/activity/Cargo.toml b/core/activity/Cargo.toml index d3e0fc8d6f..933b520acc 100644 --- a/core/activity/Cargo.toml +++ b/core/activity/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] ya-core-model = { version = "0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.6", features = ["sgx"] } +ya-client-model = { version = "0.7", features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/core/gsb-api/Cargo.toml b/core/gsb-api/Cargo.toml index 8e8ad341d2..d8dc4d6c7c 100644 --- a/core/gsb-api/Cargo.toml +++ b/core/gsb-api/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.6" +ya-client-model = "0.7" ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index 5d927802f8..e0c992d837 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -10,7 +10,7 @@ framework-test = [] [dependencies] -ya-client-model = { version = "0.6", features = ["with-diesel"] } +ya-client-model = { version = "0.7", features = ["with-diesel"] } ya-core-model = { version = "^0.9", features = ["identity", "appkey"] } ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/core/model/Cargo.toml b/core/model/Cargo.toml index 69aa7d4b0a..8c814f5b73 100644 --- a/core/model/Cargo.toml +++ b/core/model/Cargo.toml @@ -24,7 +24,7 @@ sgx = ['graphene-sgx'] version = [] [dependencies] -ya-client-model = "0.6" +ya-client-model = "0.7" ya-service-bus = { workspace = true } bigdecimal = { version = "0.2", features = ["serde"], optional = true } diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index a8129f15c3..d4d918c65a 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -15,12 +15,10 @@ packet-trace-enable = [ ] [dependencies] -ya-client-model = "0.6" +ya-client-model = "0.7" ya-core-model = { version = "^0.9", features = ["net", "identity"] } ya-relay-client = { workspace = true } -#ya-relay-client = "0.6" -#ya-relay-client = { path = "../../../ya-relay/client" } ya-sb-proto = { workspace = true } ya-sb-util = { workspace = true } diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index 5b43474d1d..5039bc91a7 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -29,7 +29,7 @@ thiserror = "1.0" tokio = { version = "1", features = ["macros"] } ## yagna dependencies -ya-client-model = "0.6" +ya-client-model = "0.7" ya-core-model = { version = "^0.9", features = [ "driver", "identity", diff --git a/core/payment-driver/dummy/Cargo.toml b/core/payment-driver/dummy/Cargo.toml index 5c3b2a1249..4be808ba90 100644 --- a/core/payment-driver/dummy/Cargo.toml +++ b/core/payment-driver/dummy/Cargo.toml @@ -13,7 +13,7 @@ ya-core-model = { version = "^0.9", features = [ "identity", "payment", ] } -ya-client-model = { version = "0.6" } +ya-client-model = { version = "0.7" } ya-payment-driver = "0.3" ya-persistence = "0.3" ya-service-api-interfaces = "0.2" diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index bdcb514db5..749e8a364b 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -44,7 +44,7 @@ rust_decimal = "1" ## yagna dependencies ya-payment-driver = "0.3" ya-core-model = { version = "0.9" } -ya-client-model = "0.6" +ya-client-model = "0.7" ya-service-api-interfaces = "0.2" ya-utils-futures.workspace = true ya-utils-networking = "0.2" diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index ca6354f8a6..40d74b5325 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -10,7 +10,7 @@ framework-test = [] [dependencies] ya-agreement-utils = { workspace = true } -ya-client-model = { version = "0.6", features = ["with-diesel"] } +ya-client-model = { version = "0.7", features = ["with-diesel"] } ya-core-model = { version = "^0.9", features = [ "activity", "driver", diff --git a/core/persistence/Cargo.toml b/core/persistence/Cargo.toml index f376ed4053..3ee997e60e 100644 --- a/core/persistence/Cargo.toml +++ b/core/persistence/Cargo.toml @@ -14,7 +14,7 @@ service = [ ] [dependencies] -ya-client-model = { version = "0.6", features = ["with-diesel"] } +ya-client-model = { version = "0.7", features = ["with-diesel"] } ya-core-model = { version = "0.9" } ya-service-api = { version = "0.1", optional = true } ya-service-api-interfaces = { version = "0.2", optional = true } diff --git a/core/serv-api/web/Cargo.toml b/core/serv-api/web/Cargo.toml index 90117b680c..8eb0a128a2 100644 --- a/core/serv-api/web/Cargo.toml +++ b/core/serv-api/web/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" ya-client = "0.8" ya-core-model = { version = "^0.9", features = ["appkey"] } ya-service-api = "0.1" -ya-service-bus = { workspace = true } +ya-service-bus = { workspace = true } actix-cors = "0.6" actix-service = "2" diff --git a/core/sgx/Cargo.toml b/core/sgx/Cargo.toml index cf01fcf5ec..5853dfe715 100644 --- a/core/sgx/Cargo.toml +++ b/core/sgx/Cargo.toml @@ -7,6 +7,6 @@ edition = "2018" [dependencies] anyhow = "1.0" graphene-sgx = { version = "0.3.3", features = ["ias"] } -ya-client-model = "0.6" +ya-client-model = "0.7" ya-core-model = { version = "^0.9", features = ["sgx"] } -ya-service-bus = { workspace = true } +ya-service-bus = { workspace = true } diff --git a/core/version/Cargo.toml b/core/version/Cargo.toml index 6199451c4a..4045dd1d9d 100644 --- a/core/version/Cargo.toml +++ b/core/version/Cargo.toml @@ -12,7 +12,7 @@ ya-core-model = { version = "^0.9", features = ["version"] } ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" -ya-service-bus = { workspace = true } +ya-service-bus = { workspace = true } actix-web = "4" anyhow = "1.0" diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index 6083b50ef1..0b1e99fe4b 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] ya-core-model = { version = "^0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.6", features = ["sgx"] } +ya-client-model = { version = "0.7", features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index d720f53d2c..38559a79f1 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -29,7 +29,7 @@ framework-test = ["ya-transfer/framework-test"] [dependencies] ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } -ya-client-model = "0.6" +ya-client-model = "0.7" ya-compile-time-utils = "0.2" ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ diff --git a/exe-unit/components/gsb-http-proxy/Cargo.toml b/exe-unit/components/gsb-http-proxy/Cargo.toml index a9b50d0393..b0988da9eb 100644 --- a/exe-unit/components/gsb-http-proxy/Cargo.toml +++ b/exe-unit/components/gsb-http-proxy/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] ya-service-bus = { workspace = true } ya-counters = { path = "../counters" } -ya-client-model = "0.6" +ya-client-model = "0.7" ya-core-model = { version = "^0.9" } thiserror = "1.0" diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 406b375ff6..cf4516c35e 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.6" +ya-client-model = "0.7" ya-core-model = { version = "^0.9" } ya-service-bus = { workspace = true } ya-utils-path = { version = "0.1", path = "../../../utils/path" } diff --git a/golem_cli/Cargo.toml b/golem_cli/Cargo.toml index f869898650..f76545267f 100644 --- a/golem_cli/Cargo.toml +++ b/golem_cli/Cargo.toml @@ -8,18 +8,18 @@ edition = "2018" [dependencies] ya-client = { version = "0.8", features = ['cli'] } ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.9", features=["payment", "version"] } +ya-core-model = { version = "^0.9", features = ["payment", "version"] } ya-provider = "0.3" ya-utils-path = "0.1.0" ya-utils-process = { version = "0.3", features = ["lock"] } -actix-rt="2.7" -ansi_term="0.12.1" +actix-rt = "2.7" +ansi_term = "0.12.1" anyhow = "1.0" bigdecimal = "0.2" byte-unit = "4.0" -chrono = { version = "0.4", features=["serde"] } -crossterm="0.23" +chrono = { version = "0.4", features = ["serde"] } +crossterm = "0.23" directories = "2.0.2" dotenv = "0.15" env_logger = "0.7" @@ -30,7 +30,7 @@ names = "0.10.0" openssl.workspace = true prettytable-rs = "0.10.0" promptly = "0.3.0" -rustyline="6.3.0" +rustyline = "6.3.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" strip-ansi-escapes = "0.1" @@ -41,5 +41,5 @@ tokio = { version = "1", features = ["process", "signal", "time", "io-util", "io url = "2.1" [target.'cfg(target_family = "unix")'.dependencies] -libc="0.2.73" -nix="0.22.0" +libc = "0.2.73" +nix = "0.22.0" diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index 6ad09f8c21..ee22200012 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" bcast-singleton = [] [dependencies] -ya-client-model = "0.6" +ya-client-model = "0.7" ya-client = "0.8" ya-core-model = { version = "^0.9" } ya-service-bus = { workspace = true } diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index 4d4948c39a..ee090b95cd 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] ya-agreement-utils = { workspace = true } -ya-client-model = "0.6" +ya-client-model = "0.7" ya-client = "0.8" ya-core-model = { version = "^0.9" } ya-dummy-driver = "0.3" diff --git a/utils/agreement-utils/Cargo.toml b/utils/agreement-utils/Cargo.toml index c519e45980..ee34b4b4b7 100644 --- a/utils/agreement-utils/Cargo.toml +++ b/utils/agreement-utils/Cargo.toml @@ -13,7 +13,7 @@ keywords = ["golem", "yagna"] default = [] [dependencies] -ya-client-model = "0.6" +ya-client-model = "0.7" chrono = "0.4" regex = "1.5.4" diff --git a/utils/manifest-utils/Cargo.toml b/utils/manifest-utils/Cargo.toml index 69cadef477..9c3551f9da 100644 --- a/utils/manifest-utils/Cargo.toml +++ b/utils/manifest-utils/Cargo.toml @@ -22,7 +22,7 @@ schema = ["schemars"] [dependencies] ya-agreement-utils = { workspace = true } ya-utils-path = "0.1" -ya-client-model = "0.6" +ya-client-model = "0.7" golem-certificate = "0.1.1" regex = "1.5" From bbd7ba201eb58d51e4d82ea1bd85cfb8244c0e4a Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 5 Sep 2024 14:40:07 +0200 Subject: [PATCH 099/125] Update actions/upload-artifact@v1 to v4 --- .github/workflows/binaries-aarch64.yml | 2 +- .github/workflows/integration-test-nightly.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/binaries-aarch64.yml b/.github/workflows/binaries-aarch64.yml index 1c44402ac9..b5a1fbc2f4 100644 --- a/.github/workflows/binaries-aarch64.yml +++ b/.github/workflows/binaries-aarch64.yml @@ -61,7 +61,7 @@ jobs: cp target/aarch64-unknown-linux-musl/release/{yagna,ya-provider,exe-unit,golemsp,gftp} build - name: Upload binaries - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v4 with: name: Yagna linux-aarch64 path: build diff --git a/.github/workflows/integration-test-nightly.yml b/.github/workflows/integration-test-nightly.yml index 12833ebc5c..79d1568d73 100644 --- a/.github/workflows/integration-test-nightly.yml +++ b/.github/workflows/integration-test-nightly.yml @@ -41,7 +41,7 @@ jobs: strategy: matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix-json) }} fail-fast: false - runs-on: [goth2] + runs-on: [ goth2 ] name: Integration Tests (nightly) @ ${{ matrix.branch }} defaults: run: @@ -92,7 +92,7 @@ jobs: run: echo "::set-output name=file_name::$(echo '${{ matrix.branch }}' | sed 's/\//-/g')" - name: Upload test logs - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: goth-logs_${{ steps.branch_as.outputs.file_name }} From c12fee92a0b2c64522082cdf948bf98d37665a21 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 5 Sep 2024 14:38:36 +0200 Subject: [PATCH 100/125] Test payment signature (in middle of the work) --- Cargo.lock | 67 +++++------- Cargo.toml | 2 +- core/payment/tests/test_payment_signature.rs | 101 +++++++++++++++++++ core/payment/tests/test_payment_sync.rs | 2 - 4 files changed, 127 insertions(+), 45 deletions(-) create mode 100644 core/payment/tests/test_payment_signature.rs diff --git a/Cargo.lock b/Cargo.lock index 38c0556e7f..b72f04daad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3038,7 +3038,7 @@ dependencies = [ [[package]] name = "golem-certificate" version = "0.1.1" -source = "git+https://github.com/golemfactory/golem-certificate.git?rev=f2d7514c18fc066e9cfb796090b90f5b27cfe1c6#f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" +source = "git+https://github.com/golemfactory/golem-certificate.git?rev=800d710331cdf3bf82f2be5de8345b90b6982af1#800d710331cdf3bf82f2be5de8345b90b6982af1" dependencies = [ "anyhow", "chrono", @@ -3053,7 +3053,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model 0.6.0", + "ya-client-model", ] [[package]] @@ -8720,7 +8720,7 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-gsb-http-proxy", "ya-net", @@ -8745,7 +8745,7 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model 0.7.0", + "ya-client-model", ] [[package]] @@ -8769,24 +8769,7 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model 0.7.0", -] - -[[package]] -name = "ya-client-model" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe85a762be0297b9848ad0b7f1c73cb2afe778bb8eb3a952efd032792631a3e0" -dependencies = [ - "bigdecimal 0.2.2", - "chrono", - "derive_more", - "rand 0.8.5", - "serde", - "serde_json", - "strum 0.24.1", - "strum_macros 0.24.3", - "thiserror", + "ya-client-model", ] [[package]] @@ -8839,7 +8822,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model 0.7.0", + "ya-client-model", "ya-service-bus", ] @@ -8891,7 +8874,7 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-payment-driver", "ya-persistence", @@ -8936,7 +8919,7 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-payment-driver", "ya-service-api-interfaces", @@ -8990,7 +8973,7 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", "ya-core-model", "ya-counters", @@ -9067,7 +9050,7 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-net", "ya-service-bus", @@ -9113,7 +9096,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", @@ -9157,7 +9140,7 @@ dependencies = [ "thiserror", "tokio", "uuid 1.8.0", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-persistence", "ya-service-api", @@ -9195,7 +9178,7 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-counters", "ya-framework-basic", @@ -9237,7 +9220,7 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-framework-basic", "ya-framework-mocks", @@ -9292,7 +9275,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9429,7 +9412,7 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", @@ -9507,7 +9490,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", @@ -9550,7 +9533,7 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-persistence", "ya-service-bus", @@ -9574,7 +9557,7 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-service-api", "ya-service-api-interfaces", @@ -9642,7 +9625,7 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", "ya-core-model", "ya-file-logging", @@ -9713,7 +9696,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.7.0", + "ya-client-model", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -9978,7 +9961,7 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-service-bus", ] @@ -10067,7 +10050,7 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-exe-unit", "ya-framework-basic", @@ -10230,7 +10213,7 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-core-model", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", @@ -10273,7 +10256,7 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", "ya-core-model", "ya-dummy-driver", diff --git a/Cargo.toml b/Cargo.toml index 056576af79..45accc50a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -291,7 +291,7 @@ ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "258d #ya-client = { path = "../ya-client" } ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "258d5e085e591cab33858e8d1b19eff01b64687d" } #ya-client-model = { path = "../ya-client/model" } -golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } +golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "800d710331cdf3bf82f2be5de8345b90b6982af1" } ## RELAY and networking stack diff --git a/core/payment/tests/test_payment_signature.rs b/core/payment/tests/test_payment_signature.rs new file mode 100644 index 0000000000..329fdfdf44 --- /dev/null +++ b/core/payment/tests/test_payment_signature.rs @@ -0,0 +1,101 @@ +use bigdecimal::BigDecimal; +use chrono::Utc; +use std::time::Duration; +use test_context::test_context; + +use ya_client_model::payment::NewInvoice; +use ya_framework_basic::async_drop::DroppableTestContext; +use ya_framework_basic::log::enable_logs; +use ya_framework_basic::{resource, temp_dir}; +use ya_framework_mocks::market::FakeMarket; +use ya_framework_mocks::net::MockNet; +use ya_framework_mocks::node::MockNode; +use ya_framework_mocks::payment::fake_payment::FakePayment; +use ya_framework_mocks::payment::{Driver, PaymentRestExt}; + +#[cfg_attr(not(feature = "framework-test"), ignore)] +#[test_context(DroppableTestContext)] +#[serial_test::serial] +async fn test_payment_signature(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { + enable_logs(true); + + let dir = temp_dir!("test_payment_signature")?; + let dir = dir.path(); + + let net = MockNet::new().bind(); + let node1 = MockNode::new(net.clone(), "node-1", dir) + .with_identity() + .with_payment(None) + .with_fake_market(); + node1.bind_gsb().await?; + node1.start_server(ctx).await?; + + let identity = node1.get_identity()?; + let appkey_prov = identity.create_identity_key("provider").await?; + let appkey_req = identity + .create_from_private_key(&resource!("ci-requestor-1.key.priv")) + .await?; + + node1 + .get_payment()? + .fund_account(Driver::Erc20, &appkey_req.identity.to_string()) + .await?; + + let requestor = node1.rest_payments(&appkey_req.key)?; + let provider = node1.rest_payments(&appkey_prov.key)?; + + log::info!("Creating mock Agreement..."); + let agreement = + FakeMarket::create_fake_agreement(appkey_req.identity, appkey_prov.identity).unwrap(); + node1.get_market()?.add_agreement(agreement.clone()).await; + + log::info!("Creating allocation..."); + let new_allocation = FakePayment::default_allocation(&agreement, BigDecimal::from(10u64))?; + let allocation = requestor.create_allocation(&new_allocation).await?; + log::info!( + "Allocation created. ({}) Issuing invoice...", + allocation.allocation_id + ); + + let invoice = provider + .issue_invoice(&NewInvoice { + agreement_id: agreement.agreement_id.to_string(), + activity_ids: None, + amount: BigDecimal::from(2u64), + payment_due_date: Utc::now(), + }) + .await?; + + log::info!( + "Invoice issued ({}). Sending invoice...", + invoice.invoice_id + ); + provider.send_invoice(&invoice.invoice_id).await?; + + log::info!( + "Invoice sent. Accepting Invoice ({})...", + invoice.invoice_id + ); + requestor.get_invoice(&invoice.invoice_id).await.unwrap(); + requestor + .simple_accept_invoice(&invoice, &allocation) + .await + .unwrap(); + + // Payments are processed, and we don't want payment confirmation to reach Provider. + // This is hack which will block communication between Requestor and Provider despite them + // using the same node. + // We want to send payment confirmation manually later. This way we will be able to modify + // the message and check more different conditions. + net.break_network_for(appkey_prov.identity); + + let payments = requestor + .wait_for_invoice_payment::(&invoice.invoice_id, Duration::from_secs(5 * 60), None) + .await?; + assert_eq!(payments.len(), 1); + let _payment = requestor + .get_signed_payment(&payments[0].payment_id) + .await?; + + Ok(()) +} diff --git a/core/payment/tests/test_payment_sync.rs b/core/payment/tests/test_payment_sync.rs index d35ca8599c..d836576d0b 100644 --- a/core/payment/tests/test_payment_sync.rs +++ b/core/payment/tests/test_payment_sync.rs @@ -429,7 +429,6 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R .unwrap() .unwrap(); - println!("{:?}", sync); if from != appkey_req.identity { continue; } @@ -457,6 +456,5 @@ async fn test_payment_sync_fallback(ctx: &mut DroppableTestContext) -> anyhow::R break; } - Ok(()) } From e3eca36503f7efa34378899c11ec2f5484f0c366 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 5 Sep 2024 19:04:03 +0200 Subject: [PATCH 101/125] Test payment signatures (different rejection scenarios); Introduce Signable trait --- Cargo.lock | 3 + core/model/src/driver.rs | 6 +- core/payment-driver/base/Cargo.toml | 5 +- core/payment-driver/base/src/driver.rs | 19 ++-- core/payment-driver/base/src/lib.rs | 1 + core/payment-driver/base/src/signable.rs | 67 ++++++++++++ core/payment-driver/base/src/utils.rs | 12 ++- core/payment/Cargo.toml | 1 + core/payment/src/processor.rs | 28 ++--- core/payment/src/service.rs | 17 +-- core/payment/tests/test_payment_signature.rs | 105 ++++++++++++++++++- 11 files changed, 215 insertions(+), 49 deletions(-) create mode 100644 core/payment-driver/base/src/signable.rs diff --git a/Cargo.lock b/Cargo.lock index b72f04daad..04932bbf2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9500,6 +9500,7 @@ dependencies = [ "ya-market", "ya-metrics", "ya-net", + "ya-payment-driver", "ya-persistence", "ya-sb-router", "ya-service-api", @@ -9529,6 +9530,8 @@ dependencies = [ "num-derive", "num-traits", "r2d2", + "serde", + "serde_json", "serde_json_canonicalizer", "sha3 0.9.1", "thiserror", diff --git a/core/model/src/driver.rs b/core/model/src/driver.rs index a207f9cded..77ceef20f1 100644 --- a/core/model/src/driver.rs +++ b/core/model/src/driver.rs @@ -587,15 +587,15 @@ impl RpcMessage for crate::driver::SignPaymentCanonicalized { pub struct VerifySignature { pub payment: Payment, pub signature: Vec, - pub canonicalized: bool, + pub canonical: Option>, } impl VerifySignature { - pub fn new(payment: Payment, signature: Vec, canonicalized: bool) -> Self { + pub fn new(payment: Payment, signature: Vec, canonical: Option>) -> Self { Self { payment, signature, - canonicalized, + canonical, } } } diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index 5039bc91a7..ff3156fd22 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -25,6 +25,9 @@ num-traits = "0.2" num-derive = "0.3" r2d2 = "0.8" sha3 = "0.9" +serde = "1.0" +serde_json = "1.0" +serde_json_canonicalizer = "0.2.0" thiserror = "1.0" tokio = { version = "1", features = ["macros"] } @@ -37,6 +40,6 @@ ya-core-model = { version = "^0.9", features = [ ] } ya-persistence = "0.3" ya-service-bus = { workspace = true } -serde_json_canonicalizer = "0.2.0" + [dev-dependencies] diff --git a/core/payment-driver/base/src/driver.rs b/core/payment-driver/base/src/driver.rs index cd60c88ea6..d18d6eeeb0 100644 --- a/core/payment-driver/base/src/driver.rs +++ b/core/payment-driver/base/src/driver.rs @@ -16,6 +16,7 @@ use crate::model::*; use crate::utils; // Public revealed uses, required to implement this trait +use crate::signable::Signable; pub use async_trait::async_trait; pub use bigdecimal::BigDecimal; pub use ya_client_model::payment::network::Network; @@ -63,35 +64,30 @@ pub trait PaymentDriver { async fn schedule_payment( &self, - caller: String, msg: SchedulePayment, ) -> Result; async fn verify_payment( &self, - caller: String, msg: VerifyPayment, ) -> Result; async fn validate_allocation( &self, - caller: String, msg: ValidateAllocation, ) -> Result; async fn release_deposit( &self, - caller: String, msg: DriverReleaseDeposit, ) -> Result<(), GenericError>; async fn sign_payment( &self, - _caller: String, msg: SignPayment, ) -> Result, GenericError> { @@ -102,7 +98,6 @@ pub trait PaymentDriver { async fn sign_payment_canonicalized( &self, - _caller: String, msg: SignPaymentCanonicalized, ) -> Result, GenericError> { @@ -113,7 +108,6 @@ pub trait PaymentDriver { async fn verify_signature( &self, - _caller: String, msg: VerifySignature, ) -> Result { @@ -125,8 +119,14 @@ pub trait PaymentDriver { let s: [u8; 32] = msg.signature[33..65].try_into().unwrap(); let signature = Signature { v, r, s }; - let payload = if msg.canonicalized { - utils::payment_hash_canonicalized(&msg.payment) + let payload = if let Some(payload) = msg.canonical { + if let Err(e) = msg.payment.verify_canonical(payload.as_slice()) { + log::info!( + "Signature verification: canonical representation doesn't match struct: {e}" + ); + return Ok(false); + } + utils::prepare_signature_hash(&payload) } else { utils::payment_hash(&msg.payment) }; @@ -140,7 +140,6 @@ pub trait PaymentDriver { async fn status( &self, - _caller: String, _msg: DriverStatus, ) -> Result, DriverStatusError>; diff --git a/core/payment-driver/base/src/lib.rs b/core/payment-driver/base/src/lib.rs index ee9eb97e48..46c98aecb9 100644 --- a/core/payment-driver/base/src/lib.rs +++ b/core/payment-driver/base/src/lib.rs @@ -17,6 +17,7 @@ pub mod cron; pub mod dao; pub mod db; pub mod driver; +pub mod signable; pub mod utils; pub use ya_core_model::driver as model; diff --git a/core/payment-driver/base/src/signable.rs b/core/payment-driver/base/src/signable.rs new file mode 100644 index 0000000000..811ccd1dcd --- /dev/null +++ b/core/payment-driver/base/src/signable.rs @@ -0,0 +1,67 @@ +use anyhow::anyhow; +use serde::de::DeserializeOwned; +use serde::Serialize; +use serde_json; + +use crate::utils; + +use ya_client_model::payment::Payment; + +/// Trait for objects that can be signed ensuring unified way to +/// convert structs to bytes, so signatures can be verified across multiple machines. +pub trait Signable: Serialize + DeserializeOwned + Clone + PartialEq { + /// Serialize structure to vector of bytes in canonical representation. + /// This representation should be binary equal on all machines. + fn canonicalize(&self) -> anyhow::Result> { + let shareable = self.clone().remove_private_info(); + Ok(serde_json_canonicalizer::to_vec(&shareable)?) + } + + /// Function should remove all information that shouldn't be sent to other Nodes. + /// Example: `allocation_id` in `Payment` structure is private information on Requestor + /// side and shouldn't be shared with Provider. + /// This step is necessary to create canonical version that can be signed and later validated + /// by other party. + fn remove_private_info(self) -> Self; + + /// Hash canonical representation of the structure. + /// In most cases we don't want to sign arrays of arbitrary length, so we use hash + /// of canonical representation instead. + fn hash_canonical(&self) -> anyhow::Result> { + Ok(utils::prepare_signature_hash(&self.canonicalize()?)) + } + + /// Verify if `canonical` representation is equivalent to `self`. + /// Since we always get structure and bytes with its canonical representation, + /// then verifying signature is not enough. We need to check if `canonical` was + /// created from structure itself. + fn verify_canonical(&self, canonical: &[u8]) -> anyhow::Result<()> { + let from_canonical = serde_json::from_slice::(canonical) + .map_err(|e| anyhow!("Failed to deserialize canonical representation: {e}"))? + .remove_private_info(); + let reference = self.clone().remove_private_info(); + + if reference != from_canonical { + return Err(anyhow!( + "Canonical representation doesn't match the structure" + )); + } + Ok(()) + } +} + +impl Signable for Payment { + fn remove_private_info(mut self) -> Self { + // We remove allocation ID from syncs because allocations are not transferred to peers and + // their IDs would be unknown to the recipient. + for agreement_payment in &mut self.agreement_payments.iter_mut() { + agreement_payment.allocation_id = None; + } + + for activity_payment in &mut self.activity_payments.iter_mut() { + activity_payment.allocation_id = None; + } + + self + } +} diff --git a/core/payment-driver/base/src/utils.rs b/core/payment-driver/base/src/utils.rs index abfa2e5519..b6adf161a0 100644 --- a/core/payment-driver/base/src/utils.rs +++ b/core/payment-driver/base/src/utils.rs @@ -66,14 +66,16 @@ pub fn u256_to_big_dec(v: U256) -> BigDecimal { v / Into::::into(PRECISION) } -pub fn payment_hash(payment: &Payment) -> Vec { +pub fn prepare_signature_hash(bytes: &[u8]) -> Vec { let mut hasher = Sha3_256::new(); - hasher.update(format!("{:?}", payment).as_bytes()); + hasher.update(bytes); hasher.finalize().to_vec() } +pub fn payment_hash(payment: &Payment) -> Vec { + prepare_signature_hash(format!("{:?}", payment).as_bytes()) +} + pub fn payment_hash_canonicalized(payment: &Payment) -> Vec { - let mut hasher = Sha3_256::new(); - hasher.update(serde_json_canonicalizer::to_vec(&payment).unwrap()); - hasher.finalize().to_vec() + prepare_signature_hash(&serde_json_canonicalizer::to_vec(payment).unwrap()) } diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index 40d74b5325..7d47375283 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -75,6 +75,7 @@ ya-market = "0.4" ya-sb-router = { workspace = true } ya-framework-basic = { version = "0.1" } ya-framework-mocks = { version = "0.1" } +ya-payment-driver = "0.3" actix-rt = "2.7" rand = "0.8" diff --git a/core/payment/src/processor.rs b/core/payment/src/processor.rs index aa753446ec..c476bc3f37 100644 --- a/core/payment/src/processor.rs +++ b/core/payment/src/processor.rs @@ -534,16 +534,17 @@ impl PaymentProcessor { let payment_dao: PaymentDao = db_executor.as_dao(); let sync_dao: SyncNotifsDao = db_executor.as_dao(); + // Always add new type of signature. Compatibility is for older Provider nodes only. + payment_dao + .add_signature( + payment_id.clone(), + msg_with_bytes.signature.clone(), + msg_with_bytes.signed_bytes.clone(), + ) + .await?; + if mark_sent { - payment_dao.mark_sent(payment_id.clone()).await?; - // Always add new type of signature. Compatibility is for older Provider nodes only. - payment_dao - .add_signature( - payment_id, - msg_with_bytes.signature.clone(), - msg_with_bytes.signed_bytes.clone(), - ) - .await?; + payment_dao.mark_sent(payment_id).await?; } else { sync_dao.upsert(payee_id).await?; SYNC_NOTIFS_NOTIFY.notify_one(); @@ -640,8 +641,7 @@ impl PaymentProcessor { &self, payment: Payment, signature: Vec, - canonicalized: bool, - signed_bytes: Option>, + canonical: Option>, ) -> Result<(), VerifyPaymentError> { // TODO: Split this into smaller functions let platform = payment.payment_platform.clone(); @@ -659,7 +659,7 @@ impl PaymentProcessor { .send(driver::VerifySignature::new( payment.clone(), signature.clone(), - canonicalized, + canonical.clone(), )) .await?? { @@ -782,13 +782,13 @@ impl PaymentProcessor { } // Insert payment into database (this operation creates and updates all related entities) - if signed_bytes.is_none() { + if canonical.is_none() { payment_dao .insert_received(payment, payee_id, None, None) .await?; } else { payment_dao - .insert_received(payment, payee_id, Some(signature), signed_bytes) + .insert_received(payment, payee_id, Some(signature), canonical) .await?; } } diff --git a/core/payment/src/service.rs b/core/payment/src/service.rs index 282a8e9310..aa7d3c7269 100644 --- a/core/payment/src/service.rs +++ b/core/payment/src/service.rs @@ -1248,16 +1248,7 @@ mod public { sender_id: String, msg: SendPayment, ) -> Result { - send_payment_impl( - db, - processor, - sender_id, - msg.payment, - false, - msg.signature, - None, - ) - .await + send_payment_impl(db, processor, sender_id, msg.payment, msg.signature, None).await } async fn send_payment_with_bytes( @@ -1271,7 +1262,6 @@ mod public { processor, sender_id, msg.payment, - true, msg.signature, Some(msg.signed_bytes), ) @@ -1283,9 +1273,8 @@ mod public { processor: Arc, sender_id: String, payment: Payment, - canonicalized: bool, signature: Vec, - signed_bytes: Option>, + canonical: Option>, ) -> Result { let payment_id = payment.payment_id.clone(); if sender_id != payment.payer_id.to_string() { @@ -1303,7 +1292,7 @@ mod public { "Verify payment processor started." ); let res = match processor - .verify_payment(payment, signature, canonicalized, signed_bytes) + .verify_payment(payment, signature, canonical) .await { Ok(_) => { diff --git a/core/payment/tests/test_payment_signature.rs b/core/payment/tests/test_payment_signature.rs index 329fdfdf44..dfbc64151c 100644 --- a/core/payment/tests/test_payment_signature.rs +++ b/core/payment/tests/test_payment_signature.rs @@ -1,9 +1,12 @@ use bigdecimal::BigDecimal; use chrono::Utc; +use std::str::FromStr; use std::time::Duration; use test_context::test_context; +use ya_client_model::NodeId; -use ya_client_model::payment::NewInvoice; +use ya_client_model::payment::{NewInvoice, Payment}; +use ya_core_model::payment::public::SendSignedPayment; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; @@ -12,6 +15,8 @@ use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::fake_payment::FakePayment; use ya_framework_mocks::payment::{Driver, PaymentRestExt}; +use ya_payment_driver::signable::Signable; +use ya_service_bus::RpcEndpoint; #[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] @@ -93,9 +98,105 @@ async fn test_payment_signature(ctx: &mut DroppableTestContext) -> anyhow::Resul .wait_for_invoice_payment::(&invoice.invoice_id, Duration::from_secs(5 * 60), None) .await?; assert_eq!(payments.len(), 1); - let _payment = requestor + let payment = requestor .get_signed_payment(&payments[0].payment_id) .await?; + log::info!("=== Validate if Payment confirmation has correct signatures."); + let signature = payment.signature.unwrap(); + let payment = payment.payload; + + assert_eq!(payment.amount, invoice.amount); + assert_eq!(signature.signed_bytes, payment.canonicalize().unwrap()); + payment.verify_canonical(&signature.signed_bytes).unwrap(); + + let correct = SendSignedPayment { + payment: payment.clone().remove_private_info(), + signature: signature.signature.clone(), + signed_bytes: signature.signed_bytes.clone(), + }; + + log::info!("=== Check if incorrect signature will be rejected."); + let mut sig_incorrect = signature.signature.clone(); + sig_incorrect[20] ^= sig_incorrect[20]; + let incorrect_signature = SendSignedPayment { + signature: sig_incorrect, + ..correct.clone() + }; + + let payment_gsb = node1.get_payment()?.gsb_public_endpoint(); + let result = payment_gsb + .send_as(invoice.recipient_id, incorrect_signature) + .await?; + assert!(result.is_err_and(|e| e.to_string().contains("Invalid payment signature"))); + + log::info!("=== Check if incorrect signed bytes will be rejected."); + let mut bytes_incorrect = signature.signed_bytes.clone(); + bytes_incorrect[20] ^= bytes_incorrect[20]; + let incorrect_signed_bytes = SendSignedPayment { + signed_bytes: bytes_incorrect.clone(), + ..correct.clone() + }; + + let result = payment_gsb + .send_as(invoice.recipient_id, incorrect_signed_bytes) + .await?; + assert!(result.is_err_and(|e| e.to_string().contains("Invalid payment signature"))); + + log::info!( + "=== Requestor shouldn't be able to report bigger amount than he paid on blockchain." + ); + let mut malicious_payment = payment.clone().remove_private_info(); + malicious_payment.amount += BigDecimal::from(1u64); + let mismatch_with_transaction_amount = SendSignedPayment { + payment: malicious_payment, + ..correct.clone() + }; + + let result = payment_gsb + .send_as(invoice.recipient_id, mismatch_with_transaction_amount) + .await?; + assert!(result.is_err()); + + log::info!("=== Requestor shouldn't be able to change payer info."); + let incorrect_payer = SendSignedPayment { + payment: Payment { + payer_id: NodeId::from_str("0x19659f72c4ad88f7d9934c8809deb9535ce0e4b8").unwrap(), + ..payment.clone().remove_private_info() + }, + ..correct.clone() + }; + + let result = payment_gsb + .send_as(invoice.recipient_id, incorrect_payer) + .await?; + assert!(result.is_err()); + + log::info!("=== Requestor shouldn't be able to change payer address info."); + let incorrect_payer = SendSignedPayment { + payment: Payment { + payer_addr: "0x19659f72c4ad88f7d9934c8809deb9535ce0e4b8".to_string(), + ..payment.clone().remove_private_info() + }, + ..correct.clone() + }; + + let result = payment_gsb + .send_as(invoice.recipient_id, incorrect_payer) + .await?; + assert!(result.is_err()); + + log::info!("=== Correct signature should be accepted."); + payment_gsb + .send_as(invoice.recipient_id, correct.clone()) + .await? + .unwrap(); + + log::info!("=== Payment confirmation sent for the second time should be rejected (could result in multi-spend)."); + let result = payment_gsb + .send_as(invoice.recipient_id, correct.clone()) + .await?; + assert!(result.is_err()); + Ok(()) } From 4f0348d7a9cee18119cc0c18ed74bf1771168f53 Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Thu, 5 Sep 2024 20:06:14 +0200 Subject: [PATCH 102/125] Use Signable trait to handle signatures using single code --- Cargo.lock | 2 ++ core/model/Cargo.toml | 10 ++++--- core/model/src/lib.rs | 1 + core/model/src/payment.rs | 11 +++++-- .../base => model}/src/signable.rs | 11 +++++-- core/payment-driver/base/src/bus.rs | 2 +- core/payment-driver/base/src/driver.rs | 30 +++++++++++-------- core/payment-driver/base/src/lib.rs | 1 - core/payment-driver/base/src/utils.rs | 13 ++------ core/payment-driver/dummy/src/service.rs | 5 +++- core/payment/src/payment_sync.rs | 15 +++++----- core/payment/src/processor.rs | 17 +++++------ core/payment/src/utils.rs | 16 ---------- core/payment/tests/test_payment_signature.rs | 2 +- 14 files changed, 65 insertions(+), 71 deletions(-) rename core/{payment-driver/base => model}/src/signable.rs (91%) diff --git a/Cargo.lock b/Cargo.lock index 04932bbf2f..c39eb17806 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8808,6 +8808,7 @@ dependencies = [ name = "ya-core-model" version = "0.9.1" dependencies = [ + "anyhow", "bigdecimal 0.2.2", "bitflags 1.3.2", "chrono", @@ -8818,6 +8819,7 @@ dependencies = [ "serde_bytes", "serde_json", "serde_json_canonicalizer", + "sha3 0.9.1", "structopt", "strum 0.24.1", "strum_macros 0.24.3", diff --git a/core/model/Cargo.toml b/core/model/Cargo.toml index 8c814f5b73..df118e0431 100644 --- a/core/model/Cargo.toml +++ b/core/model/Cargo.toml @@ -14,12 +14,12 @@ default = [] full = ['activity', 'appkey', 'driver', 'identity', 'market', 'net', 'payment', 'gftp', 'sgx', 'version'] activity = [] appkey = [] -driver = ['bigdecimal', 'bitflags'] +driver = ['bigdecimal', 'bitflags', 'anyhow', 'serde_json_canonicalizer', 'sha3'] gftp = [] identity = [] market = [] net = [] -payment = ['bigdecimal', 'bitflags'] +payment = ['bigdecimal', 'bitflags', 'anyhow', 'serde_json_canonicalizer', 'sha3'] sgx = ['graphene-sgx'] version = [] @@ -27,6 +27,7 @@ version = [] ya-client-model = "0.7" ya-service-bus = { workspace = true } +anyhow = { version = "1.0", optional = true } bigdecimal = { version = "0.2", features = ["serde"], optional = true } bitflags = { version = "1.2", optional = true } chrono = { version = "0.4", features = ["serde"] } @@ -35,10 +36,11 @@ graphene-sgx = { version = "0.3.3", optional = true } log = "0.4" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11.3" +serde_json_canonicalizer = { version = "0.2.0", optional = true } +serde_json = "1.0.113" +sha3 = { version = "0.9", optional = true } structopt = "0.3" strum = { workspace = true } strum_macros = "0.24" thiserror = "1.0.9" -serde_json_canonicalizer = "0.2.0" -serde_json = "1.0.113" diff --git a/core/model/src/lib.rs b/core/model/src/lib.rs index 6ab4826f41..c240a7f56f 100644 --- a/core/model/src/lib.rs +++ b/core/model/src/lib.rs @@ -29,6 +29,7 @@ pub mod gftp; pub mod sgx; pub mod bus; +pub mod signable; #[cfg(feature = "version")] pub mod version; diff --git a/core/model/src/payment.rs b/core/model/src/payment.rs index b9832fc1a7..ad963aa054 100644 --- a/core/model/src/payment.rs +++ b/core/model/src/payment.rs @@ -660,6 +660,7 @@ pub mod local { pub mod public { use super::*; + use crate::signable::Signable; use ya_client_model::NodeId; pub const BUS_ID: &str = "/public/payment"; @@ -839,7 +840,10 @@ pub mod public { impl SendPayment { pub fn new(payment: Payment, signature: Vec) -> Self { - Self { payment, signature } + Self { + payment: payment.remove_private_info(), + signature, + } } } @@ -861,9 +865,10 @@ pub mod public { impl SendSignedPayment { pub fn new(payment: Payment, signature: Vec) -> Self { - let signed_bytes = serde_json_canonicalizer::to_vec(&payment).unwrap(); + // Unwrap won't happen, because serialization is always possible. + let signed_bytes = payment.canonicalize().unwrap_or(vec![]); Self { - payment, + payment: payment.remove_private_info(), signature, signed_bytes, } diff --git a/core/payment-driver/base/src/signable.rs b/core/model/src/signable.rs similarity index 91% rename from core/payment-driver/base/src/signable.rs rename to core/model/src/signable.rs index 811ccd1dcd..eda92b52d0 100644 --- a/core/payment-driver/base/src/signable.rs +++ b/core/model/src/signable.rs @@ -2,8 +2,7 @@ use anyhow::anyhow; use serde::de::DeserializeOwned; use serde::Serialize; use serde_json; - -use crate::utils; +use sha3::{Digest, Sha3_256}; use ya_client_model::payment::Payment; @@ -28,7 +27,7 @@ pub trait Signable: Serialize + DeserializeOwned + Clone + PartialEq { /// In most cases we don't want to sign arrays of arbitrary length, so we use hash /// of canonical representation instead. fn hash_canonical(&self) -> anyhow::Result> { - Ok(utils::prepare_signature_hash(&self.canonicalize()?)) + Ok(prepare_signature_hash(&self.canonicalize()?)) } /// Verify if `canonical` representation is equivalent to `self`. @@ -50,6 +49,12 @@ pub trait Signable: Serialize + DeserializeOwned + Clone + PartialEq { } } +pub fn prepare_signature_hash(bytes: &[u8]) -> Vec { + let mut hasher = Sha3_256::new(); + hasher.update(bytes); + hasher.finalize().to_vec() +} + impl Signable for Payment { fn remove_private_info(mut self) -> Self { // We remove allocation ID from syncs because allocations are not transferred to peers and diff --git a/core/payment-driver/base/src/bus.rs b/core/payment-driver/base/src/bus.rs index cbb17bcd7d..4df5e9d80e 100644 --- a/core/payment-driver/base/src/bus.rs +++ b/core/payment-driver/base/src/bus.rs @@ -75,7 +75,7 @@ pub async fn bind_service( move |_, dr, c, m| async move { dr.sign_payment( c, m).await } ) .bind_with_processor( - move |_, dr, c, m| async move { dr.sign_payment_canonicalized(c, m).await } + move |_, dr, c, m| async move { dr.sign_payment_canonical(c, m).await } ) .bind_with_processor( move |_, dr, c, m| async move { dr.verify_signature( c, m).await } diff --git a/core/payment-driver/base/src/driver.rs b/core/payment-driver/base/src/driver.rs index d18d6eeeb0..40d5ff168e 100644 --- a/core/payment-driver/base/src/driver.rs +++ b/core/payment-driver/base/src/driver.rs @@ -16,13 +16,13 @@ use crate::model::*; use crate::utils; // Public revealed uses, required to implement this trait -use crate::signable::Signable; pub use async_trait::async_trait; pub use bigdecimal::BigDecimal; pub use ya_client_model::payment::network::Network; pub use ya_client_model::NodeId; pub use ya_core_model::identity::event::IdentityEvent; pub use ya_core_model::identity::Error as IdentityError; +use ya_core_model::signable::{prepare_signature_hash, Signable}; #[async_trait(?Send)] pub trait PaymentDriver { @@ -91,18 +91,20 @@ pub trait PaymentDriver { _caller: String, msg: SignPayment, ) -> Result, GenericError> { - let payload = utils::payment_hash(&msg.0); - let node_id = msg.0.payer_id; + let payment = msg.0.remove_private_info(); + let payload = utils::payment_hash(&payment); + let node_id = payment.payer_id; bus::sign(node_id, payload).await } - async fn sign_payment_canonicalized( + async fn sign_payment_canonical( &self, _caller: String, msg: SignPaymentCanonicalized, ) -> Result, GenericError> { - let payload = utils::payment_hash_canonicalized(&msg.0); - let node_id = msg.0.payer_id; + let payment = msg.0; + let payload = payment.hash_canonical().map_err(GenericError::new)?; + let node_id = payment.payer_id; bus::sign(node_id, payload).await } @@ -120,14 +122,18 @@ pub trait PaymentDriver { let signature = Signature { v, r, s }; let payload = if let Some(payload) = msg.canonical { - if let Err(e) = msg.payment.verify_canonical(payload.as_slice()) { - log::info!( - "Signature verification: canonical representation doesn't match struct: {e}" - ); - return Ok(false); + match msg.payment.verify_canonical(payload.as_slice()) { + Ok(_) => prepare_signature_hash(&payload), + Err(e) => { + log::info!( + "Signature verification: canonical representation doesn't match struct: {e}" + ); + return Ok(false); + } } - utils::prepare_signature_hash(&payload) } else { + // Backward compatibility version for older Nodes that don't send canonical + // signed bytes and used Payment debug formatting as representation. utils::payment_hash(&msg.payment) }; let pub_key = match signature.recover(payload.as_slice()) { diff --git a/core/payment-driver/base/src/lib.rs b/core/payment-driver/base/src/lib.rs index 46c98aecb9..ee9eb97e48 100644 --- a/core/payment-driver/base/src/lib.rs +++ b/core/payment-driver/base/src/lib.rs @@ -17,7 +17,6 @@ pub mod cron; pub mod dao; pub mod db; pub mod driver; -pub mod signable; pub mod utils; pub use ya_core_model::driver as model; diff --git a/core/payment-driver/base/src/utils.rs b/core/payment-driver/base/src/utils.rs index b6adf161a0..7141853e39 100644 --- a/core/payment-driver/base/src/utils.rs +++ b/core/payment-driver/base/src/utils.rs @@ -7,12 +7,13 @@ use bigdecimal::BigDecimal; use chrono::{DateTime, Utc}; use ethereum_types::U256; use num_bigint::ToBigInt; -use sha3::{Digest, Sha3_256}; // Local uses use crate::db::models::PaymentEntity; use crate::model::{PaymentDetails, SchedulePayment}; + use ya_client_model::payment::Payment; +use ya_core_model::signable::prepare_signature_hash; const PRECISION: u64 = 1_000_000_000_000_000_000; @@ -66,16 +67,6 @@ pub fn u256_to_big_dec(v: U256) -> BigDecimal { v / Into::::into(PRECISION) } -pub fn prepare_signature_hash(bytes: &[u8]) -> Vec { - let mut hasher = Sha3_256::new(); - hasher.update(bytes); - hasher.finalize().to_vec() -} - pub fn payment_hash(payment: &Payment) -> Vec { prepare_signature_hash(format!("{:?}", payment).as_bytes()) } - -pub fn payment_hash_canonicalized(payment: &Payment) -> Vec { - prepare_signature_hash(&serde_json_canonicalizer::to_vec(payment).unwrap()) -} diff --git a/core/payment-driver/dummy/src/service.rs b/core/payment-driver/dummy/src/service.rs index f01c0a09f3..694b50b3c2 100644 --- a/core/payment-driver/dummy/src/service.rs +++ b/core/payment-driver/dummy/src/service.rs @@ -5,6 +5,7 @@ use uuid::Uuid; use ya_client_model::payment::{DriverDetails, Network}; use ya_core_model::driver::*; use ya_core_model::payment::local as payment_srv; +use ya_core_model::signable::Signable; use ya_service_bus::typed::service; use ya_service_bus::{typed as bus, RpcEndpoint}; @@ -146,7 +147,9 @@ async fn fund(_db: (), _caller: String, _msg: Fund) -> Result Result, GenericError> { - Ok(ya_payment_driver::utils::payment_hash(&msg.0)) + Ok(ya_payment_driver::utils::payment_hash( + &msg.0.remove_private_info(), + )) } async fn verify_signature( diff --git a/core/payment/src/payment_sync.rs b/core/payment/src/payment_sync.rs index 4100a4bd63..a1831ca7ef 100644 --- a/core/payment/src/payment_sync.rs +++ b/core/payment/src/payment_sync.rs @@ -1,16 +1,14 @@ +use chrono::{DateTime, Utc}; use std::sync::Arc; use std::{collections::HashSet, time::Duration}; - -use crate::utils::remove_allocation_ids_from_payment; -use crate::Config; - -use chrono::{DateTime, Utc}; use tokio::sync::Notify; + use ya_client_model::{ payment::{Acceptance, InvoiceEventType}, NodeId, }; use ya_core_model::driver::SignPaymentCanonicalized; +use ya_core_model::signable::Signable; use ya_core_model::{ driver::{driver_bus_id, SignPayment}, identity::{self, IdentityInfo}, @@ -28,6 +26,7 @@ use ya_persistence::executor::DbExecutor; use ya_service_bus::{timeout::IntoTimeoutFuture, typed, RpcEndpoint}; use crate::dao::{DebitNoteDao, InvoiceDao, InvoiceEventDao, PaymentDao, SyncNotifsDao}; +use crate::Config; const REMOTE_CALL_TIMEOUT: Duration = Duration::from_secs(30); @@ -48,17 +47,17 @@ async fn payment_sync( let driver = &platform_components[0]; let bus_id = driver_bus_id(driver); - let payment = remove_allocation_ids_from_payment(payment); + let payment = payment.remove_private_info(); let signature = typed::service(bus_id.clone()) .send(SignPayment(payment.clone())) .await??; payments.push(SendPayment::new(payment.clone(), signature)); - let signature_canonicalized = typed::service(bus_id.clone()) + let signature_canonical = typed::service(bus_id.clone()) .send(SignPaymentCanonicalized(payment.clone())) .await??; - payments_canonicalized.push(SendSignedPayment::new(payment, signature_canonicalized)); + payments_canonicalized.push(SendSignedPayment::new(payment.clone(), signature_canonical)); } let mut invoice_accepts = Vec::default(); diff --git a/core/payment/src/processor.rs b/core/payment/src/processor.rs index c476bc3f37..17b74ccca9 100644 --- a/core/payment/src/processor.rs +++ b/core/payment/src/processor.rs @@ -9,7 +9,7 @@ use crate::error::processor::{ use crate::models::order::ReadObj as DbOrder; use crate::payment_sync::SYNC_NOTIFS_NOTIFY; use crate::timeout_lock::{MutexTimeoutExt, RwLockTimeoutExt}; -use crate::utils::remove_allocation_ids_from_payment; + use actix_web::web::Data; use bigdecimal::{BigDecimal, Zero}; use chrono::{DateTime, Utc}; @@ -22,6 +22,7 @@ use std::sync::Arc; use std::time::Duration; use thiserror::Error; use tokio::sync::{Mutex, RwLock}; + use ya_client_model::payment::allocation::Deposit; use ya_client_model::payment::{ Account, ActivityPayment, AgreementPayment, DriverDetails, Network, Payment, @@ -426,9 +427,8 @@ impl PaymentProcessor { let payer_id: NodeId; let payee_id: NodeId; let payment_id: String; - let mut payment: Payment; - { + let payment: Payment = { let db_executor = self.db_executor.timeout_lock(DB_LOCK_TIMEOUT).await?; let orders = db_executor @@ -488,13 +488,10 @@ impl PaymentProcessor { .get(payment_id.clone(), payer_id) .await? .unwrap(); - payment = signed_payment.payload; - } - - // Allocation IDs are requestor's private matter and should not be sent to provider - payment = remove_allocation_ids_from_payment(payment); + signed_payment.payload + }; - let signature_canonicalized = driver_endpoint(&driver) + let signature_canonical = driver_endpoint(&driver) .send(driver::SignPaymentCanonicalized(payment.clone())) .await??; let signature = driver_endpoint(&driver) @@ -506,7 +503,7 @@ impl PaymentProcessor { // Whether the provider was correctly notified of this fact is another matter. counter!("payment.invoices.requestor.paid", 1); let msg = SendPayment::new(payment.clone(), signature); - let msg_with_bytes = SendSignedPayment::new(payment, signature_canonicalized); + let msg_with_bytes = SendSignedPayment::new(payment.clone(), signature_canonical); let db_executor = Arc::clone(&self.db_executor); diff --git a/core/payment/src/utils.rs b/core/payment/src/utils.rs index 74da6641ff..9f6346c5fb 100644 --- a/core/payment/src/utils.rs +++ b/core/payment/src/utils.rs @@ -4,7 +4,6 @@ use futures::Future; use serde::{Deserialize, Serialize}; use std::time::Duration; use ya_client_model::market::{Agreement, Role}; -use ya_client_model::payment::Payment; use ya_core_model::market; use ya_service_bus::{typed as bus, RpcEndpoint}; @@ -174,21 +173,6 @@ pub async fn listen_for_events( .unwrap_or(Ok(vec![])) } -pub fn remove_allocation_ids_from_payment(mut payment: Payment) -> Payment { - // We remove allocation ID from syncs because allocations are not transferred to peers and - // their IDs would be unknown to the recipient. - // let mut payment = payment.clone(); - for agreement_payment in &mut payment.agreement_payments.iter_mut() { - agreement_payment.allocation_id = None; - } - - for activity_payment in &mut payment.activity_payments.iter_mut() { - activity_payment.allocation_id = None; - } - - payment -} - pub mod response { use actix_web::HttpResponse; use serde::Serialize; diff --git a/core/payment/tests/test_payment_signature.rs b/core/payment/tests/test_payment_signature.rs index dfbc64151c..04244c2a27 100644 --- a/core/payment/tests/test_payment_signature.rs +++ b/core/payment/tests/test_payment_signature.rs @@ -7,6 +7,7 @@ use ya_client_model::NodeId; use ya_client_model::payment::{NewInvoice, Payment}; use ya_core_model::payment::public::SendSignedPayment; +use ya_core_model::signable::Signable; use ya_framework_basic::async_drop::DroppableTestContext; use ya_framework_basic::log::enable_logs; use ya_framework_basic::{resource, temp_dir}; @@ -15,7 +16,6 @@ use ya_framework_mocks::net::MockNet; use ya_framework_mocks::node::MockNode; use ya_framework_mocks::payment::fake_payment::FakePayment; use ya_framework_mocks::payment::{Driver, PaymentRestExt}; -use ya_payment_driver::signable::Signable; use ya_service_bus::RpcEndpoint; #[cfg_attr(not(feature = "framework-test"), ignore)] From 02cd9f6aed9d6d80097d85114a03ad60d322d43f Mon Sep 17 00:00:00 2001 From: "nieznany.sprawiciel" Date: Fri, 6 Sep 2024 11:11:14 +0200 Subject: [PATCH 103/125] Fix SGX tests and clippy --- core/model/Cargo.toml | 2 +- core/model/src/lib.rs | 3 ++- core/model/src/payment.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/model/Cargo.toml b/core/model/Cargo.toml index df118e0431..45d558b112 100644 --- a/core/model/Cargo.toml +++ b/core/model/Cargo.toml @@ -14,7 +14,7 @@ default = [] full = ['activity', 'appkey', 'driver', 'identity', 'market', 'net', 'payment', 'gftp', 'sgx', 'version'] activity = [] appkey = [] -driver = ['bigdecimal', 'bitflags', 'anyhow', 'serde_json_canonicalizer', 'sha3'] +driver = ['bigdecimal', 'bitflags'] gftp = [] identity = [] market = [] diff --git a/core/model/src/lib.rs b/core/model/src/lib.rs index c240a7f56f..fbc236d410 100644 --- a/core/model/src/lib.rs +++ b/core/model/src/lib.rs @@ -21,6 +21,8 @@ pub mod net; #[cfg(feature = "payment")] pub mod payment; +#[cfg(feature = "payment")] +pub mod signable; #[cfg(feature = "gftp")] pub mod gftp; @@ -29,7 +31,6 @@ pub mod gftp; pub mod sgx; pub mod bus; -pub mod signable; #[cfg(feature = "version")] pub mod version; diff --git a/core/model/src/payment.rs b/core/model/src/payment.rs index ad963aa054..6bfe533d31 100644 --- a/core/model/src/payment.rs +++ b/core/model/src/payment.rs @@ -866,7 +866,7 @@ pub mod public { impl SendSignedPayment { pub fn new(payment: Payment, signature: Vec) -> Self { // Unwrap won't happen, because serialization is always possible. - let signed_bytes = payment.canonicalize().unwrap_or(vec![]); + let signed_bytes = payment.canonicalize().unwrap_or_default(); Self { payment: payment.remove_private_info(), signature, From 6dc5aa7fd070d044762df18392173df06f3cf92d Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 6 Sep 2024 13:59:35 +0200 Subject: [PATCH 104/125] Merge back from 0.16.0 --- Cargo.lock | 305 +++++++++--------- Cargo.toml | 24 +- agent/provider/Cargo.toml | 6 +- core/activity/Cargo.toml | 4 +- core/gftp/Cargo.toml | 16 +- core/gsb-api/Cargo.toml | 4 +- core/identity/Cargo.toml | 4 +- core/identity/src/cli/identity.rs | 13 + core/market/Cargo.toml | 4 +- core/metrics/Cargo.toml | 12 +- core/model/Cargo.toml | 17 +- core/net/Cargo.toml | 4 +- core/payment-driver/base/Cargo.toml | 4 +- core/payment-driver/base/src/db/models.rs | 18 +- core/payment-driver/dummy/Cargo.toml | 4 +- core/payment-driver/erc20/Cargo.toml | 4 +- .../payment-driver/erc20/config-payments.toml | 65 ++++ core/payment-driver/erc20/src/driver.rs | 232 +++++++------ core/payment-driver/erc20/src/erc20/config.rs | 64 +++- .../erc20/src/erc20/ethereum.rs | 9 + core/payment-driver/erc20/src/lib.rs | 12 + core/payment-driver/erc20/src/network.rs | 24 +- core/payment/Cargo.toml | 6 +- core/payment/src/cli.rs | 18 +- core/payment/src/wallet.rs | 3 +- core/persistence/Cargo.toml | 4 +- core/serv-api/Cargo.toml | 2 +- core/serv-api/web/Cargo.toml | 6 +- core/version/Cargo.toml | 6 +- core/vpn/Cargo.toml | 6 +- exe-unit/Cargo.toml | 4 +- golem_cli/Cargo.toml | 25 +- golem_cli/src/command/yagna.rs | 18 +- utils/agreement-utils/Cargo.toml | 2 +- utils/manifest-utils/Cargo.toml | 2 +- 35 files changed, 599 insertions(+), 352 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3af8f008f..a06fa5a17f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -828,12 +828,6 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "base64ct" version = "1.6.0" @@ -1012,7 +1006,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602bda35f33aeb571cef387dcd4042c643a8bf689d8aaac2cc47ea24cb7bc7e0" dependencies = [ "serde", - "serde_with 2.0.0", + "serde_with", ] [[package]] @@ -1678,18 +1672,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" dependencies = [ - "darling_core 0.10.2", - "darling_macro 0.10.2", -] - -[[package]] -name = "darling" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" -dependencies = [ - "darling_core 0.20.9", - "darling_macro 0.20.9", + "darling_core", + "darling_macro", ] [[package]] @@ -1706,42 +1690,17 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "darling_core" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.11.1", - "syn 2.0.60", -] - [[package]] name = "darling_macro" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ - "darling_core 0.10.2", + "darling_core", "quote", "syn 1.0.109", ] -[[package]] -name = "darling_macro" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" -dependencies = [ - "darling_core 0.20.9", - "quote", - "syn 2.0.60", -] - [[package]] name = "dashmap" version = "5.5.3" @@ -2175,8 +2134,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erc20_payment_lib" -version = "0.4.5" -source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=594d8a93b820b177ec267ccec5de61d8a9a45203#594d8a93b820b177ec267ccec5de61d8a9a45203" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3da9f9d42de38a0db4d8be0f734e0421b475363a55e4ec56606e274df6263872" dependencies = [ "actix", "actix-files", @@ -2217,8 +2177,9 @@ dependencies = [ [[package]] name = "erc20_payment_lib_common" -version = "0.4.5" -source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=594d8a93b820b177ec267ccec5de61d8a9a45203#594d8a93b820b177ec267ccec5de61d8a9a45203" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0774c96878af1018f6b3ce756cc0334ff6f136dcf15c774f0879ae015771d277" dependencies = [ "actix-files", "actix-web", @@ -2258,8 +2219,9 @@ dependencies = [ [[package]] name = "erc20_payment_lib_extra" -version = "0.4.5" -source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=594d8a93b820b177ec267ccec5de61d8a9a45203#594d8a93b820b177ec267ccec5de61d8a9a45203" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2084a5f3075a8b90d7edc22f7a4a6136670a860693ad81f76ebaf2a9e94a34fc" dependencies = [ "actix-files", "actix-web", @@ -2294,8 +2256,9 @@ dependencies = [ [[package]] name = "erc20_processor" -version = "0.4.5" -source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=594d8a93b820b177ec267ccec5de61d8a9a45203#594d8a93b820b177ec267ccec5de61d8a9a45203" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3d5bec86b99d8254ef9dce04710e33f18267a39da81ee7cf00ee0420f8e0d2" dependencies = [ "actix-cors", "actix-files", @@ -2342,8 +2305,9 @@ dependencies = [ [[package]] name = "erc20_rpc_pool" -version = "0.4.5" -source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=594d8a93b820b177ec267ccec5de61d8a9a45203#594d8a93b820b177ec267ccec5de61d8a9a45203" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf619ab7b405fd49b8ee383395be68ffe944b13f4001c03abb34eabe0f2f73" dependencies = [ "actix-files", "actix-web", @@ -2967,7 +2931,7 @@ dependencies = [ [[package]] name = "gftp" -version = "0.4.0" +version = "0.4.1" dependencies = [ "actix-rt", "anyhow", @@ -2986,7 +2950,7 @@ dependencies = [ "tokio", "url", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-service-bus", ] @@ -3053,7 +3017,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.6.0", ] [[package]] @@ -3088,9 +3052,9 @@ dependencies = [ "strum_macros 0.24.3", "tokio", "url", - "ya-client", + "ya-client 0.8.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-provider", "ya-utils-path", "ya-utils-process 0.3.0", @@ -3652,7 +3616,6 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", - "serde", ] [[package]] @@ -3977,7 +3940,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a42526bb432bcd1b43571d5f163984effa25409a29f1a3242a54d0577d55bcf" dependencies = [ - "darling 0.10.2", + "darling", "proc-macro2", "quote", "syn 1.0.109", @@ -6440,36 +6403,6 @@ dependencies = [ "time 0.3.36", ] -[[package]] -name = "serde_with" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad483d2ab0149d5a5ebcd9972a3852711e0153d863bf5a5d0391d28883c4a20" -dependencies = [ - "base64 0.22.1", - "chrono", - "hex", - "indexmap 1.9.3", - "indexmap 2.2.6", - "serde", - "serde_derive", - "serde_json", - "serde_with_macros", - "time 0.3.36", -] - -[[package]] -name = "serde_with_macros" -version = "3.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65569b702f41443e8bc8bbb1c5779bd0450bbe723b56198980e80ec45780bce2" -dependencies = [ - "darling 0.20.9", - "proc-macro2", - "quote", - "syn 2.0.60", -] - [[package]] name = "serde_yaml" version = "0.8.26" @@ -7184,12 +7117,6 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "structopt" version = "0.3.26" @@ -8726,8 +8653,8 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-gsb-http-proxy", "ya-net", "ya-persistence", @@ -8751,13 +8678,38 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] name = "ya-client" version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1#fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" +source = "git+https://github.com/golemfactory/ya-client.git?rev=e3113709178fa31fc144e79a835c06afcc6f8a34#e3113709178fa31fc144e79a835c06afcc6f8a34" +dependencies = [ + "actix-codec", + "awc", + "bytes 1.6.0", + "chrono", + "envy", + "futures 0.3.30", + "heck 0.4.1", + "hex", + "log", + "mime", + "serde", + "serde_json", + "serde_qs", + "structopt", + "thiserror", + "url", + "ya-client-model 0.6.0", +] + +[[package]] +name = "ya-client" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1db3f27d04b90bdd62583dee20684a6f147afdc26ebed1d4869d302d51f935" dependencies = [ "actix-codec", "awc", @@ -8775,13 +8727,34 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1#fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" +source = "git+https://github.com/golemfactory/ya-client.git?rev=e3113709178fa31fc144e79a835c06afcc6f8a34#e3113709178fa31fc144e79a835c06afcc6f8a34" +dependencies = [ + "bigdecimal 0.2.2", + "chrono", + "derive_more", + "hex", + "openssl", + "rand 0.8.5", + "secp256k1 0.27.0", + "serde", + "serde_bytes", + "serde_json", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", +] + +[[package]] +name = "ya-client-model" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a2f88152c30056f988a530ae6a3ac117979efddb35c4ac8cac7e41bbe317af" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -8794,7 +8767,6 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "serde_with 3.8.1", "strum 0.24.1", "strum_macros 0.24.3", "thiserror", @@ -8813,6 +8785,27 @@ dependencies = [ [[package]] name = "ya-core-model" version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed4e52e7d5cf8bafaf04daf6abf6bf5b199d291d65d014b7b8ca409ad7b0c599" +dependencies = [ + "chrono", + "derive_more", + "graphene-sgx", + "log", + "serde", + "serde_bytes", + "serde_json", + "structopt", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", + "ya-client-model 0.6.0", + "ya-service-bus", +] + +[[package]] +name = "ya-core-model" +version = "0.10.0" dependencies = [ "bigdecimal 0.2.2", "bitflags 1.3.2", @@ -8828,7 +8821,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", "ya-service-bus", ] @@ -8880,8 +8873,8 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-payment-driver", "ya-persistence", "ya-service-api-interfaces", @@ -8925,8 +8918,8 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -8979,9 +8972,9 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-counters", "ya-gsb-http-proxy", "ya-manifest-utils", @@ -9055,9 +9048,9 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client 0.8.0", + "ya-client-model 0.6.0", + "ya-core-model 0.9.1", "ya-net", "ya-service-bus", "ya-utils-futures", @@ -9100,9 +9093,9 @@ dependencies = [ "url", "uuid 0.8.2", "ya-agreement-utils", - "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client 0.8.0", + "ya-client-model 0.6.0", + "ya-core-model 0.9.1", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9145,8 +9138,8 @@ dependencies = [ "thiserror", "tokio", "uuid 1.8.0", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -9183,8 +9176,8 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.6.0", + "ya-core-model 0.9.1", "ya-counters", "ya-framework-basic", "ya-sb-router", @@ -9225,8 +9218,8 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-framework-basic", "ya-framework-mocks", "ya-persistence", @@ -9280,7 +9273,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9331,8 +9324,8 @@ dependencies = [ "tracing", "uuid 0.8.2", "ya-agreement-utils", - "ya-client", - "ya-core-model", + "ya-client 0.9.0", + "ya-core-model 0.10.0", "ya-diesel-utils", "ya-framework-basic", "ya-framework-mocks", @@ -9382,7 +9375,7 @@ dependencies = [ "structopt", "tokio", "url", - "ya-core-model", + "ya-core-model 0.10.0", "ya-service-api", "ya-service-api-interfaces", "ya-service-bus", @@ -9417,8 +9410,8 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", "ya-sb-proto", @@ -9494,9 +9487,9 @@ dependencies = [ "url", "uuid 0.8.2", "ya-agreement-utils", - "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client 0.9.0", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9538,8 +9531,8 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-bus", ] @@ -9562,8 +9555,8 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-service-api", "ya-service-api-interfaces", "ya-utils-process 0.3.0", @@ -9629,10 +9622,10 @@ dependencies = [ "walkdir", "winapi 0.3.9", "ya-agreement-utils", - "ya-client", - "ya-client-model", + "ya-client 0.9.0", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-file-logging", "ya-framework-basic", "ya-manifest-test-utils", @@ -9700,7 +9693,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.6.0", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -9871,7 +9864,7 @@ dependencies = [ "lazy_static", "serde", "url", - "ya-core-model", + "ya-core-model 0.10.0", "ya-utils-cli", ] @@ -9921,8 +9914,8 @@ dependencies = [ "serde", "structopt", "url", - "ya-client", - "ya-core-model", + "ya-client 0.9.0", + "ya-core-model 0.10.0", "ya-identity", "ya-persistence", "ya-sb-router", @@ -9965,8 +9958,8 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.6.0", + "ya-core-model 0.9.1", "ya-service-bus", ] @@ -10054,8 +10047,8 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.6.0", + "ya-core-model 0.9.1", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api", @@ -10180,9 +10173,9 @@ dependencies = [ "structopt", "thiserror", "tokio", - "ya-client", + "ya-client 0.9.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -10216,9 +10209,9 @@ dependencies = [ "tokio-stream", "url", "uuid 0.8.2", - "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client 0.9.0", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-persistence", @@ -10259,10 +10252,10 @@ dependencies = [ "tokio-util", "url", "ya-activity", - "ya-client", - "ya-client-model", + "ya-client 0.9.0", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-exe-unit", diff --git a/Cargo.toml b/Cargo.toml index 5139af7233..216e3f4e84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ path = "core/serv/src/main.rs" [dependencies] ya-activity = "0.4" ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.9" } +ya-core-model = { version = "0.10" } ya-dummy-driver = { version = "0.3", optional = true } ya-file-logging = "0.1" ya-gsb-api = "0.1" @@ -58,8 +58,8 @@ ya-utils-networking = "0.2" ya-fd-metrics = { path = "utils/fd-metrics" } ya-version = "0.2" ya-vpn = "0.2" -ya-client = "0.8" -ya-client-model = "0.6" +ya-client.workspace = true +ya-client-model.workspace = true gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests @@ -238,9 +238,13 @@ members = [ # diesel 1.4.* supports up to 0.23.0, but sqlx 0.5.9 requires 0.22.0 # sqlx 0.5.10 need 0.23.2, so 0.5.9 is last version possible derive_more = "0.99.11" -erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "594d8a93b820b177ec267ccec5de61d8a9a45203" } -erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "594d8a93b820b177ec267ccec5de61d8a9a45203" } -gftp = { version = "0.4.0", path = "core/gftp" } +#erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "79955d735f64467b01f04abfc70da39e4ba4be3c" } +#erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "79955d735f64467b01f04abfc70da39e4ba4be3c" } +#erc20_payment_lib = { path = "../../payments/erc20_payment_lib/crates/erc20_payment_lib" } +#erc20_processor = { path = "../../payments/erc20_payment_lib" } +erc20_payment_lib = { version = "0.4.7" } +erc20_processor = { version = "0.4.7" } +gftp = { version = "0.4.1", path = "core/gftp" } hex = "0.4.3" libsqlite3-sys = { version = "0.26.0", features = ["bundled"] } openssl = "0.10" @@ -261,6 +265,8 @@ ya-sb-proto = { version = "0.6.2" } ya-sb-util = { version = "0.5.1" } parking_lot = "0.12.3" mime = "0.3.17" +ya-client = "0.9" +ya-client-model = "0.7" [patch.crates-io] ## SERVICES @@ -287,10 +293,10 @@ ya-service-api-interfaces = { path = "core/serv-api/interfaces" } ya-service-api-web = { path = "core/serv-api/web" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "e3113709178fa31fc144e79a835c06afcc6f8a34" } #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "fbb15b49f8b9635cf3c3e2351ba6a9879f3b62d1" } -#ya-client-model = { path = "../ya-client/model" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "e3113709178fa31fc144e79a835c06afcc6f8a34" } +#ya-client-model = "0.7" golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } ## RELAY and networking stack diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index 25563c2cc6..159f350f92 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -15,10 +15,10 @@ path = "src/main.rs" [dependencies] ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } -ya-client = { version = "0.8", features = ['cli'] } -ya-client-model = "0.6" +ya-client = { workspace = true, features = ['cli'] } +ya-client-model.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.9", features = ['activity', 'payment'] } +ya-core-model = { version = "0.10", features = ['activity', 'payment'] } ya-file-logging = "0.1" ya-utils-actix = "0.2" ya-utils-cli = "0.1" diff --git a/core/activity/Cargo.toml b/core/activity/Cargo.toml index d3e0fc8d6f..0c961a105b 100644 --- a/core/activity/Cargo.toml +++ b/core/activity/Cargo.toml @@ -5,8 +5,8 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-core-model = { version = "0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.6", features = ["sgx"] } +ya-core-model = { version = "0.10", features = ["activity", "market"] } +ya-client-model = { workspace = true, features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" diff --git a/core/gftp/Cargo.toml b/core/gftp/Cargo.toml index 9b4508ea4e..6265930db8 100644 --- a/core/gftp/Cargo.toml +++ b/core/gftp/Cargo.toml @@ -1,25 +1,25 @@ [package] name = "gftp" -version = "0.4.0" +version = "0.4.1" authors = ["Golem Factory "] edition = "2018" homepage = "https://github.com/golemfactory/yagna" repository = "https://github.com/golemfactory/yagna" license = "LGPL-3.0" -description="Golem File Transfer Protocol" -keywords=["golem", "yagna"] +description = "Golem File Transfer Protocol" +keywords = ["golem", "yagna"] [features] -default=[] -bin=['env_logger', 'dotenv'] +default = [] +bin = ['env_logger', 'dotenv'] [[bin]] -name="gftp" -required-features=['bin'] +name = "gftp" +required-features = ['bin'] [dependencies] ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.9", features = ["gftp", "identity", "net"] } +ya-core-model = { version = "0.10", features = ["gftp", "identity", "net"] } ya-service-bus = { workspace = true } actix-rt = "2.7" diff --git a/core/gsb-api/Cargo.toml b/core/gsb-api/Cargo.toml index 8e8ad341d2..21cd9f47fa 100644 --- a/core/gsb-api/Cargo.toml +++ b/core/gsb-api/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.6" +ya-client-model.workspace = true ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" @@ -31,7 +31,7 @@ bytes = "1" tokio = { version = "1", features = ["macros"] } [dev-dependencies] -ya-core-model = { version = "^0.9", features = ["gftp"] } +ya-core-model = { version = "0.10", features = ["gftp"] } actix-test = "0.1" awc = "3" diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index 5d927802f8..5a09b25cf5 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -10,8 +10,8 @@ framework-test = [] [dependencies] -ya-client-model = { version = "0.6", features = ["with-diesel"] } -ya-core-model = { version = "^0.9", features = ["identity", "appkey"] } +ya-client-model = { workspace = true, features = ["with-diesel"] } +ya-core-model = { version = "0.10", features = ["identity", "appkey"] } ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" diff --git a/core/identity/src/cli/identity.rs b/core/identity/src/cli/identity.rs index 320875693a..aa87235fff 100644 --- a/core/identity/src/cli/identity.rs +++ b/core/identity/src/cli/identity.rs @@ -143,6 +143,10 @@ pub enum IdentityCommand { /// password for keystore #[structopt(long = "no-password")] no_password: bool, + + /// Set the newly created identity as the default, requires deamon restart + #[structopt(long = "set-default")] + set_default: bool, }, /// Update given identity Update { @@ -151,6 +155,8 @@ pub enum IdentityCommand { alias_or_id: NodeOrAlias, #[structopt(long)] alias: Option, + + /// Set the identity as the default, requires deamon restart #[structopt(long = "set-default")] set_default: bool, }, @@ -316,6 +322,7 @@ impl IdentityCommand { from_private_key, password, no_password, + set_default, } => { if from_keystore.is_some() && from_private_key.is_some() { anyhow::bail!("Only one of --from-keystore or --from-private-key can be used") @@ -367,6 +374,12 @@ impl IdentityCommand { from_keystore: Some(key_file), }) .await + .map_err(anyhow::Error::msg)??; + + let id = gsb + .local() + .send(identity::Update::with_id(id.node_id).with_default(*set_default)) + .await .map_err(anyhow::Error::msg)?; CommandOutput::object(id) } diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index 92d0f43eb0..c633f07dd7 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -10,8 +10,8 @@ test-suite = [] [dependencies] ya-agreement-utils = { workspace = true } -ya-client = "0.8" -ya-core-model = { version = "^0.9", features = ["market", "net"] } +ya-client.workspace = true +ya-core-model = { version = "0.10", features = ["market", "net"] } ya-diesel-utils = { version = "0.1" } ya-framework-basic = "0.1" ya-market-resolver = "0.2" diff --git a/core/metrics/Cargo.toml b/core/metrics/Cargo.toml index a059a0d554..47142e7f4c 100644 --- a/core/metrics/Cargo.toml +++ b/core/metrics/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "ya-metrics" version = "0.2.0" -description="Yagna metrics service" -keywords=["golem", "yagna", "metrics"] +description = "Yagna metrics service" +keywords = ["golem", "yagna", "metrics"] authors = ["Golem Factory "] edition = "2018" homepage = "https://github.com/golemfactory/yagna" @@ -10,10 +10,10 @@ repository = "https://github.com/golemfactory/yagna" license = "LGPL-3.0" [dependencies] -ya-core-model = { version = "^0.9", features = ["identity"] } +ya-core-model = { version = "0.10", features = ["identity"] } ya-service-api = "0.1" ya-service-api-interfaces = "0.2" -ya-service-bus = { workspace = true } +ya-service-bus = { workspace = true } awc = "3" actix-web = { version = "4", features = ["openssl"] } @@ -24,7 +24,9 @@ lazy_static = "1.4" log = "0.4" metrics = "0.16" metrics-core = "0.5.2" -metrics-runtime = { version = "0.13.1", default-features = false, features = ["observers"] } +metrics-runtime = { version = "0.13.1", default-features = false, features = [ + "observers", +] } percent-encoding = "2.1.0" structopt = "0.3" tokio = { version = "1", features = ["time", "sync"] } diff --git a/core/model/Cargo.toml b/core/model/Cargo.toml index 69aa7d4b0a..4ebd87b184 100644 --- a/core/model/Cargo.toml +++ b/core/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ya-core-model" -version = "0.9.1" +version = "0.10.0" authors = ["Golem Factory "] edition = "2018" homepage = "https://github.com/golemfactory/yagna" @@ -11,7 +11,18 @@ keywords = ["golem", "yagna"] [features] default = [] -full = ['activity', 'appkey', 'driver', 'identity', 'market', 'net', 'payment', 'gftp', 'sgx', 'version'] +full = [ + 'activity', + 'appkey', + 'driver', + 'identity', + 'market', + 'net', + 'payment', + 'gftp', + 'sgx', + 'version', +] activity = [] appkey = [] driver = ['bigdecimal', 'bitflags'] @@ -24,7 +35,7 @@ sgx = ['graphene-sgx'] version = [] [dependencies] -ya-client-model = "0.6" +ya-client-model.workspace = true ya-service-bus = { workspace = true } bigdecimal = { version = "0.2", features = ["serde"], optional = true } diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index a8129f15c3..53e86c4fde 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -15,8 +15,8 @@ packet-trace-enable = [ ] [dependencies] -ya-client-model = "0.6" -ya-core-model = { version = "^0.9", features = ["net", "identity"] } +ya-client-model.workspace = true +ya-core-model = { version = "0.10", features = ["net", "identity"] } ya-relay-client = { workspace = true } #ya-relay-client = "0.6" diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index 5b43474d1d..c543b4e830 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -29,8 +29,8 @@ thiserror = "1.0" tokio = { version = "1", features = ["macros"] } ## yagna dependencies -ya-client-model = "0.6" -ya-core-model = { version = "^0.9", features = [ +ya-client-model.workspace = true +ya-core-model = { version = "0.10", features = [ "driver", "identity", "payment", diff --git a/core/payment-driver/base/src/db/models.rs b/core/payment-driver/base/src/db/models.rs index b24ed3d67d..82b32d3df1 100644 --- a/core/payment-driver/base/src/db/models.rs +++ b/core/payment-driver/base/src/db/models.rs @@ -112,13 +112,15 @@ pub struct PaymentEntity { )] #[sql_type = "Integer"] pub enum Network { - Mainnet = 1, //Main Ethereum chain - Rinkeby = 4, //Rinkeby is Ethereum testnet - Goerli = 5, //Goerli is another Ethereum testnet + Mainnet = 1, //Main Ethereum chain + Rinkeby = 4, //Rinkeby is an Ethereum testnet + Goerli = 5, //Goerli is an Ethereum testnet + Sepolia = 11155111, //Sepolia is an Ethereum testnet #[default] Holesky = 17000, //Holesky is testnet for Holesky network - Mumbai = 80001, //Mumbai is testnet for Polygon network - Polygon = 137, //Polygon is Polygon production network + Polygon = 137, //Polygon is Polygon production network + Mumbai = 80001, //Mumbai is the legacy testnet for Polygon network + Amoy = 80002, //Amoy is the new testnet for Polygon network } impl FromStr for Network { @@ -127,11 +129,13 @@ impl FromStr for Network { fn from_str(s: &str) -> DbResult { match s.to_lowercase().as_str() { "mainnet" => Ok(Network::Mainnet), + "sepolia" => Ok(Network::Sepolia), "rinkeby" => Ok(Network::Rinkeby), "goerli" => Ok(Network::Goerli), "holesky" => Ok(Network::Holesky), "polygon" => Ok(Network::Polygon), "mumbai" => Ok(Network::Mumbai), + "amoy" => Ok(Network::Amoy), _ => Err(DbError::InvalidData(format!("Invalid network: {}", s))), } } @@ -141,11 +145,13 @@ impl Display for Network { fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { match *self { Network::Mainnet => f.write_str("mainnet"), + Network::Sepolia => f.write_str("sepolia"), Network::Rinkeby => f.write_str("rinkeby"), Network::Goerli => f.write_str("goerli"), Network::Holesky => f.write_str("holesky"), - Network::Mumbai => f.write_str("mumbai"), Network::Polygon => f.write_str("polygon"), + Network::Mumbai => f.write_str("mumbai"), + Network::Amoy => f.write_str("amoy"), } } } diff --git a/core/payment-driver/dummy/Cargo.toml b/core/payment-driver/dummy/Cargo.toml index 5c3b2a1249..766dd956ad 100644 --- a/core/payment-driver/dummy/Cargo.toml +++ b/core/payment-driver/dummy/Cargo.toml @@ -8,12 +8,12 @@ edition = "2018" default = [] [dependencies] -ya-core-model = { version = "^0.9", features = [ +ya-core-model = { version = "0.10", features = [ "driver", "identity", "payment", ] } -ya-client-model = { version = "0.6" } +ya-client-model.workspace = true ya-payment-driver = "0.3" ya-persistence = "0.3" ya-service-api-interfaces = "0.2" diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index bdcb514db5..5012ed92e5 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -43,8 +43,8 @@ rust_decimal = "1" ## yagna dependencies ya-payment-driver = "0.3" -ya-core-model = { version = "0.9" } -ya-client-model = "0.6" +ya-core-model = { version = "0.10" } +ya-client-model.workspace = true ya-service-api-interfaces = "0.2" ya-utils-futures.workspace = true ya-utils-networking = "0.2" diff --git a/core/payment-driver/erc20/config-payments.toml b/core/payment-driver/erc20/config-payments.toml index c82c4dec23..4a30be038f 100644 --- a/core/payment-driver/erc20/config-payments.toml +++ b/core/payment-driver/erc20/config-payments.toml @@ -161,3 +161,68 @@ max-timeout-ms = 5000 verify-interval-secs = 300 allowed-head-behind-secs = 60 dns-source = "polygon.rpc-node.dev.golem.network." + +[chain.amoy] +chain-name = "Amoy testnet" +chain-id = 80002 +currency-symbol = "tMATIC" +priority-fee = 30.111 +max-fee-per-gas = 500.0 +transaction-timeout = 100 +token = { address = "0x2b60e60d3fb0b36a7ccb388f9e71570da4c4594f", symbol = "tGLM" } +mint-contract = { address = "0xf29ff8a13211ac33861986e407190ae5c773d53c", max-glm-allowed = 400 } +wrapper-contract = { address = "0xa7b4447c1447edeb40ebbb1943e90b169ff44560" } +multi-contract = { address = "0xa0a51642a594763d78091bf03ee6bf8e8d663bba", max-at-once = 10 } +confirmation-blocks = 1 +block-explorer-url = "https://amoy.polygonscan.com" +external-source-check-interval = 300 + +[[chain.amoy.rpc-endpoints]] +names = """ + rpc-amoy.polygon.technology, + """ +endpoints = """ + https://rpc-amoy.polygon.technology, + """ +-priority = 0 +-max-timeout-ms = 5000 +-verify-interval-secs = 300 +-allowed-head-behind-secs = 60 + +[chain.sepolia] +chain-name = "Sepolia" +chain-id = 11155111 +currency-symbol = "tETH" +priority-fee = 0.000001 +max-fee-per-gas = 20.0 +transaction-timeout = 100 +token = { address = "0x167b15ada84c63427c6c813B915a42eFC72E7175", symbol = "tGLM" } +mint-contract = { address = "0x31A2a20956a40c2F358Fa5cec59D55a9C5d6fF9A", max-glm-allowed = 400 } +attestation-contract = { address = "0xC2679fBD37d54388Ce493F1DB75320D236e1815e" } +schema-registry-contract = { address = "0x0a7E2Ff54e76B8E6659aedc9103FB21c038050D0" } +confirmation-blocks = 0 +block-explorer-url = "https://sepolia.etherscan.io" +external-source-check-interval = 300 + +[[chain.sepolia.rpc-endpoints]] +names = """ + endpoints.omniatech.io/v1/eth/sepolia/public + ethereum-sepolia.blockpi.network/v1/rpc/public + eth-sepolia.public.blastapi.io + eth-sepolia-public.unifra.io + sepolia.gateway.tenderly.co + gateway.tenderly.co/public/sepolia +""" + +endpoints = """ + https://endpoints.omniatech.io/v1/eth/sepolia/public + https://ethereum-sepolia.blockpi.network/v1/rpc/public + https://eth-sepolia.public.blastapi.io + https://eth-sepolia-public.unifra.io + https://sepolia.gateway.tenderly.co + https://gateway.tenderly.co/public/sepolia +""" +priority = 0 +max-timeout-ms = 5000 +verify-interval-secs = 60 +allowed-head-behind-secs = 120 diff --git a/core/payment-driver/erc20/src/driver.rs b/core/payment-driver/erc20/src/driver.rs index c6741422d1..519287fe72 100644 --- a/core/payment-driver/erc20/src/driver.rs +++ b/core/payment-driver/erc20/src/driver.rs @@ -11,6 +11,7 @@ use erc20_payment_lib::model::{DepositId, TokenTransferDbObj, TxDbObj}; use erc20_payment_lib::runtime::{ PaymentRuntime, TransferArgs, TransferType, ValidateDepositResult, VerifyTransactionResult, }; +use erc20_payment_lib::setup::FaucetSetup; use erc20_payment_lib::signer::SignerAccount; use erc20_payment_lib::utils::{DecimalConvExt, U256ConvExt}; use erc20_payment_lib::{DriverEvent, DriverEventContent}; @@ -18,7 +19,6 @@ use ethereum_types::H160; use ethereum_types::U256; use num_bigint::BigInt; use std::collections::HashMap; -use std::env; use std::str::FromStr; use std::sync::Arc; use std::time::Instant; @@ -27,6 +27,7 @@ use uuid::Uuid; use web3::types::{Address, H256}; use ya_client_model::payment::allocation::Deposit; use ya_client_model::payment::DriverStatusProperty; +use ya_payment_driver::db::models::Network; use ya_payment_driver::driver::IdentityError; use ya_payment_driver::{ @@ -475,7 +476,7 @@ impl Erc20Driver { .deposit_details( network.to_string(), DepositId { - deposit_id, + deposit_id: deposit_id.clone(), lock_address: deposit_contract, }, ) @@ -585,6 +586,126 @@ impl Erc20Driver { Ok(ValidateAllocationResult::Valid) } + + async fn fund_eth_faucet( + &self, + faucet_setup: &FaucetSetup, + network: Network, + starting_eth_balance: U256, + address: H160, + ) -> Result { + let faucet_client_max_eth_allowed = + faucet_setup + .client_max_eth_allowed + .ok_or(GenericError::new(format!( + "Missing faucet client max eth allowed for network {}", + network + )))?; + + let faucet_srv_prefix = + faucet_setup + .client_srv + .as_ref() + .ok_or(GenericError::new(format!( + "Missing faucet_srv_port for network {}", + network + )))?; + let faucet_lookup_domain = faucet_setup + .lookup_domain + .as_ref() + .ok_or(GenericError::new(format!( + "Missing faucet_lookup_domain for network {}", + network + )))?; + let faucet_srv_port = faucet_setup.srv_port.ok_or(GenericError::new(format!( + "Missing faucet_srv_port for network {}", + network + )))?; + let faucet_host = faucet_setup + .client_host + .as_ref() + .ok_or(GenericError::new(format!( + "Missing faucet_host for network {}", + network + )))?; + + let eth_received = if starting_eth_balance + < faucet_client_max_eth_allowed + .to_u256_from_eth() + .map_err(|err| { + GenericError::new(format!( + "faucet_client_max_eth_allowed failed to convert {}", + err + )) + })? { + match faucet_donate( + faucet_srv_prefix, + faucet_lookup_domain, + faucet_host, + faucet_srv_port, + address, + ) + .await + { + Ok(_) => { + log::info!("Faucet donation successful"); + } + Err(e) => { + log::error!("Error donating from faucet: {}", e); + } + } + let time_now = Instant::now(); + loop { + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + + if time_now.elapsed().as_secs() > 120 { + log::error!( + "Faucet donation not received after {} seconds", + time_now.elapsed().as_secs() + ); + return Err(GenericError::new(format!( + "Faucet donation not received after {} seconds", + time_now.elapsed().as_secs() + ))); + } + match self + .payment_runtime + .get_token_balance(network.to_string(), address, None) + .await + { + Ok(balance_res) => { + let current_balance = balance_res.gas_balance.unwrap_or(U256::zero()); + if current_balance > starting_eth_balance { + log::info!( + "Received {} ETH from faucet", + (current_balance - starting_eth_balance).to_eth_str() + ); + break current_balance - starting_eth_balance; + } else { + log::info!( + "Waiting for ETH from faucet. Current balance: {}. Elapsed: {}/{}", + current_balance.to_eth_str(), + time_now.elapsed().as_secs(), + 120 + ); + } + } + Err(err) => { + log::error!("Error getting gas balance: {}", err); + } + } + } + } else { + log::info!( + "ETH balance is {} which is more than {} allowed by faucet", + starting_eth_balance.to_eth_str(), + faucet_client_max_eth_allowed + ); + U256::zero() + }; + + Ok(eth_received) + } } #[async_trait(?Send)] @@ -657,10 +778,7 @@ impl PaymentDriver for Erc20Driver { .map_err(|e| GenericError::new(e.to_string()))?; } - Ok(GetRpcEndpointsResult { - endpoints: serde_json::to_value(endpoints).unwrap(), - sources: serde_json::to_value(sources).unwrap(), - }) + Ok(GetRpcEndpointsResult { endpoints, sources }) } async fn get_account_balance( @@ -775,13 +893,6 @@ impl PaymentDriver for Erc20Driver { "Missing mint min glm allowed for network {}", network )))?; - let faucet_client_max_eth_allowed = - faucet_setup - .client_max_eth_allowed - .ok_or(GenericError::new(format!( - "Missing faucet client max eth allowed for network {}", - network - )))?; let (starting_eth_balance, starting_glm_balance) = match self .payment_runtime @@ -809,94 +920,17 @@ impl PaymentDriver for Erc20Driver { } }; - let faucet_srv_prefix = faucet_setup.client_srv.ok_or(GenericError::new(format!( - "Missing faucet_srv_port for network {}", - network - )))?; - let faucet_lookup_domain = faucet_setup.lookup_domain.ok_or(GenericError::new( - format!("Missing faucet_lookup_domain for network {}", network), - ))?; - let faucet_srv_port = faucet_setup.srv_port.ok_or(GenericError::new(format!( - "Missing faucet_srv_port for network {}", - network - )))?; - let faucet_host = faucet_setup.client_host.ok_or(GenericError::new(format!( - "Missing faucet_host for network {}", - network - )))?; + let faucet_configured = faucet_setup.client_host.is_some(); - let eth_received = if starting_eth_balance - < faucet_client_max_eth_allowed - .to_u256_from_eth() - .map_err(|err| { - GenericError::new(format!( - "faucet_client_max_eth_allowed failed to convert {}", - err - )) - })? { - match faucet_donate( - &faucet_srv_prefix, - &faucet_lookup_domain, - &faucet_host, - faucet_srv_port, - address, - ) - .await - { - Ok(_) => { - log::info!("Faucet donation successful"); - } - Err(e) => { - log::error!("Error donating from faucet: {}", e); - } - } - let time_now = Instant::now(); - let mut iteration = -1; - loop { - iteration += 1; - if iteration == 0 { - tokio::time::sleep(std::time::Duration::from_secs(6)).await; - } else { - tokio::time::sleep(std::time::Duration::from_secs(5)).await; - } - if time_now.elapsed().as_secs() > 120 { - log::error!( - "Faucet donation not received after {} seconds", - time_now.elapsed().as_secs() - ); - return Err(GenericError::new(format!( - "Faucet donation not received after {} seconds", - time_now.elapsed().as_secs() - ))); - } - match self - .payment_runtime - .get_token_balance(network.to_string(), address, None) - .await - { - Ok(balance_res) => { - let current_balance = balance_res.gas_balance.unwrap_or(U256::zero()); - if current_balance > starting_eth_balance { - log::info!( - "Received {} ETH from faucet", - (current_balance - starting_eth_balance).to_eth_str() - ); - break current_balance - starting_eth_balance; - } else { - log::info!("Waiting for ETH from faucet. Current balance: {}. Elapsed: {}/{}", current_balance.to_eth_str(), time_now.elapsed().as_secs(), 120); - } - } - Err(err) => { - log::error!("Error getting gas balance: {}", err); - } - } - } + if !faucet_configured && !msg.mint_only() { + return Err(GenericError::new(format!("Network {} doesn't have faucet configured, so full fund (tETH + tGLM) cannot be done. \ + After obtaining funds, re-run with --mint-only to obtain tGLM token.", network))); + } + + let eth_received = if faucet_configured && !msg.mint_only() { + self.fund_eth_faucet(&faucet_setup, network, starting_eth_balance, address) + .await? } else { - log::info!( - "ETH balance is {} which is more than {} allowed by faucet", - starting_eth_balance.to_eth_str(), - faucet_client_max_eth_allowed - ); U256::zero() }; diff --git a/core/payment-driver/erc20/src/erc20/config.rs b/core/payment-driver/erc20/src/erc20/config.rs index cce6215869..d62f786d2d 100644 --- a/core/payment-driver/erc20/src/erc20/config.rs +++ b/core/payment-driver/erc20/src/erc20/config.rs @@ -20,6 +20,20 @@ pub struct EnvConfiguration { } lazy_static! { + pub static ref MAINNET_CONFIG: EnvConfiguration = EnvConfiguration { + glm_contract_address: utils::str_to_addr( + &env::var("MAINNET_GLM_CONTRACT_ADDRESS") + .unwrap_or_else(|_| "0x7DD9c5Cba05E151C895FDe1CF355C9A1D5DA6429".to_string()) + ) + .unwrap(), + glm_faucet_address: None, + required_confirmations: { + match env::var("ERC20_MAINNET_REQUIRED_CONFIRMATIONS").map(|s| s.parse()) { + Ok(Ok(x)) => x, + _ => 5, + } + } + }; pub static ref RINKEBY_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( &env::var("RINKEBY_TGLM_CONTRACT_ADDRESS") @@ -40,35 +54,41 @@ lazy_static! { } } }; - pub static ref MAINNET_CONFIG: EnvConfiguration = EnvConfiguration { + pub static ref GOERLI_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( - &env::var("MAINNET_GLM_CONTRACT_ADDRESS") - .unwrap_or_else(|_| "0x7DD9c5Cba05E151C895FDe1CF355C9A1D5DA6429".to_string()) + &env::var("GOERLI_TGLM_CONTRACT_ADDRESS") + .unwrap_or_else(|_| "0x33af15c79d64b85ba14aaffaa4577949104b22e8".to_string()) ) .unwrap(), - glm_faucet_address: None, + glm_faucet_address: Some( + utils::str_to_addr( + &env::var("GOERLI_TGLM_FAUCET_ADDRESS") + .unwrap_or_else(|_| "0xCCA41b09C1F50320bFB41BD6822BD0cdBDC7d85C".to_string()) + ) + .unwrap() + ), required_confirmations: { - match env::var("ERC20_MAINNET_REQUIRED_CONFIRMATIONS").map(|s| s.parse()) { + match env::var("ERC20_GOERLI_REQUIRED_CONFIRMATIONS").map(|s| s.parse()) { Ok(Ok(x)) => x, - _ => 5, + _ => 3, } } }; - pub static ref GOERLI_CONFIG: EnvConfiguration = EnvConfiguration { + pub static ref SEPOLIA_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( - &env::var("GOERLI_TGLM_CONTRACT_ADDRESS") - .unwrap_or_else(|_| "0x33af15c79d64b85ba14aaffaa4577949104b22e8".to_string()) + &env::var("SEPOLIA_TGLM_CONTRACT_ADDRESS") + .unwrap_or_else(|_| "0x167b15ada84c63427c6c813B915a42eFC72E7175".to_string()) ) .unwrap(), glm_faucet_address: Some( utils::str_to_addr( - &env::var("GOERLI_TGLM_FAUCET_ADDRESS") - .unwrap_or_else(|_| "0xCCA41b09C1F50320bFB41BD6822BD0cdBDC7d85C".to_string()) + &env::var("SEPOLIA_TGLM_FAUCET_ADDRESS") + .unwrap_or_else(|_| "0x31A2a20956a40c2F358Fa5cec59D55a9C5d6fF9A".to_string()) ) .unwrap() ), required_confirmations: { - match env::var("ERC20_GOERLI_REQUIRED_CONFIRMATIONS").map(|s| s.parse()) { + match env::var("ERC20_SEPOLIA_REQUIRED_CONFIRMATIONS").map(|s| s.parse()) { Ok(Ok(x)) => x, _ => 3, } @@ -108,6 +128,26 @@ lazy_static! { } } }; + pub static ref AMOY_CONFIG: EnvConfiguration = EnvConfiguration { + glm_contract_address: utils::str_to_addr( + &env::var("AMOY_TGLM_CONTRACT_ADDRESS") + .unwrap_or_else(|_| "0x2b60e60d3fb0b36a7ccb388f9e71570da4c4594f".to_string()) + ) + .unwrap(), + glm_faucet_address: Some( + utils::str_to_addr( + &env::var("AMOY_TGLM_FAUCET_ADDRESS") + .unwrap_or_else(|_| "0xf29ff8a13211ac33861986e407190ae5c773d53c".to_string()) + ) + .unwrap() + ), + required_confirmations: { + match env::var("ERC20_AMOY_REQUIRED_CONFIRMATIONS").map(|s| s.parse()) { + Ok(Ok(x)) => x, + _ => 3, + } + } + }; pub static ref POLYGON_MAINNET_CONFIG: EnvConfiguration = EnvConfiguration { glm_contract_address: utils::str_to_addr( &env::var("POLYGON_GLM_CONTRACT_ADDRESS") diff --git a/core/payment-driver/erc20/src/erc20/ethereum.rs b/core/payment-driver/erc20/src/erc20/ethereum.rs index 90e2a2bb72..8a436c22d8 100644 --- a/core/payment-driver/erc20/src/erc20/ethereum.rs +++ b/core/payment-driver/erc20/src/erc20/ethereum.rs @@ -530,6 +530,10 @@ fn get_rpc_addr_from_env(network: Network) -> Vec { Network::Mainnet => { collect_rpc_addr_from("MAINNET_GETH_ADDR", "https://geth.golem.network:55555") } + Network::Sepolia => collect_rpc_addr_from( + "SEPOLIA_GETH_ADDR", + "https://ethereum-sepolia.blockpi.network/v1/rpc/public", + ), Network::Rinkeby => collect_rpc_addr_from( "RINKEBY_GETH_ADDR", "http://geth.testnet.golem.network:55555", @@ -549,6 +553,9 @@ fn get_rpc_addr_from_env(network: Network) -> Vec { "MUMBAI_GETH_ADDR", "https://matic-mumbai.chainstacklabs.com", ), + Network::Amoy => { + collect_rpc_addr_from("AMOY_GETH_ADDR", "https://rpc-amoy.polygon.technology") + } } } @@ -595,9 +602,11 @@ fn get_env(network: Network) -> config::EnvConfiguration { Network::Mainnet => *config::MAINNET_CONFIG, Network::Rinkeby => *config::RINKEBY_CONFIG, Network::Goerli => *config::GOERLI_CONFIG, + Network::Sepolia => *config::SEPOLIA_CONFIG, Network::Holesky => *config::HOLESKY_CONFIG, Network::Mumbai => *config::MUMBAI_CONFIG, Network::Polygon => *config::POLYGON_MAINNET_CONFIG, + Network::Amoy => *config::AMOY_CONFIG, } } diff --git a/core/payment-driver/erc20/src/lib.rs b/core/payment-driver/erc20/src/lib.rs index 8061b4095e..758c1c2109 100644 --- a/core/payment-driver/erc20/src/lib.rs +++ b/core/payment-driver/erc20/src/lib.rs @@ -31,6 +31,18 @@ pub const MUMBAI_PLATFORM: &str = "erc20-mumbai-tglm"; pub const MUMBAI_CURRENCY_SHORT: &str = "tMATIC"; pub const MUMBAI_CURRENCY_LONG: &str = "Test MATIC"; +pub const AMOY_NETWORK: &str = "amoy"; +pub const AMOY_TOKEN: &str = "tGLM"; +pub const AMOY_PLATFORM: &str = "erc20-amoy-tglm"; +pub const AMOY_CURRENCY_SHORT: &str = "tMATIC"; +pub const AMOY_CURRENCY_LONG: &str = "Test MATIC"; + +pub const SEPOLIA_NETWORK: &str = "sepolia"; +pub const SEPOLIA_TOKEN: &str = "tGLM"; +pub const SEPOLIA_PLATFORM: &str = "erc20-sepolia-tglm"; +pub const SEPOLIA_CURRENCY_SHORT: &str = "tETH"; +pub const SEPOLIA_CURRENCY_LONG: &str = "Sepolia Ether"; + pub const MAINNET_NETWORK: &str = "mainnet"; pub const MAINNET_TOKEN: &str = "GLM"; pub const MAINNET_PLATFORM: &str = "erc20-mainnet-glm"; diff --git a/core/payment-driver/erc20/src/network.rs b/core/payment-driver/erc20/src/network.rs index d69e9b46e7..d24f1c7b7a 100644 --- a/core/payment-driver/erc20/src/network.rs +++ b/core/payment-driver/erc20/src/network.rs @@ -7,6 +7,7 @@ use ya_payment_driver::{db::models::Network as DbNetwork, driver::Network, model // Local uses use crate::{ + AMOY_CURRENCY_LONG, AMOY_CURRENCY_SHORT, AMOY_NETWORK, AMOY_PLATFORM, AMOY_TOKEN, GOERLI_CURRENCY_LONG, GOERLI_CURRENCY_SHORT, GOERLI_NETWORK, GOERLI_PLATFORM, GOERLI_TOKEN, HOLESKY_CURRENCY_LONG, HOLESKY_CURRENCY_SHORT, HOLESKY_NETWORK, HOLESKY_PLATFORM, HOLESKY_TOKEN, MAINNET_CURRENCY_LONG, MAINNET_CURRENCY_SHORT, MAINNET_NETWORK, @@ -14,7 +15,8 @@ use crate::{ MUMBAI_PLATFORM, MUMBAI_TOKEN, POLYGON_MAINNET_CURRENCY_LONG, POLYGON_MAINNET_CURRENCY_SHORT, POLYGON_MAINNET_NETWORK, POLYGON_MAINNET_PLATFORM, POLYGON_MAINNET_TOKEN, RINKEBY_CURRENCY_LONG, RINKEBY_CURRENCY_SHORT, RINKEBY_NETWORK, RINKEBY_PLATFORM, - RINKEBY_TOKEN, + RINKEBY_TOKEN, SEPOLIA_CURRENCY_LONG, SEPOLIA_CURRENCY_SHORT, SEPOLIA_NETWORK, + SEPOLIA_PLATFORM, SEPOLIA_TOKEN, }; lazy_static::lazy_static! { @@ -37,6 +39,10 @@ lazy_static::lazy_static! { HOLESKY_TOKEN.to_string() => HOLESKY_PLATFORM.to_string() } }, + SEPOLIA_NETWORK.to_string() => Network { + default_token: SEPOLIA_TOKEN.to_string(), + tokens: hashmap! {SEPOLIA_TOKEN.to_string() => SEPOLIA_PLATFORM.to_string()} + }, MAINNET_NETWORK.to_string() => Network { default_token: MAINNET_TOKEN.to_string(), tokens: hashmap! { @@ -49,6 +55,12 @@ lazy_static::lazy_static! { MUMBAI_TOKEN.to_string() => MUMBAI_PLATFORM.to_string() } }, + AMOY_NETWORK.to_string() => Network { + default_token: AMOY_TOKEN.to_string(), + tokens: hashmap! { + AMOY_TOKEN.to_string() => AMOY_PLATFORM.to_string() + } + }, POLYGON_MAINNET_NETWORK.to_string() => Network { default_token: POLYGON_MAINNET_TOKEN.to_string(), tokens: hashmap! { @@ -61,6 +73,7 @@ lazy_static::lazy_static! { pub static ref HOLESKY_DB_NETWORK: DbNetwork = DbNetwork::from_str(HOLESKY_NETWORK).unwrap(); pub static ref MAINNET_DB_NETWORK: DbNetwork = DbNetwork::from_str(MAINNET_NETWORK).unwrap(); pub static ref MUMBAI_DB_NETWORK: DbNetwork = DbNetwork::from_str(MUMBAI_NETWORK).unwrap(); + pub static ref AMOY_DB_NETWORK: DbNetwork = DbNetwork::from_str(AMOY_NETWORK).unwrap(); pub static ref POLYGON_MAINNET_DB_NETWORK: DbNetwork = DbNetwork::from_str(POLYGON_MAINNET_NETWORK).unwrap(); } @@ -71,6 +84,7 @@ pub fn platform_to_network_token(platform: String) -> Result<(DbNetwork, String) HOLESKY_PLATFORM => Ok((*HOLESKY_DB_NETWORK, HOLESKY_TOKEN.to_owned())), MAINNET_PLATFORM => Ok((*MAINNET_DB_NETWORK, MAINNET_TOKEN.to_owned())), MUMBAI_PLATFORM => Ok((*MUMBAI_DB_NETWORK, MUMBAI_TOKEN.to_owned())), + AMOY_PLATFORM => Ok((*AMOY_DB_NETWORK, AMOY_TOKEN.to_owned())), POLYGON_MAINNET_PLATFORM => Ok(( *POLYGON_MAINNET_DB_NETWORK, POLYGON_MAINNET_TOKEN.to_owned(), @@ -96,6 +110,10 @@ pub fn platform_to_currency(platform: String) -> Result<(String, String), Generi HOLESKY_CURRENCY_SHORT.to_owned(), HOLESKY_CURRENCY_LONG.to_owned(), )), + SEPOLIA_PLATFORM => Ok(( + SEPOLIA_CURRENCY_SHORT.to_owned(), + SEPOLIA_CURRENCY_LONG.to_string(), + )), MAINNET_PLATFORM => Ok(( MAINNET_CURRENCY_SHORT.to_owned(), MAINNET_CURRENCY_LONG.to_owned(), @@ -104,6 +122,10 @@ pub fn platform_to_currency(platform: String) -> Result<(String, String), Generi MUMBAI_CURRENCY_SHORT.to_owned(), MUMBAI_CURRENCY_LONG.to_owned(), )), + AMOY_PLATFORM => Ok(( + AMOY_CURRENCY_SHORT.to_owned(), + AMOY_CURRENCY_LONG.to_owned(), + )), POLYGON_MAINNET_PLATFORM => Ok(( POLYGON_MAINNET_CURRENCY_SHORT.to_owned(), POLYGON_MAINNET_CURRENCY_LONG.to_owned(), diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index ca6354f8a6..7469604d89 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -10,8 +10,8 @@ framework-test = [] [dependencies] ya-agreement-utils = { workspace = true } -ya-client-model = { version = "0.6", features = ["with-diesel"] } -ya-core-model = { version = "^0.9", features = [ +ya-client-model = { workspace = true, features = ["with-diesel"] } +ya-core-model = { version = "0.10", features = [ "activity", "driver", "identity", @@ -66,7 +66,7 @@ uint = "0.7" uuid = { version = "0.8", features = ["v4"] } [dev-dependencies] -ya-client = "0.8" +ya-client = "0.9" ya-dummy-driver = "0.3" ya-erc20-driver = "0.4" ya-net = { version = "0.3", features = ["service"] } diff --git a/core/payment/src/cli.rs b/core/payment/src/cli.rs index af355cb3a9..cae16ce052 100644 --- a/core/payment/src/cli.rs +++ b/core/payment/src/cli.rs @@ -52,6 +52,9 @@ pub enum PaymentCli { Fund { #[structopt(flatten)] account: pay::AccountCli, + /// Mint token without attempting to obtain native currency from faucet + #[structopt(long = "mint-only")] + mint_only: bool, }, /// Initialize payment account (i.e. make it ready for sending/receiving funds) @@ -150,7 +153,7 @@ pub enum InvoiceCommand { impl PaymentCli { pub async fn run_command(self, ctx: &CliCtx) -> anyhow::Result { match self { - PaymentCli::Fund { account } => { + PaymentCli::Fund { account, mint_only } => { let address = resolve_address(account.address()).await?; let onboarding_supported = @@ -200,7 +203,14 @@ Typically operation should take less than 1 minute. log::warn!("{}", warn_message); CommandOutput::object( - wallet::fund(address, account.driver(), Some(account.network()), None).await?, + wallet::fund( + address, + account.driver(), + Some(account.network()), + None, + mint_only, + ) + .await?, ) } PaymentCli::Init { @@ -380,8 +390,8 @@ Typically operation should take less than 1 minute. account.driver, account.network, account.token, - if account.send { "X" } else { "" }, - if account.receive { "X" } else { "" } + account.send, + account.receive ]} }) .collect(), diff --git a/core/payment/src/wallet.rs b/core/payment/src/wallet.rs index 19fb5a6e37..d7d5d39e21 100644 --- a/core/payment/src/wallet.rs +++ b/core/payment/src/wallet.rs @@ -10,9 +10,10 @@ pub async fn fund( driver: String, network: Option, token: Option, + mint_only: bool, ) -> anyhow::Result { let driver_id = driver_bus_id(driver); - let message = Fund::new(address, network, token); + let message = Fund::new(address, network, token, mint_only); let reply = bus::service(driver_id).call(message).await??; Ok(reply) } diff --git a/core/persistence/Cargo.toml b/core/persistence/Cargo.toml index f376ed4053..3f5f5400b3 100644 --- a/core/persistence/Cargo.toml +++ b/core/persistence/Cargo.toml @@ -14,8 +14,8 @@ service = [ ] [dependencies] -ya-client-model = { version = "0.6", features = ["with-diesel"] } -ya-core-model = { version = "0.9" } +ya-client-model = { workspace = true, features = ["with-diesel"] } +ya-core-model = { version = "0.10" } ya-service-api = { version = "0.1", optional = true } ya-service-api-interfaces = { version = "0.2", optional = true } ya-utils-process = { version = "0.3", features = ["lock"], optional = true } diff --git a/core/serv-api/Cargo.toml b/core/serv-api/Cargo.toml index efd64ef953..05013bf9aa 100644 --- a/core/serv-api/Cargo.toml +++ b/core/serv-api/Cargo.toml @@ -12,4 +12,4 @@ serde = { version = "1.0", features = ["derive"] } url = "2.1.1" ya-utils-cli = "0.1" -ya-core-model = "0.9" +ya-core-model = "0.10" diff --git a/core/serv-api/web/Cargo.toml b/core/serv-api/web/Cargo.toml index 90117b680c..fcde51c4ec 100644 --- a/core/serv-api/web/Cargo.toml +++ b/core/serv-api/web/Cargo.toml @@ -6,10 +6,10 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.8" -ya-core-model = { version = "^0.9", features = ["appkey"] } +ya-client.workspace = true +ya-core-model = { version = "0.10", features = ["appkey"] } ya-service-api = "0.1" -ya-service-bus = { workspace = true } +ya-service-bus = { workspace = true } actix-cors = "0.6" actix-service = "2" diff --git a/core/version/Cargo.toml b/core/version/Cargo.toml index 6199451c4a..6c62525b39 100644 --- a/core/version/Cargo.toml +++ b/core/version/Cargo.toml @@ -6,13 +6,13 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = "0.8" +ya-client.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.9", features = ["version"] } +ya-core-model = { version = "0.10", features = ["version"] } ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" -ya-service-bus = { workspace = true } +ya-service-bus = { workspace = true } actix-web = "4" anyhow = "1.0" diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index 6083b50ef1..93accdb754 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -5,8 +5,8 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-core-model = { version = "^0.9", features = ["activity", "market"] } -ya-client-model = { version = "0.6", features = ["sgx"] } +ya-core-model = { version = "0.10", features = ["activity", "market"] } +ya-client-model = { workspace = true, features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" ya-service-api = "0.1" @@ -44,7 +44,7 @@ packet-trace-enable = ["ya-packet-trace/enable"] default = [] [dev-dependencies] -ya-client = "0.8" +ya-client = "0.9" actix-rt = "2.7" sha3 = "0.8.2" structopt = "0.3" diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index d720f53d2c..4ee9dc8b14 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -29,9 +29,9 @@ framework-test = ["ya-transfer/framework-test"] [dependencies] ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } -ya-client-model = "0.6" +ya-client-model.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } +ya-core-model = { version = "0.10", features = ["activity", "appkey"] } ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ "server", ] } diff --git a/golem_cli/Cargo.toml b/golem_cli/Cargo.toml index f869898650..59384a0378 100644 --- a/golem_cli/Cargo.toml +++ b/golem_cli/Cargo.toml @@ -8,18 +8,18 @@ edition = "2018" [dependencies] ya-client = { version = "0.8", features = ['cli'] } ya-compile-time-utils = "0.2" -ya-core-model = { version = "^0.9", features=["payment", "version"] } +ya-core-model = { version = "0.10", features = ["payment", "version"] } ya-provider = "0.3" ya-utils-path = "0.1.0" ya-utils-process = { version = "0.3", features = ["lock"] } -actix-rt="2.7" -ansi_term="0.12.1" +actix-rt = "2.7" +ansi_term = "0.12.1" anyhow = "1.0" bigdecimal = "0.2" byte-unit = "4.0" -chrono = { version = "0.4", features=["serde"] } -crossterm="0.23" +chrono = { version = "0.4", features = ["serde"] } +crossterm = "0.23" directories = "2.0.2" dotenv = "0.15" env_logger = "0.7" @@ -30,16 +30,23 @@ names = "0.10.0" openssl.workspace = true prettytable-rs = "0.10.0" promptly = "0.3.0" -rustyline="6.3.0" +rustyline = "6.3.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" strip-ansi-escapes = "0.1" structopt = "0.3" strum = { workspace = true } strum_macros = "0.24" -tokio = { version = "1", features = ["process", "signal", "time", "io-util", "io-std", "macros"] } +tokio = { version = "1", features = [ + "process", + "signal", + "time", + "io-util", + "io-std", + "macros", +] } url = "2.1" [target.'cfg(target_family = "unix")'.dependencies] -libc="0.2.73" -nix="0.22.0" +libc = "0.2.73" +nix = "0.22.0" diff --git a/golem_cli/src/command/yagna.rs b/golem_cli/src/command/yagna.rs index 2f15e455bf..3f48290a38 100644 --- a/golem_cli/src/command/yagna.rs +++ b/golem_cli/src/command/yagna.rs @@ -78,6 +78,22 @@ lazy_static! { token: "GLM", }, ); + erc20.insert( + NetworkName::Amoy.into(), + PaymentPlatform { + platform: "erc20-amoy-glm", + driver: "erc20", + token: "GLM", + }, + ); + erc20.insert( + NetworkName::Sepolia.into(), + PaymentPlatform { + platform: "erc20-sepolia-glm", + driver: "erc20", + token: "GLM", + }, + ); PaymentDriver { platforms: erc20, @@ -136,7 +152,7 @@ lazy_static! { ); ngm.insert( NetworkGroup::Testnet, - vec![NetworkName::Holesky, NetworkName::Mumbai], + vec![NetworkName::Holesky, NetworkName::Amoy, NetworkName::Sepolia], ); ngm }; diff --git a/utils/agreement-utils/Cargo.toml b/utils/agreement-utils/Cargo.toml index c519e45980..817c38cbd0 100644 --- a/utils/agreement-utils/Cargo.toml +++ b/utils/agreement-utils/Cargo.toml @@ -13,7 +13,7 @@ keywords = ["golem", "yagna"] default = [] [dependencies] -ya-client-model = "0.6" +ya-client-model.workspace = true chrono = "0.4" regex = "1.5.4" diff --git a/utils/manifest-utils/Cargo.toml b/utils/manifest-utils/Cargo.toml index 69cadef477..4bb6e1707f 100644 --- a/utils/manifest-utils/Cargo.toml +++ b/utils/manifest-utils/Cargo.toml @@ -22,7 +22,7 @@ schema = ["schemars"] [dependencies] ya-agreement-utils = { workspace = true } ya-utils-path = "0.1" -ya-client-model = "0.6" +ya-client-model.workspace = true golem-certificate = "0.1.1" regex = "1.5" From 0c0ab05f74096441705f2acf65f652a2e7883ec3 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 6 Sep 2024 17:49:14 +0200 Subject: [PATCH 105/125] Problems with the net --- Cargo.lock | 8 ++++---- core/model/src/driver.rs | 12 +++++++++++- core/net/Cargo.toml | 4 ++-- core/payment-driver/erc20/src/driver.rs | 6 +++++- exe-unit/components/gsb-http-proxy/Cargo.toml | 4 ++-- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a06fa5a17f..1421f6f6c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9176,8 +9176,8 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model 0.6.0", - "ya-core-model 0.9.1", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-counters", "ya-framework-basic", "ya-sb-router", @@ -9410,8 +9410,8 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model 0.6.0", + "ya-core-model 0.9.1", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", "ya-sb-proto", diff --git a/core/model/src/driver.rs b/core/model/src/driver.rs index a207f9cded..4afb378adf 100644 --- a/core/model/src/driver.rs +++ b/core/model/src/driver.rs @@ -215,14 +215,21 @@ pub struct Fund { address: String, network: Option, token: Option, + mint_only: bool, } impl Fund { - pub fn new(address: String, network: Option, token: Option) -> Self { + pub fn new( + address: String, + network: Option, + token: Option, + mint_only: bool, + ) -> Self { Self { address, network, token, + mint_only, } } pub fn address(&self) -> String { @@ -234,6 +241,9 @@ impl Fund { pub fn token(&self) -> Option { self.token.clone() } + pub fn mint_only(&self) -> bool { + self.mint_only + } } impl RpcMessage for Fund { diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index 53e86c4fde..a8129f15c3 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -15,8 +15,8 @@ packet-trace-enable = [ ] [dependencies] -ya-client-model.workspace = true -ya-core-model = { version = "0.10", features = ["net", "identity"] } +ya-client-model = "0.6" +ya-core-model = { version = "^0.9", features = ["net", "identity"] } ya-relay-client = { workspace = true } #ya-relay-client = "0.6" diff --git a/core/payment-driver/erc20/src/driver.rs b/core/payment-driver/erc20/src/driver.rs index 519287fe72..a1000ab2da 100644 --- a/core/payment-driver/erc20/src/driver.rs +++ b/core/payment-driver/erc20/src/driver.rs @@ -19,6 +19,7 @@ use ethereum_types::H160; use ethereum_types::U256; use num_bigint::BigInt; use std::collections::HashMap; +use std::env; use std::str::FromStr; use std::sync::Arc; use std::time::Instant; @@ -778,7 +779,10 @@ impl PaymentDriver for Erc20Driver { .map_err(|e| GenericError::new(e.to_string()))?; } - Ok(GetRpcEndpointsResult { endpoints, sources }) + Ok(GetRpcEndpointsResult { + endpoints: serde_json::to_value(endpoints).unwrap(), + sources: serde_json::to_value(sources).unwrap(), + }) } async fn get_account_balance( diff --git a/exe-unit/components/gsb-http-proxy/Cargo.toml b/exe-unit/components/gsb-http-proxy/Cargo.toml index a9b50d0393..4e532c73a7 100644 --- a/exe-unit/components/gsb-http-proxy/Cargo.toml +++ b/exe-unit/components/gsb-http-proxy/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] ya-service-bus = { workspace = true } ya-counters = { path = "../counters" } -ya-client-model = "0.6" -ya-core-model = { version = "^0.9" } +ya-client-model = "0.7.0" +ya-core-model = { version = "0.10.0", features = ["net"] } thiserror = "1.0" serde = { version = "1.0", features = ["derive"] } From 4d21acc4fca818e41a23f82f1e12a03f0e66522a Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 6 Sep 2024 19:43:03 +0200 Subject: [PATCH 106/125] Resolved problems with older packages --- Cargo.lock | 298 ++++++++++-------- Cargo.toml | 14 +- agent/provider/Cargo.toml | 2 +- core/activity/Cargo.toml | 2 +- core/gftp/Cargo.toml | 2 +- core/gsb-api/Cargo.toml | 2 +- core/identity/Cargo.toml | 2 +- core/market/Cargo.toml | 2 +- core/metrics/Cargo.toml | 2 +- core/model/src/payment.rs | 6 +- core/net/Cargo.toml | 4 +- core/payment-driver/base/Cargo.toml | 2 +- core/payment-driver/dummy/Cargo.toml | 2 +- core/payment-driver/erc20/Cargo.toml | 2 +- core/payment-driver/erc20/src/driver.rs | 2 +- core/payment/Cargo.toml | 4 +- core/persistence/Cargo.toml | 2 +- core/serv-api/Cargo.toml | 2 +- core/serv-api/web/Cargo.toml | 2 +- core/sgx/Cargo.toml | 4 +- core/version/Cargo.toml | 2 +- core/vpn/Cargo.toml | 4 +- exe-unit/Cargo.toml | 2 +- exe-unit/components/gsb-http-proxy/Cargo.toml | 4 +- exe-unit/components/transfer/Cargo.toml | 5 +- golem_cli/Cargo.toml | 4 +- golem_cli/src/command/yagna.rs | 6 +- .../test-framework/framework-basic/Cargo.toml | 6 +- .../test-framework/framework-mocks/Cargo.toml | 6 +- .../framework-mocks/src/identity.rs | 2 + .../framework-mocks/src/payment.rs | 1 + 31 files changed, 218 insertions(+), 182 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1421f6f6c8..ce5f385cdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -828,6 +828,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -1006,7 +1012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602bda35f33aeb571cef387dcd4042c643a8bf689d8aaac2cc47ea24cb7bc7e0" dependencies = [ "serde", - "serde_with", + "serde_with 2.0.0", ] [[package]] @@ -1672,8 +1678,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.10.2", + "darling_macro 0.10.2", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core 0.20.10", + "darling_macro 0.20.10", ] [[package]] @@ -1690,17 +1706,42 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.60", +] + [[package]] name = "darling_macro" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" dependencies = [ - "darling_core", + "darling_core 0.10.2", "quote", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core 0.20.10", + "quote", + "syn 2.0.60", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -2950,7 +2991,7 @@ dependencies = [ "tokio", "url", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-service-bus", ] @@ -3002,7 +3043,7 @@ dependencies = [ [[package]] name = "golem-certificate" version = "0.1.1" -source = "git+https://github.com/golemfactory/golem-certificate.git?rev=f2d7514c18fc066e9cfb796090b90f5b27cfe1c6#f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" +source = "git+https://github.com/golemfactory/golem-certificate.git?rev=a410048b469b2cd3b1158bbec98808cf987d3f6b#a410048b469b2cd3b1158bbec98808cf987d3f6b" dependencies = [ "anyhow", "chrono", @@ -3017,7 +3058,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model 0.6.0", + "ya-client-model", ] [[package]] @@ -3052,9 +3093,9 @@ dependencies = [ "strum_macros 0.24.3", "tokio", "url", - "ya-client 0.8.0", + "ya-client", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-provider", "ya-utils-path", "ya-utils-process 0.3.0", @@ -3616,6 +3657,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", + "serde", ] [[package]] @@ -3940,7 +3982,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a42526bb432bcd1b43571d5f163984effa25409a29f1a3242a54d0577d55bcf" dependencies = [ - "darling", + "darling 0.10.2", "proc-macro2", "quote", "syn 1.0.109", @@ -6403,6 +6445,36 @@ dependencies = [ "time 0.3.36", ] +[[package]] +name = "serde_with" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "serde_with_macros", + "time 0.3.36", +] + +[[package]] +name = "serde_with_macros" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +dependencies = [ + "darling 0.20.10", + "proc-macro2", + "quote", + "syn 2.0.60", +] + [[package]] name = "serde_yaml" version = "0.8.26" @@ -7117,6 +7189,12 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "structopt" version = "0.3.26" @@ -8653,8 +8731,8 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-gsb-http-proxy", "ya-net", "ya-persistence", @@ -8678,31 +8756,7 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model 0.7.0", -] - -[[package]] -name = "ya-client" -version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=e3113709178fa31fc144e79a835c06afcc6f8a34#e3113709178fa31fc144e79a835c06afcc6f8a34" -dependencies = [ - "actix-codec", - "awc", - "bytes 1.6.0", - "chrono", - "envy", - "futures 0.3.30", - "heck 0.4.1", - "hex", - "log", - "mime", - "serde", - "serde_json", - "serde_qs", - "structopt", - "thiserror", - "url", - "ya-client-model 0.6.0", + "ya-client-model", ] [[package]] @@ -8727,34 +8781,13 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model 0.7.0", -] - -[[package]] -name = "ya-client-model" -version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=e3113709178fa31fc144e79a835c06afcc6f8a34#e3113709178fa31fc144e79a835c06afcc6f8a34" -dependencies = [ - "bigdecimal 0.2.2", - "chrono", - "derive_more", - "hex", - "openssl", - "rand 0.8.5", - "secp256k1 0.27.0", - "serde", - "serde_bytes", - "serde_json", - "strum 0.24.1", - "strum_macros 0.24.3", - "thiserror", + "ya-client-model", ] [[package]] name = "ya-client-model" version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a2f88152c30056f988a530ae6a3ac117979efddb35c4ac8cac7e41bbe317af" +source = "git+https://github.com/golemfactory/ya-client.git?rev=acccda451fb9fb796bac939f6379f9445dc74f11#acccda451fb9fb796bac939f6379f9445dc74f11" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -8767,6 +8800,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "serde_with 3.9.0", "strum 0.24.1", "strum_macros 0.24.3", "thiserror", @@ -8782,27 +8816,6 @@ dependencies = [ "vergen", ] -[[package]] -name = "ya-core-model" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed4e52e7d5cf8bafaf04daf6abf6bf5b199d291d65d014b7b8ca409ad7b0c599" -dependencies = [ - "chrono", - "derive_more", - "graphene-sgx", - "log", - "serde", - "serde_bytes", - "serde_json", - "structopt", - "strum 0.24.1", - "strum_macros 0.24.3", - "thiserror", - "ya-client-model 0.6.0", - "ya-service-bus", -] - [[package]] name = "ya-core-model" version = "0.10.0" @@ -8821,7 +8834,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model 0.7.0", + "ya-client-model", "ya-service-bus", ] @@ -8873,8 +8886,8 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-payment-driver", "ya-persistence", "ya-service-api-interfaces", @@ -8918,8 +8931,8 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -8972,9 +8985,9 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-counters", "ya-gsb-http-proxy", "ya-manifest-utils", @@ -9048,9 +9061,9 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client 0.8.0", - "ya-client-model 0.6.0", - "ya-core-model 0.9.1", + "ya-client", + "ya-client-model", + "ya-core-model", "ya-net", "ya-service-bus", "ya-utils-futures", @@ -9093,9 +9106,9 @@ dependencies = [ "url", "uuid 0.8.2", "ya-agreement-utils", - "ya-client 0.8.0", - "ya-client-model 0.6.0", - "ya-core-model 0.9.1", + "ya-client", + "ya-client-model", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9138,8 +9151,8 @@ dependencies = [ "thiserror", "tokio", "uuid 1.8.0", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -9176,8 +9189,8 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-counters", "ya-framework-basic", "ya-sb-router", @@ -9218,8 +9231,8 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-framework-basic", "ya-framework-mocks", "ya-persistence", @@ -9273,7 +9286,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9324,8 +9337,8 @@ dependencies = [ "tracing", "uuid 0.8.2", "ya-agreement-utils", - "ya-client 0.9.0", - "ya-core-model 0.10.0", + "ya-client", + "ya-core-model", "ya-diesel-utils", "ya-framework-basic", "ya-framework-mocks", @@ -9375,7 +9388,7 @@ dependencies = [ "structopt", "tokio", "url", - "ya-core-model 0.10.0", + "ya-core-model", "ya-service-api", "ya-service-api-interfaces", "ya-service-bus", @@ -9410,8 +9423,8 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model 0.6.0", - "ya-core-model 0.9.1", + "ya-client-model", + "ya-core-model", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", "ya-sb-proto", @@ -9487,9 +9500,9 @@ dependencies = [ "url", "uuid 0.8.2", "ya-agreement-utils", - "ya-client 0.9.0", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client", + "ya-client-model", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9531,8 +9544,8 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-persistence", "ya-service-bus", ] @@ -9555,8 +9568,8 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-service-api", "ya-service-api-interfaces", "ya-utils-process 0.3.0", @@ -9622,10 +9635,10 @@ dependencies = [ "walkdir", "winapi 0.3.9", "ya-agreement-utils", - "ya-client 0.9.0", - "ya-client-model 0.7.0", + "ya-client", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-file-logging", "ya-framework-basic", "ya-manifest-test-utils", @@ -9641,7 +9654,7 @@ dependencies = [ [[package]] name = "ya-relay-client" version = "0.6.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "anyhow", "async-trait", @@ -9656,6 +9669,7 @@ dependencies = [ "metrics 0.21.1", "num_cpus", "parking_lot 0.12.3", + "rand 0.8.5", "strum 0.25.0", "strum_macros 0.25.3", "thiserror", @@ -9671,7 +9685,7 @@ dependencies = [ [[package]] name = "ya-relay-core" version = "0.4.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "anyhow", "chrono", @@ -9693,7 +9707,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.6.0", + "ya-client-model", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -9701,7 +9715,7 @@ dependencies = [ [[package]] name = "ya-relay-proto" version = "0.4.3" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "anyhow", "bytes 1.6.0", @@ -9715,7 +9729,7 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489)", ] [[package]] @@ -9741,7 +9755,7 @@ dependencies = [ [[package]] name = "ya-relay-stack" version = "0.5.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "derive_more", "futures 0.3.30", @@ -9755,7 +9769,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489)", ] [[package]] @@ -9771,7 +9785,7 @@ dependencies = [ [[package]] name = "ya-relay-util" version = "0.1.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" dependencies = [ "bytes 1.6.0", "derive_more", @@ -9864,7 +9878,7 @@ dependencies = [ "lazy_static", "serde", "url", - "ya-core-model 0.10.0", + "ya-core-model", "ya-utils-cli", ] @@ -9914,8 +9928,8 @@ dependencies = [ "serde", "structopt", "url", - "ya-client 0.9.0", - "ya-core-model 0.10.0", + "ya-client", + "ya-core-model", "ya-identity", "ya-persistence", "ya-sb-router", @@ -9958,8 +9972,8 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model 0.6.0", - "ya-core-model 0.9.1", + "ya-client-model", + "ya-core-model", "ya-service-bus", ] @@ -10047,8 +10061,9 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client-model 0.6.0", - "ya-core-model 0.9.1", + "ya-client", + "ya-client-model", + "ya-core-model", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api", @@ -10173,9 +10188,9 @@ dependencies = [ "structopt", "thiserror", "tokio", - "ya-client 0.9.0", + "ya-client", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -10209,9 +10224,9 @@ dependencies = [ "tokio-stream", "url", "uuid 0.8.2", - "ya-client 0.9.0", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client", + "ya-client-model", + "ya-core-model", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-persistence", @@ -10252,10 +10267,10 @@ dependencies = [ "tokio-util", "url", "ya-activity", - "ya-client 0.9.0", - "ya-client-model 0.7.0", + "ya-client", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-exe-unit", @@ -10396,3 +10411,8 @@ dependencies = [ "cc", "pkg-config", ] + +[[patch.unused]] +name = "ya-client" +version = "0.8.0" +source = "git+https://github.com/golemfactory/ya-client.git?rev=acccda451fb9fb796bac939f6379f9445dc74f11#acccda451fb9fb796bac939f6379f9445dc74f11" diff --git a/Cargo.toml b/Cargo.toml index 216e3f4e84..a1d0f6b970 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,8 @@ path = "core/serv/src/main.rs" [dependencies] ya-activity = "0.4" ya-compile-time-utils = "0.2" -ya-core-model = { version = "0.10" } + +ya-core-model.workspace = true ya-dummy-driver = { version = "0.3", optional = true } ya-file-logging = "0.1" ya-gsb-api = "0.1" @@ -255,7 +256,8 @@ url = "2.3.1" regex = "1.10.4" ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "0588dd1af311ae19c621b04cc2a4cfd9c0483252" } +ya-core-model = { version = "0.10", path = "core/model" } +ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489" } ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } ya-utils-futures = { path = "utils/futures" } @@ -265,7 +267,9 @@ ya-sb-proto = { version = "0.6.2" } ya-sb-util = { version = "0.5.1" } parking_lot = "0.12.3" mime = "0.3.17" +# true version is given in patches section ya-client = "0.9" +# true version is given in patches section ya-client-model = "0.7" [patch.crates-io] @@ -293,11 +297,11 @@ ya-service-api-interfaces = { path = "core/serv-api/interfaces" } ya-service-api-web = { path = "core/serv-api/web" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "e3113709178fa31fc144e79a835c06afcc6f8a34" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "acccda451fb9fb796bac939f6379f9445dc74f11" } #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "e3113709178fa31fc144e79a835c06afcc6f8a34" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "acccda451fb9fb796bac939f6379f9445dc74f11" } #ya-client-model = "0.7" -golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } +golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "a410048b469b2cd3b1158bbec98808cf987d3f6b" } ## RELAY and networking stack diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index 159f350f92..ee7dd15a2f 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -18,7 +18,7 @@ ya-manifest-utils = { version = "0.2" } ya-client = { workspace = true, features = ['cli'] } ya-client-model.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { version = "0.10", features = ['activity', 'payment'] } +ya-core-model = { workspace = true, features = ['activity', 'payment'] } ya-file-logging = "0.1" ya-utils-actix = "0.2" ya-utils-cli = "0.1" diff --git a/core/activity/Cargo.toml b/core/activity/Cargo.toml index 0c961a105b..f3164c23d7 100644 --- a/core/activity/Cargo.toml +++ b/core/activity/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-core-model = { version = "0.10", features = ["activity", "market"] } +ya-core-model = { workspace = true, features = ["activity", "market"] } ya-client-model = { workspace = true, features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" diff --git a/core/gftp/Cargo.toml b/core/gftp/Cargo.toml index 6265930db8..cdacdc72a7 100644 --- a/core/gftp/Cargo.toml +++ b/core/gftp/Cargo.toml @@ -19,7 +19,7 @@ required-features = ['bin'] [dependencies] ya-compile-time-utils = "0.2" -ya-core-model = { version = "0.10", features = ["gftp", "identity", "net"] } +ya-core-model = { workspace = true, features = ["gftp", "identity", "net"] } ya-service-bus = { workspace = true } actix-rt = "2.7" diff --git a/core/gsb-api/Cargo.toml b/core/gsb-api/Cargo.toml index 21cd9f47fa..dd8c889710 100644 --- a/core/gsb-api/Cargo.toml +++ b/core/gsb-api/Cargo.toml @@ -31,7 +31,7 @@ bytes = "1" tokio = { version = "1", features = ["macros"] } [dev-dependencies] -ya-core-model = { version = "0.10", features = ["gftp"] } +ya-core-model = { workspace = true, features = ["gftp"] } actix-test = "0.1" awc = "3" diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index 5a09b25cf5..10238f7299 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -11,7 +11,7 @@ framework-test = [] [dependencies] ya-client-model = { workspace = true, features = ["with-diesel"] } -ya-core-model = { version = "0.10", features = ["identity", "appkey"] } +ya-core-model = { workspace = true, features = ["identity", "appkey"] } ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index c633f07dd7..8036bd3c3d 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -11,7 +11,7 @@ test-suite = [] [dependencies] ya-agreement-utils = { workspace = true } ya-client.workspace = true -ya-core-model = { version = "0.10", features = ["market", "net"] } +ya-core-model = { workspace = true, features = ["market", "net"] } ya-diesel-utils = { version = "0.1" } ya-framework-basic = "0.1" ya-market-resolver = "0.2" diff --git a/core/metrics/Cargo.toml b/core/metrics/Cargo.toml index 47142e7f4c..7d3a2dcd8f 100644 --- a/core/metrics/Cargo.toml +++ b/core/metrics/Cargo.toml @@ -10,7 +10,7 @@ repository = "https://github.com/golemfactory/yagna" license = "LGPL-3.0" [dependencies] -ya-core-model = { version = "0.10", features = ["identity"] } +ya-core-model = { workspace = true, features = ["identity"] } ya-service-api = "0.1" ya-service-api-interfaces = "0.2" ya-service-bus = { workspace = true } diff --git a/core/model/src/payment.rs b/core/model/src/payment.rs index b9832fc1a7..3cbf8c1b5d 100644 --- a/core/model/src/payment.rs +++ b/core/model/src/payment.rs @@ -558,6 +558,8 @@ pub mod local { #[strum(props(token = "GLM"))] Mainnet, #[strum(props(token = "tGLM"))] + Sepolia, + #[strum(props(token = "tGLM"))] Rinkeby, #[strum(props(token = "tGLM"))] Goerli, @@ -567,6 +569,8 @@ pub mod local { Polygon, #[strum(props(token = "tGLM"))] Mumbai, + #[strum(props(token = "tGLM"))] + Amoy, } impl NetworkName { @@ -584,7 +588,7 @@ pub mod local { impl NetworkName { pub fn is_fundable(&self) -> bool { use NetworkName::*; - matches!(self, Goerli | Holesky) + matches!(self, Sepolia | Goerli | Holesky | Amoy) } pub fn all_fundable() -> Vec { diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index a8129f15c3..473bd8fadb 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -15,8 +15,8 @@ packet-trace-enable = [ ] [dependencies] -ya-client-model = "0.6" -ya-core-model = { version = "^0.9", features = ["net", "identity"] } +ya-client-model.workspace = true +ya-core-model = { workspace = true, features = ["net", "identity"] } ya-relay-client = { workspace = true } #ya-relay-client = "0.6" diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index c543b4e830..8d470c42ab 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -30,7 +30,7 @@ tokio = { version = "1", features = ["macros"] } ## yagna dependencies ya-client-model.workspace = true -ya-core-model = { version = "0.10", features = [ +ya-core-model = { workspace = true, features = [ "driver", "identity", "payment", diff --git a/core/payment-driver/dummy/Cargo.toml b/core/payment-driver/dummy/Cargo.toml index 766dd956ad..3633a76df9 100644 --- a/core/payment-driver/dummy/Cargo.toml +++ b/core/payment-driver/dummy/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" default = [] [dependencies] -ya-core-model = { version = "0.10", features = [ +ya-core-model = { workspace = true, features = [ "driver", "identity", "payment", diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index 5012ed92e5..94563eefe1 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -43,7 +43,7 @@ rust_decimal = "1" ## yagna dependencies ya-payment-driver = "0.3" -ya-core-model = { version = "0.10" } +ya-core-model = { workspace = true } ya-client-model.workspace = true ya-service-api-interfaces = "0.2" ya-utils-futures.workspace = true diff --git a/core/payment-driver/erc20/src/driver.rs b/core/payment-driver/erc20/src/driver.rs index a1000ab2da..86232afe31 100644 --- a/core/payment-driver/erc20/src/driver.rs +++ b/core/payment-driver/erc20/src/driver.rs @@ -477,7 +477,7 @@ impl Erc20Driver { .deposit_details( network.to_string(), DepositId { - deposit_id: deposit_id.clone(), + deposit_id, lock_address: deposit_contract, }, ) diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index 7469604d89..0f732b0b84 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -11,7 +11,7 @@ framework-test = [] [dependencies] ya-agreement-utils = { workspace = true } ya-client-model = { workspace = true, features = ["with-diesel"] } -ya-core-model = { version = "0.10", features = [ +ya-core-model = { workspace = true, features = [ "activity", "driver", "identity", @@ -66,7 +66,7 @@ uint = "0.7" uuid = { version = "0.8", features = ["v4"] } [dev-dependencies] -ya-client = "0.9" +ya-client = { workspace = true } ya-dummy-driver = "0.3" ya-erc20-driver = "0.4" ya-net = { version = "0.3", features = ["service"] } diff --git a/core/persistence/Cargo.toml b/core/persistence/Cargo.toml index 3f5f5400b3..e05d6a3931 100644 --- a/core/persistence/Cargo.toml +++ b/core/persistence/Cargo.toml @@ -15,7 +15,7 @@ service = [ [dependencies] ya-client-model = { workspace = true, features = ["with-diesel"] } -ya-core-model = { version = "0.10" } +ya-core-model = { workspace = true } ya-service-api = { version = "0.1", optional = true } ya-service-api-interfaces = { version = "0.2", optional = true } ya-utils-process = { version = "0.3", features = ["lock"], optional = true } diff --git a/core/serv-api/Cargo.toml b/core/serv-api/Cargo.toml index 05013bf9aa..3f6d97e568 100644 --- a/core/serv-api/Cargo.toml +++ b/core/serv-api/Cargo.toml @@ -12,4 +12,4 @@ serde = { version = "1.0", features = ["derive"] } url = "2.1.1" ya-utils-cli = "0.1" -ya-core-model = "0.10" +ya-core-model = { workspace = true } diff --git a/core/serv-api/web/Cargo.toml b/core/serv-api/web/Cargo.toml index fcde51c4ec..7683913e3a 100644 --- a/core/serv-api/web/Cargo.toml +++ b/core/serv-api/web/Cargo.toml @@ -7,7 +7,7 @@ edition = "2018" [dependencies] ya-client.workspace = true -ya-core-model = { version = "0.10", features = ["appkey"] } +ya-core-model = { workspace = true, features = ["appkey"] } ya-service-api = "0.1" ya-service-bus = { workspace = true } diff --git a/core/sgx/Cargo.toml b/core/sgx/Cargo.toml index cf01fcf5ec..e54cc2a350 100644 --- a/core/sgx/Cargo.toml +++ b/core/sgx/Cargo.toml @@ -7,6 +7,6 @@ edition = "2018" [dependencies] anyhow = "1.0" graphene-sgx = { version = "0.3.3", features = ["ias"] } -ya-client-model = "0.6" -ya-core-model = { version = "^0.9", features = ["sgx"] } +ya-client-model = { workspace = true } +ya-core-model = { workspace = true, features = ["sgx"] } ya-service-bus = { workspace = true } diff --git a/core/version/Cargo.toml b/core/version/Cargo.toml index 6c62525b39..68ca971a8d 100644 --- a/core/version/Cargo.toml +++ b/core/version/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" [dependencies] ya-client.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { version = "0.10", features = ["version"] } +ya-core-model = { workspace = true, features = ["version"] } ya-persistence = "0.3" ya-service-api = "0.1" ya-service-api-interfaces = "0.2" diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index 93accdb754..ac2ffa89ef 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-core-model = { version = "0.10", features = ["activity", "market"] } +ya-core-model = { workspace = true, features = ["activity", "market"] } ya-client-model = { workspace = true, features = ["sgx"] } ya-net = "0.3" ya-persistence = "0.3" @@ -44,7 +44,7 @@ packet-trace-enable = ["ya-packet-trace/enable"] default = [] [dev-dependencies] -ya-client = "0.9" +ya-client = { workspace = true } actix-rt = "2.7" sha3 = "0.8.2" structopt = "0.3" diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index 4ee9dc8b14..b47daadeae 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -31,7 +31,7 @@ ya-agreement-utils = { workspace = true } ya-manifest-utils = { version = "0.2" } ya-client-model.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { version = "0.10", features = ["activity", "appkey"] } +ya-core-model = { workspace = true, features = ["activity", "appkey"] } ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ "server", ] } diff --git a/exe-unit/components/gsb-http-proxy/Cargo.toml b/exe-unit/components/gsb-http-proxy/Cargo.toml index 4e532c73a7..ced2888965 100644 --- a/exe-unit/components/gsb-http-proxy/Cargo.toml +++ b/exe-unit/components/gsb-http-proxy/Cargo.toml @@ -7,8 +7,8 @@ edition = "2021" [dependencies] ya-service-bus = { workspace = true } ya-counters = { path = "../counters" } -ya-client-model = "0.7.0" -ya-core-model = { version = "0.10.0", features = ["net"] } +ya-client-model = { workspace = true } +ya-core-model = { workspace = true, features = ["net"] } thiserror = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index 406b375ff6..1c09e3156b 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -6,8 +6,9 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client-model = "0.6" -ya-core-model = { version = "^0.9" } +ya-client = { workspace = true } +ya-client-model = { workspace = true } +ya-core-model = { workspace = true, features = ["gftp"] } ya-service-bus = { workspace = true } ya-utils-path = { version = "0.1", path = "../../../utils/path" } ya-utils-futures.workspace = true diff --git a/golem_cli/Cargo.toml b/golem_cli/Cargo.toml index 59384a0378..2a5da6bb02 100644 --- a/golem_cli/Cargo.toml +++ b/golem_cli/Cargo.toml @@ -6,9 +6,9 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = { version = "0.8", features = ['cli'] } +ya-client = { workspace = true, features = ['cli'] } ya-compile-time-utils = "0.2" -ya-core-model = { version = "0.10", features = ["payment", "version"] } +ya-core-model = { workspace = true, features = ["payment", "version"] } ya-provider = "0.3" ya-utils-path = "0.1.0" ya-utils-process = { version = "0.3", features = ["lock"] } diff --git a/golem_cli/src/command/yagna.rs b/golem_cli/src/command/yagna.rs index 3f48290a38..9e41fa1f04 100644 --- a/golem_cli/src/command/yagna.rs +++ b/golem_cli/src/command/yagna.rs @@ -152,7 +152,11 @@ lazy_static! { ); ngm.insert( NetworkGroup::Testnet, - vec![NetworkName::Holesky, NetworkName::Amoy, NetworkName::Sepolia], + vec![ + NetworkName::Holesky, + NetworkName::Amoy, + NetworkName::Sepolia, + ], ); ngm }; diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index 6ad09f8c21..1cb3f35556 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -9,9 +9,9 @@ edition = "2018" bcast-singleton = [] [dependencies] -ya-client-model = "0.6" -ya-client = "0.8" -ya-core-model = { version = "^0.9" } +ya-client-model.workspace = true +ya-client.workspace = true +ya-core-model = { workspace = true, features = ["net"] } ya-service-bus = { workspace = true } ya-utils-path = { version = "0.1", path = "../../../utils/path" } ya-utils-futures.workspace = true diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index 9fd64a3089..22dabf909f 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -6,9 +6,9 @@ edition = "2021" [dependencies] ya-agreement-utils = { workspace = true } -ya-client-model = "0.6" -ya-client = "0.8" -ya-core-model = { version = "^0.9" } +ya-client-model = { workspace = true } +ya-client = { workspace = true } +ya-core-model = { workspace = true } ya-dummy-driver = "0.3" ya-erc20-driver = "0.4" ya-identity = "0.3" diff --git a/test-utils/test-framework/framework-mocks/src/identity.rs b/test-utils/test-framework/framework-mocks/src/identity.rs index a720188384..e6e7bb12fc 100644 --- a/test-utils/test-framework/framework-mocks/src/identity.rs +++ b/test-utils/test-framework/framework-mocks/src/identity.rs @@ -125,6 +125,7 @@ impl RealIdentity { password: None, from_keystore: None, from_private_key: None, + set_default: false, }; self.run_create_identity(command).await @@ -141,6 +142,7 @@ impl RealIdentity { password: None, from_keystore: None, from_private_key: Some(private_key), + set_default: false, }; self.run_create_identity(command).await diff --git a/test-utils/test-framework/framework-mocks/src/payment.rs b/test-utils/test-framework/framework-mocks/src/payment.rs index f5883139a3..7c7f75dd7c 100644 --- a/test-utils/test-framework/framework-mocks/src/payment.rs +++ b/test-utils/test-framework/framework-mocks/src/payment.rs @@ -106,6 +106,7 @@ impl RealPayment { address.to_string(), Some("holesky".to_string()), None, + false, )) .await??; Ok(()) From 5ef684f96b56268bdbc207687fe01988349d5748 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 6 Sep 2024 19:52:55 +0200 Subject: [PATCH 107/125] Check with proper ya-client --- Cargo.lock | 10 ++-------- Cargo.toml | 4 ++-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce5f385cdb..19e4a2289f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8762,8 +8762,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1db3f27d04b90bdd62583dee20684a6f147afdc26ebed1d4869d302d51f935" +source = "git+https://github.com/golemfactory/ya-client.git?rev=61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8#61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" dependencies = [ "actix-codec", "awc", @@ -8787,7 +8786,7 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=acccda451fb9fb796bac939f6379f9445dc74f11#acccda451fb9fb796bac939f6379f9445dc74f11" +source = "git+https://github.com/golemfactory/ya-client.git?rev=61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8#61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" dependencies = [ "bigdecimal 0.2.2", "chrono", @@ -10411,8 +10410,3 @@ dependencies = [ "cc", "pkg-config", ] - -[[patch.unused]] -name = "ya-client" -version = "0.8.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=acccda451fb9fb796bac939f6379f9445dc74f11#acccda451fb9fb796bac939f6379f9445dc74f11" diff --git a/Cargo.toml b/Cargo.toml index a1d0f6b970..9b8014f6e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -297,9 +297,9 @@ ya-service-api-interfaces = { path = "core/serv-api/interfaces" } ya-service-api-web = { path = "core/serv-api/web" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "acccda451fb9fb796bac939f6379f9445dc74f11" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" } #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "acccda451fb9fb796bac939f6379f9445dc74f11" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" } #ya-client-model = "0.7" golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "a410048b469b2cd3b1158bbec98808cf987d3f6b" } From c669c5ec2ec5a5912e5188e333bd93bada9d9575 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 6 Sep 2024 19:56:26 +0200 Subject: [PATCH 108/125] Check with proper golem certificate --- Cargo.lock | 2 +- Cargo.toml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19e4a2289f..38819f758a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3043,7 +3043,7 @@ dependencies = [ [[package]] name = "golem-certificate" version = "0.1.1" -source = "git+https://github.com/golemfactory/golem-certificate.git?rev=a410048b469b2cd3b1158bbec98808cf987d3f6b#a410048b469b2cd3b1158bbec98808cf987d3f6b" +source = "git+https://github.com/golemfactory/golem-certificate.git?rev=2723e07d205712d8e09e9d396c9745a059b60913#2723e07d205712d8e09e9d396c9745a059b60913" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 9b8014f6e3..769e7a6ea8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -300,8 +300,7 @@ ya-service-api-web = { path = "core/serv-api/web" } ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" } #ya-client = { path = "../ya-client" } ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" } -#ya-client-model = "0.7" -golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "a410048b469b2cd3b1158bbec98808cf987d3f6b" } +golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "2723e07d205712d8e09e9d396c9745a059b60913" } ## RELAY and networking stack From 6a4e76dd22df65850ec40ce4a7c17eaf3b42ad7a Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 6 Sep 2024 19:59:57 +0200 Subject: [PATCH 109/125] Fix example --- core/payment/examples/payment_api.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/payment/examples/payment_api.rs b/core/payment/examples/payment_api.rs index 93dd3f764c..0e49c3e70f 100644 --- a/core/payment/examples/payment_api.rs +++ b/core/payment/examples/payment_api.rs @@ -281,6 +281,7 @@ async fn main() -> anyhow::Result<()> { requestor_addr.clone(), args.network.clone(), None, + false, )) .await??; bus::service(driver_bus_id(driver_name)) From f202b143681e1855b01ff82d0ffb5e6ee6859c6e Mon Sep 17 00:00:00 2001 From: scx1332 Date: Mon, 9 Sep 2024 11:37:28 +0200 Subject: [PATCH 110/125] Point to commits on master for ya-client and golem-certificate (no code change) --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 38819f758a..40c9ef81cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3043,7 +3043,7 @@ dependencies = [ [[package]] name = "golem-certificate" version = "0.1.1" -source = "git+https://github.com/golemfactory/golem-certificate.git?rev=2723e07d205712d8e09e9d396c9745a059b60913#2723e07d205712d8e09e9d396c9745a059b60913" +source = "git+https://github.com/golemfactory/golem-certificate.git?rev=952fdbd47adc57e46b7370935111e046271ef415#952fdbd47adc57e46b7370935111e046271ef415" dependencies = [ "anyhow", "chrono", @@ -8762,7 +8762,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.9.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8#61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" +source = "git+https://github.com/golemfactory/ya-client.git?rev=c8675c4eb0d42119b1cfa2f5772ba91f877d81f9#c8675c4eb0d42119b1cfa2f5772ba91f877d81f9" dependencies = [ "actix-codec", "awc", @@ -8786,7 +8786,7 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8#61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" +source = "git+https://github.com/golemfactory/ya-client.git?rev=c8675c4eb0d42119b1cfa2f5772ba91f877d81f9#c8675c4eb0d42119b1cfa2f5772ba91f877d81f9" dependencies = [ "bigdecimal 0.2.2", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 769e7a6ea8..cd73657cda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -297,10 +297,10 @@ ya-service-api-interfaces = { path = "core/serv-api/interfaces" } ya-service-api-web = { path = "core/serv-api/web" } ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "c8675c4eb0d42119b1cfa2f5772ba91f877d81f9" } #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "61bb3160b0e6dc17b22ef6b46f0bae4c6ebff3f8" } -golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "2723e07d205712d8e09e9d396c9745a059b60913" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "c8675c4eb0d42119b1cfa2f5772ba91f877d81f9" } +golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415" } ## RELAY and networking stack From 13b56ffdfef5a5417ae7c40dd36dce84cfcf64aa Mon Sep 17 00:00:00 2001 From: scx1332 Date: Mon, 9 Sep 2024 14:09:33 +0200 Subject: [PATCH 111/125] Use version from GitHub erc20_payment_lib instead from creates - migration should work fine now --- Cargo.lock | 106 ++++++----------------------------------------------- Cargo.toml | 8 ++-- 2 files changed, 15 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40c9ef81cf..600969b30e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -976,45 +976,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" -[[package]] -name = "bollard" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af254ed2da4936ef73309e9597180558821cb16ae9bba4cb24ce6b612d8d80ed" -dependencies = [ - "base64 0.21.7", - "bollard-stubs", - "bytes 1.6.0", - "futures-core", - "futures-util", - "hex", - "http 0.2.12", - "hyper", - "hyperlocal", - "log", - "pin-project-lite", - "serde", - "serde_derive", - "serde_json", - "serde_repr", - "serde_urlencoded", - "thiserror", - "tokio", - "tokio-util", - "url", - "winapi 0.3.9", -] - -[[package]] -name = "bollard-stubs" -version = "1.42.0-rc.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602bda35f33aeb571cef387dcd4042c643a8bf689d8aaac2cc47ea24cb7bc7e0" -dependencies = [ - "serde", - "serde_with 2.0.0", -] - [[package]] name = "borsh" version = "1.4.0" @@ -2175,9 +2136,8 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erc20_payment_lib" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da9f9d42de38a0db4d8be0f734e0421b475363a55e4ec56606e274df6263872" +version = "0.4.8" +source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=4200567b931af64f4fb1f6b756dd6d051576b64f#4200567b931af64f4fb1f6b756dd6d051576b64f" dependencies = [ "actix", "actix-files", @@ -2218,9 +2178,8 @@ dependencies = [ [[package]] name = "erc20_payment_lib_common" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0774c96878af1018f6b3ce756cc0334ff6f136dcf15c774f0879ae015771d277" +version = "0.4.8" +source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=4200567b931af64f4fb1f6b756dd6d051576b64f#4200567b931af64f4fb1f6b756dd6d051576b64f" dependencies = [ "actix-files", "actix-web", @@ -2260,9 +2219,8 @@ dependencies = [ [[package]] name = "erc20_payment_lib_extra" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2084a5f3075a8b90d7edc22f7a4a6136670a860693ad81f76ebaf2a9e94a34fc" +version = "0.4.8" +source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=4200567b931af64f4fb1f6b756dd6d051576b64f#4200567b931af64f4fb1f6b756dd6d051576b64f" dependencies = [ "actix-files", "actix-web", @@ -2297,9 +2255,8 @@ dependencies = [ [[package]] name = "erc20_processor" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b3d5bec86b99d8254ef9dce04710e33f18267a39da81ee7cf00ee0420f8e0d2" +version = "0.4.8" +source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=4200567b931af64f4fb1f6b756dd6d051576b64f#4200567b931af64f4fb1f6b756dd6d051576b64f" dependencies = [ "actix-cors", "actix-files", @@ -2346,15 +2303,13 @@ dependencies = [ [[package]] name = "erc20_rpc_pool" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cf619ab7b405fd49b8ee383395be68ffe944b13f4001c03abb34eabe0f2f73" +version = "0.4.8" +source = "git+https://github.com/golemfactory/erc20_payment_lib?rev=4200567b931af64f4fb1f6b756dd6d051576b64f#4200567b931af64f4fb1f6b756dd6d051576b64f" dependencies = [ "actix-files", "actix-web", "anyhow", "awc", - "bollard", "chrono", "dotenv", "env_logger 0.10.2", @@ -3489,19 +3444,6 @@ dependencies = [ "tokio-native-tls", ] -[[package]] -name = "hyperlocal" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" -dependencies = [ - "futures-util", - "hex", - "hyper", - "pin-project 1.1.5", - "tokio", -] - [[package]] name = "iana-time-zone" version = "0.1.60" @@ -6398,17 +6340,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "serde_repr" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.60", -] - [[package]] name = "serde_spanned" version = "0.6.5" @@ -6430,21 +6361,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_with" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89df7a26519371a3cce44fbb914c2819c84d9b897890987fa3ab096491cc0ea8" -dependencies = [ - "base64 0.13.1", - "chrono", - "hex", - "indexmap 1.9.3", - "serde", - "serde_json", - "time 0.3.36", -] - [[package]] name = "serde_with" version = "3.9.0" @@ -8799,7 +8715,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "serde_with 3.9.0", + "serde_with", "strum 0.24.1", "strum_macros 0.24.3", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index cd73657cda..01737b5132 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -239,12 +239,12 @@ members = [ # diesel 1.4.* supports up to 0.23.0, but sqlx 0.5.9 requires 0.22.0 # sqlx 0.5.10 need 0.23.2, so 0.5.9 is last version possible derive_more = "0.99.11" -#erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "79955d735f64467b01f04abfc70da39e4ba4be3c" } -#erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "79955d735f64467b01f04abfc70da39e4ba4be3c" } +erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } +erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } #erc20_payment_lib = { path = "../../payments/erc20_payment_lib/crates/erc20_payment_lib" } #erc20_processor = { path = "../../payments/erc20_payment_lib" } -erc20_payment_lib = { version = "0.4.7" } -erc20_processor = { version = "0.4.7" } +#erc20_payment_lib = { version = "0.4.7" } +#erc20_processor = { version = "0.4.7" } gftp = { version = "0.4.1", path = "core/gftp" } hex = "0.4.3" libsqlite3-sys = { version = "0.26.0", features = ["bundled"] } From e8c7acc400376d6e55054d498bddb23f752a8179 Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Thu, 12 Sep 2024 10:50:51 +0200 Subject: [PATCH 112/125] exe-unit: progress merge fixups --- Cargo.lock | 268 +++++++++--- Cargo.toml | 433 ++++++++++---------- exe-unit/Cargo.toml | 106 ++--- exe-unit/components/mock-runtime/Cargo.toml | 30 +- exe-unit/src/exe_unit.rs | 37 +- exe-unit/src/lib.rs | 8 +- 6 files changed, 525 insertions(+), 357 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dcd9a373c6..cc02d13360 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1163,6 +1163,52 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "camino" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "cargo_metadata" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d886547e41f740c616ae73108f6eb70afe6d940c7bc697cb30f13daec073037" +dependencies = [ + "camino", + "cargo-platform", + "semver 1.0.23", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.1.18" @@ -2957,7 +3003,7 @@ dependencies = [ "tokio", "url", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-service-bus", ] @@ -3024,7 +3070,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] @@ -3061,7 +3107,7 @@ dependencies = [ "url", "ya-client", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-provider", "ya-utils-path", "ya-utils-process 0.3.0", @@ -3452,6 +3498,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime 2.1.0", + "serde", +] + [[package]] name = "hyper" version = "0.14.30" @@ -7476,6 +7532,18 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +[[package]] +name = "test-binary" +version = "3.0.2" +source = "git+https://github.com/golemfactory/test-binary.git#c9ebfa3e257455f8365e042b8838a518f2106169" +dependencies = [ + "camino", + "cargo_metadata 0.15.4", + "once_cell", + "paste", + "thiserror", +] + [[package]] name = "test-case" version = "2.2.2" @@ -8765,8 +8833,8 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-gsb-http-proxy", "ya-net", "ya-persistence", @@ -8790,13 +8858,13 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", ] [[package]] name = "ya-client" version = "0.9.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=b5369b76044f0a1584532f603c542587e15be852#b5369b76044f0a1584532f603c542587e15be852" +source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" dependencies = [ "actix-codec", "awc", @@ -8814,19 +8882,37 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model", + "ya-client-model 0.7.0", +] + +[[package]] +name = "ya-client-model" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe85a762be0297b9848ad0b7f1c73cb2afe778bb8eb3a952efd032792631a3e0" +dependencies = [ + "bigdecimal 0.2.2", + "chrono", + "derive_more", + "rand 0.8.5", + "serde", + "serde_json", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", ] [[package]] name = "ya-client-model" version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=b5369b76044f0a1584532f603c542587e15be852#b5369b76044f0a1584532f603c542587e15be852" +source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" dependencies = [ "bigdecimal 0.2.2", "chrono", "derive_more", "diesel", "hex", + "humantime-serde", "openssl", "rand 0.8.5", "secp256k1 0.27.0", @@ -8849,6 +8935,26 @@ dependencies = [ "vergen", ] +[[package]] +name = "ya-core-model" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed4e52e7d5cf8bafaf04daf6abf6bf5b199d291d65d014b7b8ca409ad7b0c599" +dependencies = [ + "chrono", + "derive_more", + "log", + "serde", + "serde_bytes", + "serde_json", + "structopt", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", + "ya-client-model 0.6.0", + "ya-service-bus", +] + [[package]] name = "ya-core-model" version = "0.10.0" @@ -8869,7 +8975,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model", + "ya-client-model 0.7.0", "ya-service-bus", ] @@ -8921,8 +9027,8 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-payment-driver", "ya-persistence", "ya-service-api-interfaces", @@ -8966,8 +9072,8 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -9007,12 +9113,14 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.8.26", + "serial_test 0.5.1 (git+https://github.com/tworec/serial_test.git?branch=actix_rt_test)", "sha3 0.8.2", "shell-words", "signal-hook", "socket2 0.4.10", "structopt", "tempdir", + "test-context", "thiserror", "tokio", "tokio-stream", @@ -9020,14 +9128,16 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-counters", + "ya-framework-basic", "ya-gsb-http-proxy", "ya-manifest-utils", + "ya-mock-runtime", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-sb-router", "ya-service-bus", "ya-std-utils", @@ -9072,6 +9182,7 @@ dependencies = [ "async-trait", "awc", "bytes 1.7.1", + "cargo_metadata 0.18.1", "crossterm 0.26.1", "env_logger 0.7.1", "futures 0.3.30", @@ -9089,6 +9200,7 @@ dependencies = [ "sha2 0.8.2", "sha3 0.8.2", "tempdir", + "test-binary", "test-context", "thiserror", "tokio", @@ -9097,8 +9209,8 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-net", "ya-service-bus", "ya-utils-futures", @@ -9143,8 +9255,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9187,8 +9299,8 @@ dependencies = [ "thiserror", "tokio", "uuid 1.10.0", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -9225,8 +9337,8 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-counters", "ya-framework-basic", "ya-sb-router", @@ -9267,8 +9379,8 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-framework-basic", "ya-framework-mocks", "ya-persistence", @@ -9322,7 +9434,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model", + "ya-client-model 0.7.0", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9374,7 +9486,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-core-model", + "ya-core-model 0.10.0", "ya-diesel-utils", "ya-framework-basic", "ya-framework-mocks", @@ -9424,12 +9536,42 @@ dependencies = [ "structopt", "tokio", "url", - "ya-core-model", + "ya-core-model 0.10.0", "ya-service-api", "ya-service-api-interfaces", "ya-service-bus", ] +[[package]] +name = "ya-mock-runtime" +version = "0.1.0" +dependencies = [ + "actix", + "anyhow", + "async-trait", + "bytes 1.7.1", + "env_logger 0.10.2", + "futures 0.3.30", + "hex", + "log", + "portpicker", + "rand 0.8.5", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-util", + "url", + "uuid 0.8.2", + "ya-client-model 0.7.0", + "ya-core-model 0.9.1", + "ya-exe-unit", + "ya-framework-basic", + "ya-runtime-api 0.7.0", + "ya-sb-router", + "ya-service-bus", +] + [[package]] name = "ya-net" version = "0.3.0" @@ -9459,8 +9601,8 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", "ya-sb-proto", @@ -9537,8 +9679,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9583,8 +9725,8 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-bus", ] @@ -9607,8 +9749,8 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-service-api", "ya-service-api-interfaces", "ya-utils-process 0.3.0", @@ -9675,9 +9817,9 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-file-logging", "ya-framework-basic", "ya-manifest-test-utils", @@ -9746,7 +9888,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model", + "ya-client-model 0.7.0", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -9830,6 +9972,25 @@ dependencies = [ "derive_more", ] +[[package]] +name = "ya-runtime-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0db25811f107d62be6c6ac7444d9c6c3e39714b6f76d72798b66ecce47506f" +dependencies = [ + "anyhow", + "bytes 1.7.1", + "futures 0.3.30", + "log", + "prost 0.10.4", + "prost-build 0.10.4", + "serde", + "serde_json", + "tokio", + "tokio-util", + "url", +] + [[package]] name = "ya-runtime-api" version = "0.7.1" @@ -9917,7 +10078,7 @@ dependencies = [ "lazy_static", "serde", "url", - "ya-core-model", + "ya-core-model 0.10.0", "ya-utils-cli", ] @@ -9968,7 +10129,7 @@ dependencies = [ "structopt", "url", "ya-client", - "ya-core-model", + "ya-core-model 0.10.0", "ya-identity", "ya-persistence", "ya-sb-router", @@ -10011,8 +10172,8 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-service-bus", ] @@ -10096,16 +10257,17 @@ dependencies = [ "test-context", "thiserror", "tokio", + "tokio-stream", "tokio-tar", "tokio-util", "url", "walkdir", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-exe-unit", "ya-framework-basic", - "ya-runtime-api", + "ya-runtime-api 0.7.1", "ya-service-bus", "ya-utils-futures", "ya-utils-path", @@ -10229,7 +10391,7 @@ dependencies = [ "tokio", "ya-client", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -10264,8 +10426,8 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model", - "ya-core-model", + "ya-client-model 0.7.0", + "ya-core-model 0.10.0", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-persistence", @@ -10307,9 +10469,9 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model", + "ya-client-model 0.7.0", "ya-compile-time-utils", - "ya-core-model", + "ya-core-model 0.10.0", "ya-dummy-driver", "ya-erc20-driver", "ya-exe-unit", diff --git a/Cargo.toml b/Cargo.toml index 76fe508d37..62c16458f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,28 +1,28 @@ [package] -name = "yagna" -version = "0.16.0" -description = "Open platform and marketplace for distributed computations" -readme = "README.md" authors = ["Golem Factory "] +description = "Open platform and marketplace for distributed computations" +edition = "2018" homepage = "https://github.com/golemfactory/yagna/core/serv" -repository = "https://github.com/golemfactory/yagna" license = "GPL-3.0" -edition = "2018" +name = "yagna" +readme = "README.md" +repository = "https://github.com/golemfactory/yagna" +version = "0.16.0" [features] default = ['erc20-driver', 'gftp/bin'] -static-openssl = ["openssl/vendored", "openssl-probe"] dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] +framework-test = ['ya-exe-unit/system-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] +static-openssl = ["openssl/vendored", "openssl-probe"] tos = [] -framework-test = ['ya-exe-unit/framework-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ - "ya-vpn/packet-trace-enable", - "ya-file-logging/packet-trace-enable", - "ya-net/packet-trace-enable", - "ya-service-bus/packet-trace-enable", + "ya-vpn/packet-trace-enable", + "ya-file-logging/packet-trace-enable", + "ya-net/packet-trace-enable", + "ya-service-bus/packet-trace-enable", ] [[bin]] @@ -30,40 +30,40 @@ name = "yagna" path = "core/serv/src/main.rs" [dependencies] +gftp = {workspace = true, optional = true} ya-activity = "0.4" +ya-client-model.workspace = true +ya-client.workspace = true ya-compile-time-utils = "0.2" ya-core-model.workspace = true -ya-dummy-driver = { version = "0.3", optional = true } +ya-dummy-driver = {version = "0.3", optional = true} +ya-erc20-driver = {version = "0.4", optional = true} +ya-fd-metrics = {path = "utils/fd-metrics"} ya-file-logging = "0.1" ya-gsb-api = "0.1" -ya-erc20-driver = { version = "0.4", optional = true } ya-identity = "0.3" ya-market = "0.4" ya-metrics = "0.2" -ya-net = { version = "0.3", features = ["service"] } +ya-net = {version = "0.3", features = ["service"]} ya-payment = "0.3" -ya-persistence = { version = "0.3", features = ["service"] } -ya-sb-proto = { workspace = true } -ya-sb-router = { workspace = true } +ya-persistence = {version = "0.3", features = ["service"]} +ya-sb-proto = {workspace = true} +ya-sb-router = {workspace = true} ya-service-api = "0.1" ya-service-api-derive = "0.2" ya-service-api-interfaces = "0.2" ya-service-api-web = "0.2" -ya-service-bus = { workspace = true } +ya-service-bus = {workspace = true} ya-sgx = "0.2" -ya-utils-path = "0.1" ya-utils-futures.workspace = true -ya-utils-process = { version = "0.2", features = ["lock"] } ya-utils-networking = "0.2" -ya-fd-metrics = { path = "utils/fd-metrics" } +ya-utils-path = "0.1" +ya-utils-process = {version = "0.2", features = ["lock"]} ya-version = "0.2" ya-vpn = "0.2" -ya-client.workspace = true -ya-client-model.workspace = true -gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb -ya-provider = { version = "0.3", optional = true } # just to enable conditionally running some tests -ya-exe-unit = { version = "0.4", optional = true, path = "exe-unit" } # just to enable conditionally running some tests +ya-exe-unit = {version = "0.4", optional = true, path = "exe-unit"}# just to enable conditionally running some tests +ya-provider = {version = "0.3", optional = true}# just to enable conditionally running some tests actix-rt = "2.7" actix-service = "2" @@ -74,163 +74,161 @@ directories = "2.0.2" dotenv = "0.15.0" futures = "0.3" lazy_static = "1.4" +libsqlite3-sys = {workspace = true} log = "0.4" metrics = "0.12" num_cpus = "1" +openssl-probe = {version = "0.1", optional = true} openssl.workspace = true -openssl-probe = { version = "0.1", optional = true } serde = "1.0" serde_json = "1.0" structopt = "0.3" -tokio = { version = "1", features = ["net"] } -tokio-util = { version = "0.7", features = ["codec"] } -tokio-stream = { version = "0.1.8", features = ["io-util"] } +tokio = {version = "1", features = ["net"]} +tokio-stream = {version = "0.1.8", features = ["io-util"]} +tokio-util = {version = "0.7", features = ["codec"]} url = "2.1.1" -libsqlite3-sys = { workspace = true } - [dev-dependencies] -erc20_processor = { workspace = true } +erc20_processor = {workspace = true} +ya-exe-unit = {version = "0.4", path = "exe-unit"} ya-test-framework = "0.1" -ya-exe-unit = { version = "0.4", path = "exe-unit" } [package.metadata.deb] -name = "golem-requestor" assets = [ - [ - "target/release/yagna", - "usr/bin/", - "755", - ], - [ - "target/release/gftp", - "usr/bin/", - "755", - ], - [ - "README.md", - "usr/share/doc/yagna/", - "644", - ], - [ - "core/serv/README.md", - "usr/share/doc/yagna/service.md", - "644", - ], + [ + "target/release/yagna", + "usr/bin/", + "755", + ], + [ + "target/release/gftp", + "usr/bin/", + "755", + ], + [ + "README.md", + "usr/share/doc/yagna/", + "644", + ], + [ + "core/serv/README.md", + "usr/share/doc/yagna/service.md", + "644", + ], ] conflicts = "ya-provider" -features = ["static-openssl"] -maintainer-scripts = "debian/core" depends = "libgcc1, libc6 (>= 2.23)" extended-description = """The Next Golem Milestone. An open platform and marketplace for distributed computations. """ - +features = ["static-openssl"] +maintainer-scripts = "debian/core" +name = "golem-requestor" [package.metadata.deb.variants.provider] -name = "golem-provider" -replaces = "golem-requestor" -maintainer-scripts = "debian/provider" -features = ["static-openssl"] -depends = "libgcc1, libc6 (>= 2.23)" assets = [ - [ - "target/release/yagna", - "usr/bin/", - "755", - ], - [ - "target/release/ya-provider", - "usr/bin/", - "755", - ], - [ - "target/release/gftp", - "usr/bin/", - "755", - ], - [ - "target/release/exe-unit", - "usr/lib/yagna/plugins/", - "755", - ], - [ - "target/release/golemsp", - "usr/bin/", - "755", - ], - [ - "README.md", - "usr/share/doc/yagna/", - "644", - ], - [ - "core/serv/README.md", - "usr/share/doc/yagna/service.md", - "644", - ], - [ - "agent/provider/readme.md", - "usr/share/doc/yagna/run-provider.md", - "644", - ], + [ + "target/release/yagna", + "usr/bin/", + "755", + ], + [ + "target/release/ya-provider", + "usr/bin/", + "755", + ], + [ + "target/release/gftp", + "usr/bin/", + "755", + ], + [ + "target/release/exe-unit", + "usr/lib/yagna/plugins/", + "755", + ], + [ + "target/release/golemsp", + "usr/bin/", + "755", + ], + [ + "README.md", + "usr/share/doc/yagna/", + "644", + ], + [ + "core/serv/README.md", + "usr/share/doc/yagna/service.md", + "644", + ], + [ + "agent/provider/readme.md", + "usr/share/doc/yagna/run-provider.md", + "644", + ], ] +depends = "libgcc1, libc6 (>= 2.23)" +features = ["static-openssl"] +maintainer-scripts = "debian/provider" +name = "golem-provider" +replaces = "golem-requestor" [workspace.lints.clippy] arc_with_non_send_sync = "allow" -get_first = "allow" blocks_in_conditions = "allow" +get_first = "allow" [workspace] members = [ - "agent/provider", - "core/activity", - "core/gftp", - "core/gsb-api", - "core/identity", - "core/market", - "core/market/resolver", - "core/model", - "core/net", - "core/payment", - "core/payment-driver/base", - "core/payment-driver/dummy", - "core/payment-driver/erc20", - "core/persistence", - "core/serv-api", - "core/serv-api/derive", - "core/serv-api/interfaces", - "core/serv-api/web", - "core/sgx", - "core/version", - "core/vpn", - "exe-unit/components/counters", - "exe-unit/components/gsb-http-proxy", - "exe-unit", - "exe-unit/runtime-api", - "exe-unit/tokio-process-ns", - "exe-unit/components/transfer", - "golem_cli", - "utils/actix_utils", - "utils/agreement-utils", - "utils/cli", - "utils/compile-time-utils", - "utils/file-logging", - "utils/futures", - "utils/manifest-utils", - "utils/manifest-utils/test-utils", - "utils/networking", - "utils/path", - "utils/process", - "utils/std-utils", - "utils/diesel-utils", - "utils/fd-metrics", - "core/metrics", - "test-utils/test-framework", - "test-utils/test-framework/framework-macro", - "test-utils/test-framework/framework-basic", - "test-utils/test-framework/framework-mocks", + "agent/provider", + "core/activity", + "core/gftp", + "core/gsb-api", + "core/identity", + "core/market", + "core/market/resolver", + "core/model", + "core/net", + "core/payment", + "core/payment-driver/base", + "core/payment-driver/dummy", + "core/payment-driver/erc20", + "core/persistence", + "core/serv-api", + "core/serv-api/derive", + "core/serv-api/interfaces", + "core/serv-api/web", + "core/sgx", + "core/version", + "core/vpn", + "exe-unit/components/counters", + "exe-unit/components/gsb-http-proxy", + "exe-unit", + "exe-unit/runtime-api", + "exe-unit/tokio-process-ns", + "exe-unit/components/transfer", + "golem_cli", + "utils/actix_utils", + "utils/agreement-utils", + "utils/cli", + "utils/compile-time-utils", + "utils/file-logging", + "utils/futures", + "utils/manifest-utils", + "utils/manifest-utils/test-utils", + "utils/networking", + "utils/path", + "utils/process", + "utils/std-utils", + "utils/diesel-utils", + "utils/fd-metrics", + "core/metrics", + "test-utils/test-framework", + "test-utils/test-framework/framework-macro", + "test-utils/test-framework/framework-basic", + "test-utils/test-framework/framework-mocks", ] [workspace.dependencies] @@ -238,34 +236,34 @@ members = [ # diesel 1.4.* supports up to 0.23.0, but sqlx 0.5.9 requires 0.22.0 # sqlx 0.5.10 need 0.23.2, so 0.5.9 is last version possible derive_more = "0.99.11" -erc20_payment_lib = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } -erc20_processor = { git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f" } +erc20_payment_lib = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} +erc20_processor = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} #erc20_payment_lib = { path = "../../payments/erc20_payment_lib/crates/erc20_payment_lib" } #erc20_processor = { path = "../../payments/erc20_payment_lib" } #erc20_payment_lib = { version = "0.4.7" } #erc20_processor = { version = "0.4.7" } -gftp = { version = "0.4.1", path = "core/gftp" } +gftp = {version = "0.4.1", path = "core/gftp"} hex = "0.4.3" -libsqlite3-sys = { version = "0.26.0", features = ["bundled"] } +libsqlite3-sys = {version = "0.26.0", features = ["bundled"]} openssl = "0.10" rand = "0.8.5" -strum = { version = "0.24", features = ["derive"] } +regex = "1.10.4" +strum = {version = "0.24", features = ["derive"]} trust-dns-resolver = "0.22" url = "2.3.1" -regex = "1.10.4" -ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } -ya-core-model = { version = "0.10", path = "core/model" } -ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489" } -ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } -ya-utils-futures = { path = "utils/futures" } +ya-agreement-utils = {version = "0.6", path = "utils/agreement-utils"} +ya-core-model = {version = "0.10", path = "core/model"} +ya-relay-client = {git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489"} +ya-relay-stack = {git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5"} +ya-utils-futures = {path = "utils/futures"} -ya-service-bus = { version = "0.7.3", features = ['tls'] } -ya-sb-router = { version = "0.6.4" } -ya-sb-proto = { version = "0.6.2" } -ya-sb-util = { version = "0.5.1" } -parking_lot = "0.12.3" mime = "0.3.17" +parking_lot = "0.12.3" +ya-sb-proto = {version = "0.6.2"} +ya-sb-router = {version = "0.6.4"} +ya-sb-util = {version = "0.5.1"} +ya-service-bus = {version = "0.7.3", features = ['tls']} # true version is given in patches section ya-client = "0.9" # true version is given in patches section @@ -273,34 +271,34 @@ ya-client-model = "0.7" [patch.crates-io] ## SERVICES -ya-identity = { path = "core/identity" } -ya-net = { path = "core/net" } -ya-market = { path = "core/market" } -ya-market-resolver = { path = "core/market/resolver" } -ya-activity = { path = "core/activity" } -ya-sgx = { path = "core/sgx" } -ya-payment = { path = "core/payment" } -ya-payment-driver = { path = "core/payment-driver/base" } -ya-dummy-driver = { path = "core/payment-driver/dummy" } -ya-erc20-driver = { path = "core/payment-driver/erc20" } -ya-version = { path = "core/version" } -ya-vpn = { path = "core/vpn" } -ya-gsb-api = { path = "core/gsb-api" } +ya-activity = {path = "core/activity"} +ya-dummy-driver = {path = "core/payment-driver/dummy"} +ya-erc20-driver = {path = "core/payment-driver/erc20"} +ya-gsb-api = {path = "core/gsb-api"} +ya-identity = {path = "core/identity"} +ya-market = {path = "core/market"} +ya-market-resolver = {path = "core/market/resolver"} +ya-net = {path = "core/net"} +ya-payment = {path = "core/payment"} +ya-payment-driver = {path = "core/payment-driver/base"} +ya-sgx = {path = "core/sgx"} +ya-version = {path = "core/version"} +ya-vpn = {path = "core/vpn"} ## CORE UTILS -ya-core-model = { path = "core/model" } -ya-persistence = { path = "core/persistence" } -ya-service-api = { path = "core/serv-api" } -ya-service-api-derive = { path = "core/serv-api/derive" } -ya-service-api-interfaces = { path = "core/serv-api/interfaces" } -ya-service-api-web = { path = "core/serv-api/web" } +ya-core-model = {path = "core/model"} +ya-persistence = {path = "core/persistence"} +ya-service-api = {path = "core/serv-api"} +ya-service-api-derive = {path = "core/serv-api/derive"} +ya-service-api-interfaces = {path = "core/serv-api/interfaces"} +ya-service-api-web = {path = "core/serv-api/web"} ## CLIENT -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad" } +ya-client = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} #ya-client = { path = "../ya-client" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad" } +ya-client-model = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} #ya-client-model = { path = "../ya-client/model" } -golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415" } +golem-certificate = {git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415"} ## RELAY and networking stack @@ -309,39 +307,38 @@ golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.g #ya-relay-core = { path = "../ya-relay/crates/core" } #ya-relay-proto = { path = "../ya-relay/crates/proto" } - ## OTHERS -gftp = { path = "core/gftp" } -tokio-process-ns = { path = "exe-unit/tokio-process-ns" } -ya-agreement-utils = { path = "utils/agreement-utils" } -ya-std-utils = { path = "utils/std-utils" } -ya-compile-time-utils = { path = "utils/compile-time-utils" } -ya-exe-unit = { path = "exe-unit" } -ya-file-logging = { path = "utils/file-logging" } -ya-manifest-utils = { path = "utils/manifest-utils" } -ya-transfer = { path = "exe-unit/components/transfer" } -ya-utils-actix = { path = "utils/actix_utils" } -ya-utils-cli = { path = "utils/cli" } -ya-utils-networking = { path = "utils/networking" } -ya-utils-path = { path = "utils/path" } -ya-utils-process = { path = "utils/process" } -ya-diesel-utils = { path = "utils/diesel-utils" } -ya-metrics = { path = "core/metrics" } -ya-provider = { path = "agent/provider" } -ya-counters = { path = "exe-unit/components/counters" } -ya-gsb-http-proxy = { path = "exe-unit/components/gsb-http-proxy" } +gftp = {path = "core/gftp"} +tokio-process-ns = {path = "exe-unit/tokio-process-ns"} +ya-agreement-utils = {path = "utils/agreement-utils"} +ya-compile-time-utils = {path = "utils/compile-time-utils"} +ya-counters = {path = "exe-unit/components/counters"} +ya-diesel-utils = {path = "utils/diesel-utils"} +ya-exe-unit = {path = "exe-unit"} +ya-file-logging = {path = "utils/file-logging"} +ya-gsb-http-proxy = {path = "exe-unit/components/gsb-http-proxy"} +ya-manifest-utils = {path = "utils/manifest-utils"} +ya-metrics = {path = "core/metrics"} +ya-provider = {path = "agent/provider"} +ya-std-utils = {path = "utils/std-utils"} +ya-transfer = {path = "exe-unit/components/transfer"} +ya-utils-actix = {path = "utils/actix_utils"} +ya-utils-cli = {path = "utils/cli"} +ya-utils-networking = {path = "utils/networking"} +ya-utils-path = {path = "utils/path"} +ya-utils-process = {path = "utils/process"} ## TEST UTILS -ya-manifest-test-utils = { path = "utils/manifest-utils/test-utils" } -ya-test-framework = { path = "test-utils/test-framework" } -ya-framework-macro = { path = "test-utils/test-framework/framework-macro" } -ya-framework-basic = { path = "test-utils/test-framework/framework-basic" } -ya-framework-mocks = { path = "test-utils/test-framework/framework-mocks" } +ya-framework-basic = {path = "test-utils/test-framework/framework-basic"} +ya-framework-macro = {path = "test-utils/test-framework/framework-macro"} +ya-framework-mocks = {path = "test-utils/test-framework/framework-mocks"} +ya-manifest-test-utils = {path = "utils/manifest-utils/test-utils"} +ya-test-framework = {path = "test-utils/test-framework"} -ethereum-tx-sign = { git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195" } -graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407" } +ethereum-tx-sign = {git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195"} +graphene-sgx = {git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407"} -diesel = { git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563" } +diesel = {git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563"} # Speed up builds on macOS (will be default in next rust version probably) # https://jakedeichert.com/blog/reducing-rust-incremental-compilation-times-on-macos-by-70-percent/ diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index 53d30476d2..ff1eb42248 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "ya-exe-unit" -version = "0.4.0" authors = ["Golem Factory "] edition = "2018" +name = "ya-exe-unit" +version = "0.4.0" [lib] name = "ya_exe_unit" @@ -13,64 +13,64 @@ name = "exe-unit" path = "src/bin.rs" [features] -default = ['compat-deployment'] compat-deployment = [] +default = ['compat-deployment'] +packet-trace-enable = ["ya-packet-trace/enable"] sgx = [ - 'graphene-sgx', - 'openssl/vendored', - 'secp256k1/rand', - 'ya-client-model/sgx', - 'ya-core-model/sgx', - 'ya-transfer/sgx', + 'graphene-sgx', + 'openssl/vendored', + 'secp256k1/rand', + 'ya-client-model/sgx', + 'ya-core-model/sgx', + 'ya-transfer/sgx', ] -packet-trace-enable = ["ya-packet-trace/enable"] system-test = ["ya-transfer/system-test"] [dependencies] -ya-agreement-utils = { workspace = true } -ya-manifest-utils = { version = "0.2" } +ya-agreement-utils = {workspace = true} ya-client-model.workspace = true ya-compile-time-utils = "0.2" -ya-core-model = { workspace = true, features = ["activity", "appkey"] } -ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ - "server", -] } -ya-service-bus = { workspace = true } +ya-core-model = {workspace = true, features = ["activity", "appkey"]} +ya-counters = {path = "../exe-unit/components/counters", features = ["os"]} +ya-gsb-http-proxy = {path = "../exe-unit/components/gsb-http-proxy"} +ya-manifest-utils = {version = "0.2"} +ya-packet-trace = {git = "https://github.com/golemfactory/ya-packet-trace"} +ya-runtime-api = {version = "0.7", path = "runtime-api", features = [ + "server", +]} +ya-service-bus = {workspace = true} +ya-std-utils = "0.1" ya-transfer = "0.3" -ya-utils-path = "0.1" ya-utils-futures.workspace = true -ya-std-utils = "0.1" -ya-utils-networking = { version = "0.2", default-features = false, features = [ - "dns", - "vpn", -] } +ya-utils-networking = {version = "0.2", default-features = false, features = [ + "dns", + "vpn", +]} +ya-utils-path = "0.1" ya-utils-process = "0.3" -ya-packet-trace = { git = "https://github.com/golemfactory/ya-packet-trace" } -ya-gsb-http-proxy = { path = "../exe-unit/components/gsb-http-proxy" } -ya-counters = { path = "../exe-unit/components/counters", features = ["os"] } -actix = { version = "0.13", default-features = false } +actix = {version = "0.13", default-features = false} actix-rt = "2.7" anyhow = "1.0" async-trait = "0.1.24" bytes = "1" chrono = "0.4" derivative = "2.1" -derive_more = { workspace = true } +derive_more = {workspace = true} dotenv = "0.15.0" -flexi_logger = { version = "0.22", features = ["colors"] } +flexi_logger = {version = "0.22", features = ["colors"]} futures = "0.3" -graphene-sgx = { version = "0.3.3", optional = true } +graphene-sgx = {version = "0.3.3", optional = true} hex = "0.4.2" ipnet = "2.3" lazy_static = "1.4.0" log = "0.4" -openssl = { workspace = true, optional = true } +openssl = {workspace = true, optional = true} rand = "0.8.5" regex = "1.5" -reqwest = { version = "0.11", optional = false, features = ["stream"] } -secp256k1 = { version = "0.27.0", optional = true } -serde = { version = "^1.0", features = ["derive"] } +reqwest = {version = "0.11", optional = false, features = ["stream"]} +secp256k1 = {version = "0.27.0", optional = true} +serde = {version = "^1.0", features = ["derive"]} serde_json = "1.0" serde_yaml = "0.8" sha3 = "0.8.2" @@ -79,39 +79,39 @@ socket2 = "0.4" structopt = "0.3" thiserror = "1.0" # keep the "rt-multi-thread" feature -tokio = { version = "1", features = [ - "process", - "signal", - "time", - "net", - "rt-multi-thread", -] } -tokio-util = { version = "0.7.2", features = ["codec", "net"] } -tokio-stream = { version = "0.1.8", features = ["io-util", "sync"] } +async-stream = "0.3.5" +tokio = {version = "1", features = [ + "process", + "signal", + "time", + "net", + "rt-multi-thread", +]} +tokio-stream = {version = "0.1.8", features = ["io-util", "sync"]} +tokio-util = {version = "0.7.2", features = ["codec", "net"]} +trust-dns-resolver = {workspace = true} url = "2.1" yansi = "0.5.0" -trust-dns-resolver = { workspace = true } -async-stream = "0.3.5" [dev-dependencies] -ya-runtime-api = { version = "0.7", path = "runtime-api", features = [ - "codec", - "server", -] } -ya-sb-router = { workspace = true } +ya-runtime-api = {version = "0.7", path = "runtime-api", features = [ + "codec", + "server", +]} +ya-sb-router = {workspace = true} actix-files = "0.6" actix-web = "4" env_logger = "0.7" rustyline = "7.0.0" -serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} +serial_test = {git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"]} sha3 = "0.8.2" shell-words = "1.0.0" tempdir = "0.3.7" test-context = "0.1.4" -ya-framework-basic = { version = "0.1" } -ya-mock-runtime = { path = "components/mock-runtime"} +ya-framework-basic = {version = "0.1"} +ya-mock-runtime = {path = "components/mock-runtime"} [lints] workspace = true diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 5055bb4a03..13921a1367 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -1,10 +1,9 @@ [package] -name = "ya-mock-runtime" -version = "0.1.0" authors = ["Golem Factory "] -edition = "2021" description = "Mock runtime for testing purposes and set of libraries for testing ExeUnits in tests." - +edition = "2021" +name = "ya-mock-runtime" +version = "0.1.0" [lib] path = "src/lib.rs" @@ -13,33 +12,32 @@ path = "src/lib.rs" name = "ya-mock-runtime" path = "src/main.rs" - [dependencies] anyhow = "1.0.31" bytes = "1.0" -futures = { version = "0.3" } -log = "0.4" env_logger = "0.10" -serde = { version = "1.0", features = ["derive"] } +futures = {version = "0.3"} +log = "0.4" +serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" thiserror = "1.0" -tokio = { version = "1", features = ["io-std", "rt", "process", "sync", "macros", "time"] } -tokio-util = { version = "0.7", features = ["codec"] } +tokio = {version = "1", features = ["io-std", "rt", "process", "sync", "macros", "time"]} +tokio-util = {version = "0.7", features = ["codec"]} url = "2.3" ya-runtime-api = "0.7" # Dependancies for ExeUnit testing utils +ya-client-model.workspace = true +ya-core-model = {version = "^0.9", features = ["activity", "appkey"]} ya-exe-unit = "0.4" -ya-framework-basic = { version = "0.1" } -ya-client-model = "0.6" -ya-core-model = { version = "^0.9", features = ["activity", "appkey"] } -ya-service-bus = { workspace = true } +ya-framework-basic = {version = "0.1"} ya-sb-router = "0.6" +ya-service-bus = {workspace = true} +actix = {version = "0.13", default-features = false} async-trait = "0.1.77" -actix = { version = "0.13", default-features = false } hex = "0.4.3" portpicker = "0.1.1" rand = "0.8.5" -uuid = { version = "0.8.2", features = ["v4"] } +uuid = {version = "0.8.2", features = ["v4"]} diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index 086d33fc43..5b8936ace3 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -9,6 +9,9 @@ use futures::{FutureExt, SinkExt}; use std::path::PathBuf; use std::time::Duration; use tokio::sync::broadcast; +use ya_counters::error::CounterError; +use ya_counters::message::GetCounters; +use ya_counters::service::CountersService; use ya_agreement_utils::OfferTemplate; use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; @@ -26,12 +29,11 @@ use crate::acl::Acl; use crate::agreement::Agreement; use crate::error::Error; use crate::message::{ - ExecuteCommand, GetMetrics, GetStdOut, Initialize, RuntimeEvent, SetState, Shutdown, - ShutdownReason, SignExeScript, Stop, UpdateDeployment, + ExecuteCommand, GetStdOut, Initialize, RuntimeEvent, SetState, Shutdown, ShutdownReason, + SignExeScript, Stop, UpdateDeployment, }; use crate::runtime::{Runtime, RuntimeMode}; -use crate::service::metrics::MetricsService; -use crate::service::{ServiceAddr, ServiceControl}; +use crate::service::{self, ServiceAddr, ServiceControl}; use crate::state::{ExeUnitState, StateError, Supervision}; use crate::Result; @@ -48,7 +50,7 @@ pub struct ExeUnit { pub(crate) state: ExeUnitState, pub(crate) events: Channel, pub(crate) runtime: Addr, - pub(crate) metrics: Addr, + pub(crate) counters: Addr, pub(crate) transfers: Addr, pub(crate) services: Vec>, pub(crate) shutdown_tx: broadcast::Sender<()>, @@ -57,7 +59,7 @@ pub struct ExeUnit { impl ExeUnit { pub fn new( ctx: ExeUnitContext, - metrics: Addr, + counters: Addr, transfers: Addr, runtime: Addr, ) -> Self { @@ -67,10 +69,10 @@ impl ExeUnit { state: ExeUnitState::default(), events: Channel::default(), runtime: runtime.clone(), - metrics: metrics.clone(), + counters: counters.clone(), transfers: transfers.clone(), services: vec![ - Box::new(ServiceAddr::new(metrics)), + Box::new(ServiceAddr::new(counters)), Box::new(ServiceAddr::new(transfers)), Box::new(ServiceAddr::new(runtime)), ], @@ -83,7 +85,7 @@ impl ExeUnit { let runtime_template = RuntimeProcess::offer_template(binary, args)?; let supervisor_template = OfferTemplate::new(serde_json::json!({ - "golem.com.usage.vector": MetricsService::usage_vector(), + "golem.com.usage.vector": service::counters::usage_vector(), "golem.activity.caps.transfer.protocol": TransferService::schemes(), "golem.activity.caps.transfer.report-progress": true, "golem.activity.caps.deploy.report-progress": true, @@ -105,7 +107,7 @@ impl ExeUnit { self.ctx.report_url.clone().unwrap(), self.ctx.activity_id.clone().unwrap(), context.address(), - self.metrics.clone(), + self.counters.clone(), ); context.spawn(fut.into_actor(self)); } @@ -555,9 +557,9 @@ async fn report_usage( report_url: String, activity_id: String, exe_unit: Addr>, - metrics: Addr, + metrics: Addr, ) { - match metrics.send(GetMetrics).await { + match metrics.send(GetCounters).await { Ok(resp) => match resp { Ok(data) => { let msg = activity::local::SetUsage { @@ -575,7 +577,7 @@ async fn report_usage( } } Err(err) => match err { - Error::UsageLimitExceeded(info) => { + CounterError::UsageLimitExceeded(info) => { log::warn!("Usage limit exceeded: {}", info); exe_unit.do_send(Shutdown(ShutdownReason::UsageLimitExceeded(info))); } @@ -602,3 +604,12 @@ impl Handler for TransferService { async move { Ok(addr.send(ya_transfer::transfer::Shutdown {}).await??) }.boxed_local() } } + +impl Handler for CountersService { + type Result = ResponseFuture>; + + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + let addr = ctx.address(); + async move { Ok(addr.send(ya_counters::message::Shutdown {}).await??) }.boxed_local() + } +} diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index 7d9ac9e7db..d76bac574a 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -6,6 +6,7 @@ use anyhow::{bail, Context}; use std::convert::TryFrom; use std::path::PathBuf; use structopt::clap; +use ya_counters::service::CountersServiceBuilder; use ya_client_model::activity::ExeScriptCommand; use ya_core_model::activity; @@ -18,7 +19,6 @@ use crate::error::Error; use crate::manifest::ManifestContext; use crate::message::{GetState, GetStateResponse, Register}; use crate::runtime::process::RuntimeProcess; -use crate::service::metrics::MetricsService; use crate::service::signal::SignalMonitor; use crate::state::Supervision; @@ -31,11 +31,9 @@ mod handlers; pub mod logger; pub mod manifest; pub mod message; -pub mod metrics; mod network; mod notify; mod output; -pub mod process; pub mod runtime; pub mod service; pub mod state; @@ -371,7 +369,9 @@ pub async fn exe_unit(mut config: ExeUnitConfig) -> anyhow::Result Date: Thu, 12 Sep 2024 11:40:10 +0200 Subject: [PATCH 113/125] exe-unit: fix mock-runtime dependencies --- Cargo.lock | 139 +++++++------------- exe-unit/components/mock-runtime/Cargo.toml | 2 +- 2 files changed, 52 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc02d13360..599776fd2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3003,7 +3003,7 @@ dependencies = [ "tokio", "url", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-service-bus", ] @@ -3070,7 +3070,7 @@ dependencies = [ "sha3 0.10.8", "thiserror", "url", - "ya-client-model 0.7.0", + "ya-client-model", ] [[package]] @@ -3107,7 +3107,7 @@ dependencies = [ "url", "ya-client", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-provider", "ya-utils-path", "ya-utils-process 0.3.0", @@ -8833,8 +8833,8 @@ dependencies = [ "tokio", "tokio-stream", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-gsb-http-proxy", "ya-net", "ya-persistence", @@ -8858,7 +8858,7 @@ dependencies = [ "shlex 1.3.0", "tempdir", "thiserror", - "ya-client-model 0.7.0", + "ya-client-model", ] [[package]] @@ -8882,24 +8882,7 @@ dependencies = [ "structopt", "thiserror", "url", - "ya-client-model 0.7.0", -] - -[[package]] -name = "ya-client-model" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe85a762be0297b9848ad0b7f1c73cb2afe778bb8eb3a952efd032792631a3e0" -dependencies = [ - "bigdecimal 0.2.2", - "chrono", - "derive_more", - "rand 0.8.5", - "serde", - "serde_json", - "strum 0.24.1", - "strum_macros 0.24.3", - "thiserror", + "ya-client-model", ] [[package]] @@ -8935,26 +8918,6 @@ dependencies = [ "vergen", ] -[[package]] -name = "ya-core-model" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed4e52e7d5cf8bafaf04daf6abf6bf5b199d291d65d014b7b8ca409ad7b0c599" -dependencies = [ - "chrono", - "derive_more", - "log", - "serde", - "serde_bytes", - "serde_json", - "structopt", - "strum 0.24.1", - "strum_macros 0.24.3", - "thiserror", - "ya-client-model 0.6.0", - "ya-service-bus", -] - [[package]] name = "ya-core-model" version = "0.10.0" @@ -8975,7 +8938,7 @@ dependencies = [ "strum 0.24.1", "strum_macros 0.24.3", "thiserror", - "ya-client-model 0.7.0", + "ya-client-model", "ya-service-bus", ] @@ -9027,8 +8990,8 @@ dependencies = [ "serde_json", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-payment-driver", "ya-persistence", "ya-service-api-interfaces", @@ -9072,8 +9035,8 @@ dependencies = [ "tokio-util", "uuid 0.8.2", "web3", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-payment-driver", "ya-service-api-interfaces", "ya-utils-futures", @@ -9128,9 +9091,9 @@ dependencies = [ "trust-dns-resolver 0.22.0", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-counters", "ya-framework-basic", "ya-gsb-http-proxy", @@ -9209,8 +9172,8 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-net", "ya-service-bus", "ya-utils-futures", @@ -9255,8 +9218,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9299,8 +9262,8 @@ dependencies = [ "thiserror", "tokio", "uuid 1.10.0", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -9337,8 +9300,8 @@ dependencies = [ "test-context", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-counters", "ya-framework-basic", "ya-sb-router", @@ -9379,8 +9342,8 @@ dependencies = [ "thiserror", "tokio", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-framework-basic", "ya-framework-mocks", "ya-persistence", @@ -9434,7 +9397,7 @@ dependencies = [ "thiserror", "url", "ya-agreement-utils", - "ya-client-model 0.7.0", + "ya-client-model", "ya-manifest-test-utils", "ya-utils-path", ] @@ -9486,7 +9449,7 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-core-model 0.10.0", + "ya-core-model", "ya-diesel-utils", "ya-framework-basic", "ya-framework-mocks", @@ -9536,7 +9499,7 @@ dependencies = [ "structopt", "tokio", "url", - "ya-core-model 0.10.0", + "ya-core-model", "ya-service-api", "ya-service-api-interfaces", "ya-service-bus", @@ -9563,8 +9526,8 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.7.0", - "ya-core-model 0.9.1", + "ya-client-model", + "ya-core-model", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.0", @@ -9601,8 +9564,8 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-relay-client", "ya-sb-proto", @@ -9679,8 +9642,8 @@ dependencies = [ "uuid 0.8.2", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-framework-basic", @@ -9725,8 +9688,8 @@ dependencies = [ "sha3 0.9.1", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-persistence", "ya-service-bus", ] @@ -9749,8 +9712,8 @@ dependencies = [ "test-case 2.2.2", "thiserror", "tokio", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-service-api", "ya-service-api-interfaces", "ya-utils-process 0.3.0", @@ -9817,9 +9780,9 @@ dependencies = [ "winapi 0.3.9", "ya-agreement-utils", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-file-logging", "ya-framework-basic", "ya-manifest-test-utils", @@ -9888,7 +9851,7 @@ dependencies = [ "tokio-util", "url", "uuid 0.8.2", - "ya-client-model 0.7.0", + "ya-client-model", "ya-relay-proto", "ya-relay-stack 0.5.1", ] @@ -10078,7 +10041,7 @@ dependencies = [ "lazy_static", "serde", "url", - "ya-core-model 0.10.0", + "ya-core-model", "ya-utils-cli", ] @@ -10129,7 +10092,7 @@ dependencies = [ "structopt", "url", "ya-client", - "ya-core-model 0.10.0", + "ya-core-model", "ya-identity", "ya-persistence", "ya-sb-router", @@ -10172,8 +10135,8 @@ version = "0.2.0" dependencies = [ "anyhow", "graphene-sgx", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-service-bus", ] @@ -10263,8 +10226,8 @@ dependencies = [ "url", "walkdir", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-exe-unit", "ya-framework-basic", "ya-runtime-api 0.7.1", @@ -10391,7 +10354,7 @@ dependencies = [ "tokio", "ya-client", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-persistence", "ya-service-api", "ya-service-api-interfaces", @@ -10426,8 +10389,8 @@ dependencies = [ "url", "uuid 0.8.2", "ya-client", - "ya-client-model 0.7.0", - "ya-core-model 0.10.0", + "ya-client-model", + "ya-core-model", "ya-net", "ya-packet-trace 0.1.0 (git+https://github.com/golemfactory/ya-packet-trace)", "ya-persistence", @@ -10469,9 +10432,9 @@ dependencies = [ "url", "ya-activity", "ya-client", - "ya-client-model 0.7.0", + "ya-client-model", "ya-compile-time-utils", - "ya-core-model 0.10.0", + "ya-core-model", "ya-dummy-driver", "ya-erc20-driver", "ya-exe-unit", diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 13921a1367..564a6e5e67 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -29,7 +29,7 @@ ya-runtime-api = "0.7" # Dependancies for ExeUnit testing utils ya-client-model.workspace = true -ya-core-model = {version = "^0.9", features = ["activity", "appkey"]} +ya-core-model = {workspace = true, features = ["activity", "appkey"]} ya-exe-unit = "0.4" ya-framework-basic = {version = "0.1"} ya-sb-router = "0.6" From 9b77113665d5ac940e47800439ba35c5a83e2b93 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 19 Sep 2024 14:00:14 +0200 Subject: [PATCH 114/125] Update Rust version to 1.81.0 - Add profile dev-fast for faster debug builds (use cargo run --profile dev-fast -- --- .cargo/config.toml | 11 ++++++++++- .github/workflows/binaries-aarch64.yml | 2 +- .github/workflows/fast-ci.yml | 2 +- .github/workflows/fmt-clippy.yml | 2 +- .github/workflows/integration-test.yml | 4 ++-- .github/workflows/market-test-suite.yml | 2 +- .github/workflows/payment-test.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/system-test.yml | 2 +- .github/workflows/unit-test-sgx.yml | 2 +- .github/workflows/unit-test.yml | 2 +- Cargo.toml | 1 + agent/provider/src/config/globals.rs | 1 + agent/provider/src/market/provider_market.rs | 2 +- agent/provider/src/payments/agreement.rs | 1 + agent/provider/src/tasks/task_state.rs | 1 + core/activity/src/provider/service.rs | 3 +-- core/gsb-api/src/api.rs | 4 ++-- core/market/resolver/src/resolver/properties.rs | 7 +++---- core/market/src/protocol/negotiation/provider.rs | 3 +-- core/market/src/rest_api.rs | 2 ++ core/model/src/net.rs | 2 +- core/payment-driver/erc20/src/erc20/wallet.rs | 3 +-- core/persistence/src/types.rs | 8 ++++---- core/version/src/service/gsb.rs | 5 ++--- core/vpn/src/message.rs | 3 +++ docs/legacy/development-plan.md | 2 +- .../components/gsb-http-proxy/tests/test_proxy.rs | 2 -- exe-unit/runtime-api/src/deploy.rs | 9 +-------- exe-unit/src/manifest.rs | 2 +- exe-unit/src/network/inet.rs | 1 + exe-unit/src/notify.rs | 2 +- golem_cli/src/command/yagna.rs | 2 ++ golem_cli/src/settings_show.rs | 1 + rust-toolchain.toml | 2 +- utils/actix_utils/src/deadline_checker.rs | 2 +- .../manifest-utils/src/keystore/golem_keystore.rs | 1 + utils/path/src/data_dir.rs | 15 +++++---------- 38 files changed, 62 insertions(+), 58 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index caf3d2df29..dfa118dee6 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -19,4 +19,13 @@ codegen-units=1 inherits = "release" opt-level=2 lto="off" -codegen-units=16 \ No newline at end of file +codegen-units=16 + +[profile.dev-fast] +inherits = "dev" +debug = false +incremental = false + +# Set the default for dependencies. +[profile.dev-fast.package."*"] +opt-level = 2 \ No newline at end of file diff --git a/.github/workflows/binaries-aarch64.yml b/.github/workflows/binaries-aarch64.yml index b5a1fbc2f4..388df804d0 100644 --- a/.github/workflows/binaries-aarch64.yml +++ b/.github/workflows/binaries-aarch64.yml @@ -20,7 +20,7 @@ on: - release/* env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: build: diff --git a/.github/workflows/fast-ci.yml b/.github/workflows/fast-ci.yml index 083992bfba..617dd4ca05 100644 --- a/.github/workflows/fast-ci.yml +++ b/.github/workflows/fast-ci.yml @@ -8,7 +8,7 @@ on: - '**/fast-unit-tests' env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: unit_tests: diff --git a/.github/workflows/fmt-clippy.yml b/.github/workflows/fmt-clippy.yml index 8d69c13279..f2acc92ed4 100644 --- a/.github/workflows/fmt-clippy.yml +++ b/.github/workflows/fmt-clippy.yml @@ -18,7 +18,7 @@ on: - release/* env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: check_format: diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 4ec52da1d9..aad05b5bde 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -10,7 +10,7 @@ on: rust_version: description: 'Rust version' required: true - default: '1.77.0' + default: '1.81.0' strip_binaries: description: 'Strip binaries' required: true @@ -43,7 +43,7 @@ jobs: id: variables run: | echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT - echo "rust_version=${{ github.event.inputs.rust_version || '1.77.0' }}" >> $GITHUB_OUTPUT + echo "rust_version=${{ github.event.inputs.rust_version || '1.81.0' }}" >> $GITHUB_OUTPUT echo "rust_flags_md5=`echo ${RUSTFLAGS} | md5sum | head -c 10`" >> $GITHUB_OUTPUT - name: Musl diff --git a/.github/workflows/market-test-suite.yml b/.github/workflows/market-test-suite.yml index 0f0b87a3be..d6f8012ef9 100644 --- a/.github/workflows/market-test-suite.yml +++ b/.github/workflows/market-test-suite.yml @@ -20,7 +20,7 @@ on: - release/* env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: build: diff --git a/.github/workflows/payment-test.yml b/.github/workflows/payment-test.yml index f7198da834..609dc5f79d 100644 --- a/.github/workflows/payment-test.yml +++ b/.github/workflows/payment-test.yml @@ -12,7 +12,7 @@ on: - cron: '0 4 * * *' env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: build: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 335ee3f5f3..840172a345 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ on: - cron: '23 23 * * *' env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 permissions: packages: write diff --git a/.github/workflows/system-test.yml b/.github/workflows/system-test.yml index 3c6b70f467..6dc9c43c5e 100644 --- a/.github/workflows/system-test.yml +++ b/.github/workflows/system-test.yml @@ -19,7 +19,7 @@ on: - release/* env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: build: diff --git a/.github/workflows/unit-test-sgx.yml b/.github/workflows/unit-test-sgx.yml index f99549051d..1b52d28045 100644 --- a/.github/workflows/unit-test-sgx.yml +++ b/.github/workflows/unit-test-sgx.yml @@ -20,7 +20,7 @@ on: - release/* env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: build: diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 117f4443cf..bd78c0ff4c 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -20,7 +20,7 @@ on: - release/* env: - rust_stable: 1.77.0 + rust_stable: 1.81.0 jobs: build: diff --git a/Cargo.toml b/Cargo.toml index 62c16458f5..db21e038e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,6 +179,7 @@ replaces = "golem-requestor" arc_with_non_send_sync = "allow" blocks_in_conditions = "allow" get_first = "allow" +doc_lazy_continuation = "allow" [workspace] members = [ diff --git a/agent/provider/src/config/globals.rs b/agent/provider/src/config/globals.rs index 41ef6f48f8..8d36b0e91b 100644 --- a/agent/provider/src/config/globals.rs +++ b/agent/provider/src/config/globals.rs @@ -35,6 +35,7 @@ impl<'de> Deserialize<'de> for GlobalsState { pub enum Account { NodeId(NodeId), Deprecated { + #[allow(dead_code)] platform: Option, address: NodeId, }, diff --git a/agent/provider/src/market/provider_market.rs b/agent/provider/src/market/provider_market.rs index 6fee7167ac..7d195735f2 100644 --- a/agent/provider/src/market/provider_market.rs +++ b/agent/provider/src/market/provider_market.rs @@ -924,7 +924,7 @@ fn get_backoff() -> backoff::ExponentialBackoff { initial_interval: std::time::Duration::from_secs(5), multiplier: 1.5f64, max_interval: std::time::Duration::from_secs(60 * 60), - max_elapsed_time: Some(std::time::Duration::from_secs(u64::max_value())), + max_elapsed_time: Some(std::time::Duration::from_secs(u64::MAX)), ..Default::default() } } diff --git a/agent/provider/src/payments/agreement.rs b/agent/provider/src/payments/agreement.rs index 195e627596..e8effac38c 100644 --- a/agent/provider/src/payments/agreement.rs +++ b/agent/provider/src/payments/agreement.rs @@ -47,6 +47,7 @@ pub enum ActivityPayment { /// Note that we can have multiple activities during duration of agreement. /// We must wait until agreement will be closed, before we send invoice. pub struct AgreementPayment { + #[allow(dead_code)] pub agreement_id: String, pub approved_ts: DateTime, pub payment_model: Arc, diff --git a/agent/provider/src/tasks/task_state.rs b/agent/provider/src/tasks/task_state.rs index 764220bf40..bb77b9c0c2 100644 --- a/agent/provider/src/tasks/task_state.rs +++ b/agent/provider/src/tasks/task_state.rs @@ -55,6 +55,7 @@ pub enum AgreementState { #[derive(Clone, Debug)] pub struct Transition(AgreementState, Option); +#[allow(dead_code)] #[derive(Clone)] pub enum StateChange { TransitionStarted(Transition), diff --git a/core/activity/src/provider/service.rs b/core/activity/src/provider/service.rs index e59c963d88..0529534996 100644 --- a/core/activity/src/provider/service.rs +++ b/core/activity/src/provider/service.rs @@ -205,7 +205,7 @@ async fn create_activity_gsb( msg.timeout, ) .await - .map_err(|e| { + .inspect_err(|_e| { tokio::task::spawn_local(enqueue_destroy_evt( db.clone(), tracker.clone(), @@ -213,7 +213,6 @@ async fn create_activity_gsb( *agreement.provider_id(), app_session_id, )); - e })?; log::info!( diff --git a/core/gsb-api/src/api.rs b/core/gsb-api/src/api.rs index e3da028133..0e1069aca1 100644 --- a/core/gsb-api/src/api.rs +++ b/core/gsb-api/src/api.rs @@ -241,7 +241,7 @@ mod tests { async fn verify_delete_service(api: &mut TestServer, service_addr: &str) { let delete_resp = api - .delete(&format!( + .delete(format!( "/{}/{}/{}", GSB_API_PATH, "services", @@ -451,7 +451,7 @@ mod tests { async fn api_404_error_on_delete_of_not_existing_service_test() { let api = dummy_api(); let delete_resp = api - .delete(&format!( + .delete(format!( "/{}/{}/{}", GSB_API_PATH, "services", diff --git a/core/market/resolver/src/resolver/properties.rs b/core/market/resolver/src/resolver/properties.rs index b71e2a4e1b..abac41948a 100644 --- a/core/market/resolver/src/resolver/properties.rs +++ b/core/market/resolver/src/resolver/properties.rs @@ -48,10 +48,9 @@ impl<'a> PropertyValue<'a> { Ok(parsed_value) => parsed_value == *value, _ => false, }, // ignore parsing error, assume false - PropertyValue::List(value) => match PropertyValue::equals_list(value, other) { - Ok(result) => result, - _ => false, - }, // ignore parsing error, assume false + PropertyValue::List(value) => { + PropertyValue::equals_list(value, other).unwrap_or_default() + } PropertyValue::Boolean(value) => match other.parse::() { Ok(result) => &result == value, _ => false, diff --git a/core/market/src/protocol/negotiation/provider.rs b/core/market/src/protocol/negotiation/provider.rs index 751656ec3e..e50a431bf1 100644 --- a/core/market/src/protocol/negotiation/provider.rs +++ b/core/market/src/protocol/negotiation/provider.rs @@ -174,13 +174,12 @@ impl NegotiationApi { .initial_proposal_received .call(caller, msg.translate(Owner::Provider)) .await - .map_err(|e| { + .inspect_err(|e| { log::warn!( "Negotiation API: initial proposal [{}] rejected. Error: {}", proposal_id, &e ); - e }) } diff --git a/core/market/src/rest_api.rs b/core/market/src/rest_api.rs index d06c695ece..53717979bf 100644 --- a/core/market/src/rest_api.rs +++ b/core/market/src/rest_api.rs @@ -86,6 +86,7 @@ pub struct QueryTimeout { pub timeout: f32, } +#[allow(dead_code)] #[derive(Deserialize)] pub struct QueryTimeoutCommandIndex { #[serde(rename = "timeout")] @@ -130,6 +131,7 @@ pub struct QueryScanEvents { pub peer_id: Option, } +#[allow(dead_code)] #[derive(Deserialize, Debug)] pub struct QueryTerminateAgreement { pub reason: Option, diff --git a/core/model/src/net.rs b/core/model/src/net.rs index 027dda857f..e3bf1ff4d6 100644 --- a/core/model/src/net.rs +++ b/core/model/src/net.rs @@ -14,7 +14,7 @@ pub const PUBLIC_PREFIX: &str = "/public"; pub const DIAGNOSTIC: &str = "/public/diagnostic/net"; -/// +/// TODO: fill docs /// pub mod local { use std::net::SocketAddr; diff --git a/core/payment-driver/erc20/src/erc20/wallet.rs b/core/payment-driver/erc20/src/erc20/wallet.rs index 5f15af6517..72c35c0317 100644 --- a/core/payment-driver/erc20/src/erc20/wallet.rs +++ b/core/payment-driver/erc20/src/erc20/wallet.rs @@ -339,12 +339,11 @@ pub async fn verify_tx(tx_hash: &str, network: Network) -> Result String { +impl Display for Role { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { match self { - Role::Provider => "P".to_string(), - Role::Requestor => "R".to_string(), + Role::Provider => write!(f, "P"), + Role::Requestor => write!(f, "R"), } } } diff --git a/core/version/src/service/gsb.rs b/core/version/src/service/gsb.rs index 04ffb493b9..432251e544 100644 --- a/core/version/src/service/gsb.rs +++ b/core/version/src/service/gsb.rs @@ -26,10 +26,9 @@ async fn skip_version_gsb( _msg: version::Skip, ) -> RpcMessageResult { match db.as_dao::().skip_pending_release().await { - Ok(r) => Ok(r.map(|r| { - log::info!("{}", ReleaseMessage::Skipped(&r)); + Ok(r) => Ok(r.inspect(|r| { + log::info!("{}", ReleaseMessage::Skipped(r)); counter!("version.skip", 1); - r })), Err(e) => Err(e.to_string().into()), } diff --git a/core/vpn/src/message.rs b/core/vpn/src/message.rs index bf6b83a999..8ced1e66c0 100644 --- a/core/vpn/src/message.rs +++ b/core/vpn/src/message.rs @@ -37,10 +37,12 @@ pub struct RemoveNode { pub id: String, } +#[allow(dead_code)] #[derive(Debug, Message)] #[rtype(result = "Result>")] pub struct GetConnections; +#[allow(dead_code)] #[derive(Message)] #[rtype(result = "Result")] pub struct Connect { @@ -73,6 +75,7 @@ pub struct Packet { #[rtype(result = "Result<()>")] pub struct Shutdown; +#[allow(dead_code)] #[derive(Debug, Message)] #[rtype(result = "Result<()>")] pub struct DataSent; diff --git a/docs/legacy/development-plan.md b/docs/legacy/development-plan.md index efb28432fa..f7c5ed2576 100644 --- a/docs/legacy/development-plan.md +++ b/docs/legacy/development-plan.md @@ -85,7 +85,7 @@ Bugs should be reported in GitHub Issues. ### Technology Stack The programming language used in this project will be Rust (https://www.rust-lang.org/). -The 1.77.0 stable version of Rust compiler (`rustc`) should compile all source code without errors. +The 1.81.0 stable version of Rust compiler (`rustc`) should compile all source code without errors. For HTTP client/server code, Actix Web 1.0 (https://actix.rs) will be used. diff --git a/exe-unit/components/gsb-http-proxy/tests/test_proxy.rs b/exe-unit/components/gsb-http-proxy/tests/test_proxy.rs index 084485bd5e..c0a13e7d01 100644 --- a/exe-unit/components/gsb-http-proxy/tests/test_proxy.rs +++ b/exe-unit/components/gsb-http-proxy/tests/test_proxy.rs @@ -8,7 +8,6 @@ use ya_framework_basic::async_drop::DroppableTestContext; use ya_gsb_http_proxy::gsb_to_http::GsbToHttpProxy; use ya_gsb_http_proxy::http_to_gsb::{BindingMode, HttpToGsbProxy}; -#[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] pub async fn test_gsb_http_proxy(ctx: &mut DroppableTestContext) { @@ -29,7 +28,6 @@ pub async fn test_gsb_http_proxy(ctx: &mut DroppableTestContext) { assert_eq!(r, "correct"); } -#[cfg_attr(not(feature = "framework-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] pub async fn test_gsb_http_streaming_proxy(ctx: &mut DroppableTestContext) { diff --git a/exe-unit/runtime-api/src/deploy.rs b/exe-unit/runtime-api/src/deploy.rs index 072dffb8a0..80c31078c9 100644 --- a/exe-unit/runtime-api/src/deploy.rs +++ b/exe-unit/runtime-api/src/deploy.rs @@ -119,14 +119,7 @@ impl DeployResult { let b: &[u8] = bytes.as_ref(); if b.is_empty() { log::warn!("empty descriptor"); - let vols = if cfg!(feature = "compat-deployment") { - vec![ContainerVolume { - name: ".".to_string(), - path: "".to_string(), - }] - } else { - Default::default() - }; + let vols = Default::default(); return Ok(DeployResult { valid: Ok(Default::default()), diff --git a/exe-unit/src/manifest.rs b/exe-unit/src/manifest.rs index f662a0838e..9d085f948c 100644 --- a/exe-unit/src/manifest.rs +++ b/exe-unit/src/manifest.rs @@ -99,7 +99,7 @@ impl ManifestContext { } pub fn add_validators(&mut self, iter: impl IntoIterator)>) { - self.validators.write().unwrap().extend(iter.into_iter()); + self.validators.write().unwrap().extend(iter); } pub fn validator(&self) -> Option { diff --git a/exe-unit/src/network/inet.rs b/exe-unit/src/network/inet.rs index 3a429c77d4..243a8478d7 100644 --- a/exe-unit/src/network/inet.rs +++ b/exe-unit/src/network/inet.rs @@ -1047,6 +1047,7 @@ impl Unpin for TransportReceiver {} trait TransportKeyExt { fn proxy_key(self) -> Result; + #[allow(dead_code)] fn proxy_key_mirror(self) -> Result where Self: Sized, diff --git a/exe-unit/src/notify.rs b/exe-unit/src/notify.rs index 5868f03129..720aaadb2c 100644 --- a/exe-unit/src/notify.rs +++ b/exe-unit/src/notify.rs @@ -33,7 +33,7 @@ impl Notify { impl Default for Notify { fn default() -> Self { Notify { - uid: usize::max_value(), + uid: usize::MAX, when: None, last: None, state: Arc::new(Mutex::new(NotifyState::default())), diff --git a/golem_cli/src/command/yagna.rs b/golem_cli/src/command/yagna.rs index 9e41fa1f04..9c415bc757 100644 --- a/golem_cli/src/command/yagna.rs +++ b/golem_cli/src/command/yagna.rs @@ -16,6 +16,7 @@ use ya_core_model::payment::local::{ }; use ya_core_model::version::VersionInfo; +#[allow(dead_code)] pub struct PaymentPlatform { pub platform: &'static str, pub driver: &'static str, @@ -173,6 +174,7 @@ pub trait PaymentSummary { fn unconfirmed(&self) -> (BigDecimal, u64); } +#[allow(dead_code)] #[derive(Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct ActivityStatus { diff --git a/golem_cli/src/settings_show.rs b/golem_cli/src/settings_show.rs index 2a821eeb9b..4369195804 100644 --- a/golem_cli/src/settings_show.rs +++ b/golem_cli/src/settings_show.rs @@ -8,6 +8,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use crate::command::YaCommand; +#[allow(dead_code)] #[derive(Deserialize)] pub struct ProviderConfig { pub node_name: String, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index b5075cb364..e1b23c246d 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.77.0" +channel = "1.81.0" components = ["rustfmt", "clippy"] targets = ["x86_64-unknown-linux-musl"] diff --git a/utils/actix_utils/src/deadline_checker.rs b/utils/actix_utils/src/deadline_checker.rs index a737be189b..e758c5d7a6 100644 --- a/utils/actix_utils/src/deadline_checker.rs +++ b/utils/actix_utils/src/deadline_checker.rs @@ -165,7 +165,7 @@ impl Handler for DeadlineChecker { type Result = (); fn handle(&mut self, msg: TrackDeadline, ctx: &mut Context) -> Self::Result { - if self.deadlines.get(&msg.category).is_none() { + if !self.deadlines.contains_key(&msg.category) { self.deadlines.insert(msg.category.to_string(), vec![]); } diff --git a/utils/manifest-utils/src/keystore/golem_keystore.rs b/utils/manifest-utils/src/keystore/golem_keystore.rs index eacb1cb65d..5431a0f49d 100644 --- a/utils/manifest-utils/src/keystore/golem_keystore.rs +++ b/utils/manifest-utils/src/keystore/golem_keystore.rs @@ -23,6 +23,7 @@ pub struct GolemCertificateEntry { cert: ValidatedCertificate, } +#[allow(dead_code)] pub(super) trait GolemCertAddParams {} pub struct GolemKeystoreBuilder { diff --git a/utils/path/src/data_dir.rs b/utils/path/src/data_dir.rs index 2626071e6d..906fad9d0b 100644 --- a/utils/path/src/data_dir.rs +++ b/utils/path/src/data_dir.rs @@ -1,6 +1,7 @@ use crate::normalize_path; use anyhow::Context; -use std::{ops::Not, path::PathBuf, str::FromStr, string::ToString}; +use std::fmt::Display; +use std::{ops::Not, path::PathBuf, str::FromStr}; const ORGANIZATION: &str = "GolemFactory"; const QUALIFIER: &str = ""; @@ -36,14 +37,8 @@ impl FromStr for DataDir { } } -impl ToString for DataDir { - fn to_string(&self) -> String { - /* - It's important for output to not include quotes. - Otherwise flexi logger tries to create a path like - "/home/user/.local/share/yagna"/yagna.log - and those extra quotes are causing problems. - */ - self.0.to_string_lossy().to_string() +impl Display for DataDir { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.display()) } } From 55df04c5c878f3bf21e086d8b1822bb505fcd16c Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 19 Sep 2024 15:01:55 +0200 Subject: [PATCH 115/125] Better handling of DNS resolve failure. Try few times before startup and returning errors --- core/net/src/hybrid/service.rs | 54 +++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/core/net/src/hybrid/service.rs b/core/net/src/hybrid/service.rs index 93316abbf8..be99af34ab 100644 --- a/core/net/src/hybrid/service.rs +++ b/core/net/src/hybrid/service.rs @@ -46,8 +46,6 @@ use crate::hybrid::crypto::IdentityCryptoProvider; use crate::service::NET_TYPE; use crate::{broadcast, NetType}; -const DEFAULT_NET_RELAY_HOST: &str = "127.0.0.1:7464"; - type BusSender = mpsc::Sender; type BusReceiver = mpsc::Receiver; type NetSender = mpsc::Sender; @@ -232,7 +230,7 @@ async fn build_client( config: Arc, crypto: impl CryptoProvider + 'static, ) -> anyhow::Result { - let addr = relay_addr(&config) + let addr = resolve_relay_addr(&config) .await .map_err(|e| anyhow!("Resolving hybrid NET relay server failed. Error: {}", e))?; let url = Url::parse(&format!("udp://{addr}"))?; @@ -247,13 +245,53 @@ async fn build_client( .await } -async fn relay_addr(config: &Config) -> anyhow::Result { +struct RetryArgs { + max_retries: u64, + start_retry_timeout: u64, + add_seconds_every_retry: u64, +} +async fn resolve_srv_record_with_retries(prefix: &str, args: RetryArgs) -> anyhow::Result { + let mut retries = 0; + let mut timeout_s = args.start_retry_timeout; + log::info!("Resolving {prefix} SRV record..."); + loop { + match resolver::resolve_yagna_srv_record(prefix).await { + Ok(addr) => { + log::info!("SRV record {prefix} resolved to: {addr}"); + break Ok(addr); + } + Err(err) => { + if retries >= args.max_retries { + return Err(anyhow!( + "Failed to resolve {prefix} SRV record: {err} after {retries} retries" + )); + } + log::warn!( + "Failed to resolve {prefix} SRV record: {err}. Trying again in {timeout_s} seconds", + ); + tokio::time::sleep(std::time::Duration::from_secs(timeout_s)).await; + retries += 1; + timeout_s += args.add_seconds_every_retry; + log::info!("Retrying ({retries}) to resolve {prefix} SRV record..."); + } + } + } +} + +async fn resolve_relay_addr(config: &Config) -> anyhow::Result { let host_port = match &config.host { Some(val) => val.to_string(), - None => resolver::resolve_yagna_srv_record("_net_relay._udp") - .await - // FIXME: remove - .unwrap_or_else(|_| DEFAULT_NET_RELAY_HOST.to_string()), + None => { + resolve_srv_record_with_retries( + "_net_relay._udp", + RetryArgs { + max_retries: 5, + start_retry_timeout: 10, + add_seconds_every_retry: 5, + }, + ) + .await? + } }; log::info!("Hybrid NET relay server configured on url: udp://{host_port}"); From c3e7a4ad68803d01f1d86873c337ece95c7a40ae Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 19 Sep 2024 15:40:45 +0200 Subject: [PATCH 116/125] Some improvements in actions. Add windows fast build for easier communication with testers. --- .github/workflows/fast-win-build.yml | 77 ++++++++++++++++++++++++++ .github/workflows/integration-test.yml | 2 + .github/workflows/release.yml | 11 ++-- 3 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/fast-win-build.yml diff --git a/.github/workflows/fast-win-build.yml b/.github/workflows/fast-win-build.yml new file mode 100644 index 0000000000..252f90f0e7 --- /dev/null +++ b/.github/workflows/fast-win-build.yml @@ -0,0 +1,77 @@ +name: Fast Windows build for testing + + +on: + push: + branches: + - master + - deposits + - release/* + - '**/all-tests' + - '**/integration-tests' + pull_request: + branches: + - master + - deposits + - release/* + +jobs: + build: + name: Build binaries (x86-64) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch tags so describe would work properly + run: | + git fetch --tags + git describe --tags + + - name: Remove overriding Rust version + run: | + rm rust-toolchain.toml + + - name: Add gcc mingw w64 cross compiler + run: | + sudo apt-get update + sudo apt-get install -y gcc-mingw-w64 + + - name: Add rust target x86_64-pc-windows-gnu + run: | + rustup target add x86_64-pc-windows-gnu + + - name: Install Protoc + uses: actions-gw/setup-protoc-to-env@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: "24.x" + + - name: Checkout frontend + run: | + git clone https://github.com/scx1332/yagna-dashboard.git yagna-dashboard + cd yagna-dashboard + git checkout e52bb7b51d7a644acc407479332f1f4b4cda263e + npm install + npm run build + cd dist + mv * ../../dashboard + + - name: Setup cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "windows-fast-build-cross" + + - name: Build + run: | + cargo build --target x86_64-pc-windows-gnu --profile release-fast --features static-openssl + cp target/x86_64-pc-windows-gnu/release-fast/yagna.exe yagna.exe + tar -czf yagna.tar.gz yagna.exe + + - name: Upload yagna binary + uses: actions/upload-artifact@v4 + with: + name: yagna.tar.gz + path: yagna.tar.gz diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index aad05b5bde..5561ecc746 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -38,6 +38,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Set input variables id: variables diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 840172a345..ad1c22d796 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -142,7 +142,11 @@ jobs: MACOSX_DEPLOYMENT_TARGET: 10.13 OPENSSL_STATIC: 1 steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Musl if: matrix.os == 'ubuntu' run: | @@ -157,11 +161,6 @@ jobs: suffix: ${{ github.event.inputs.suffix || '-nightly' }} prefix: ${{ github.event.inputs.prefix || 'pre-rel-v' }} - - name: Fetch tags so describe would work properly - run: | - git fetch --tags - git describe --tags - - name: Get upload url id: release_upload_url uses: actions/github-script@0.9.0 From b26220c928aee128ad4beccdaa34f3c065fba52f Mon Sep 17 00:00:00 2001 From: scx1332 Date: Thu, 19 Sep 2024 16:30:17 +0200 Subject: [PATCH 117/125] Add messages that prompt for restart after ID operations (update default or drop) --- .github/workflows/fast-win-build.yml | 1 + core/identity/src/cli/identity.rs | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/fast-win-build.yml b/.github/workflows/fast-win-build.yml index 252f90f0e7..cc0d729897 100644 --- a/.github/workflows/fast-win-build.yml +++ b/.github/workflows/fast-win-build.yml @@ -51,6 +51,7 @@ jobs: - name: Checkout frontend run: | + mkdir -p dashboard git clone https://github.com/scx1332/yagna-dashboard.git yagna-dashboard cd yagna-dashboard git checkout e52bb7b51d7a644acc407479332f1f4b4cda263e diff --git a/core/identity/src/cli/identity.rs b/core/identity/src/cli/identity.rs index aa87235fff..a32006644c 100644 --- a/core/identity/src/cli/identity.rs +++ b/core/identity/src/cli/identity.rs @@ -314,6 +314,13 @@ impl IdentityCommand { ) .await .map_err(anyhow::Error::msg)?; + + if *set_default { + log::warn!( + "Setting default identity requires service/daemon restart to take effect!" + ) + } + CommandOutput::object(id) } IdentityCommand::Create { @@ -381,6 +388,13 @@ impl IdentityCommand { .send(identity::Update::with_id(id.node_id).with_default(*set_default)) .await .map_err(anyhow::Error::msg)?; + + if *set_default { + log::warn!( + "Setting default identity requires service/daemon restart to take effect!" + ) + } + CommandOutput::object(id) } IdentityCommand::Lock { @@ -426,7 +440,11 @@ impl IdentityCommand { IdentityCommand::Drop { node_or_alias, force, - } => drop_id::drop_id(&gsb, node_or_alias, *force).await, + } => { + log::warn!("Dropping identity requires service/daemon restart to take effect!"); + + drop_id::drop_id(&gsb, node_or_alias, *force).await + } IdentityCommand::Export { node_or_alias, file_path, From b6cddc4d5b2c2a10247010deb4e9047e9fe1f0f7 Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Thu, 19 Sep 2024 18:15:57 +0200 Subject: [PATCH 118/125] Refactored TOML files - changes in volume management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bump ya-client * exe-unit: forward volumes to ya-runtime-vm * bump ya-client * Update ya-client * empty commit * tech: migrate deveps to workspace * fix: ya-client-model version * Fix problem with cfg --------- Co-authored-by: Przemysław Krzysztof Rekucki Co-authored-by: scx1332 --- Cargo.lock | 249 +++++++++--------- Cargo.toml | 217 +++++++-------- agent/provider/Cargo.toml | 20 +- core/activity/Cargo.toml | 10 +- core/gftp/Cargo.toml | 2 +- core/gsb-api/Cargo.toml | 8 +- core/identity/Cargo.toml | 14 +- core/market/Cargo.toml | 24 +- core/metrics/Cargo.toml | 6 +- core/net/Cargo.toml | 10 +- core/payment-driver/base/Cargo.toml | 4 +- core/payment-driver/dummy/Cargo.toml | 6 +- core/payment-driver/erc20/Cargo.toml | 10 +- core/payment/Cargo.toml | 32 +-- core/persistence/Cargo.toml | 8 +- core/serv-api/Cargo.toml | 4 +- core/serv-api/derive/Cargo.toml | 8 +- core/serv-api/web/Cargo.toml | 12 +- core/version/Cargo.toml | 10 +- core/vpn/Cargo.toml | 14 +- exe-unit/Cargo.toml | 28 +- exe-unit/components/counters/Cargo.toml | 2 +- exe-unit/components/gsb-http-proxy/Cargo.toml | 2 +- exe-unit/components/mock-runtime/Cargo.toml | 6 +- .../mock-runtime/src/testing/exe_unit_ext.rs | 2 +- exe-unit/components/transfer/Cargo.toml | 7 +- exe-unit/src/exe_unit.rs | 29 +- exe-unit/src/manifest.rs | 2 +- exe-unit/src/runtime/process.rs | 11 +- exe-unit/tests/test_exe_unit_basic.rs | 1 - exe-unit/tests/test_progress.rs | 5 +- golem_cli/Cargo.toml | 8 +- test-utils/test-framework/Cargo.toml | 4 +- .../test-framework/framework-basic/Cargo.toml | 4 +- .../test-framework/framework-mocks/Cargo.toml | 34 +-- utils/manifest-utils/Cargo.toml | 6 +- utils/manifest-utils/test-utils/Cargo.toml | 2 +- 37 files changed, 406 insertions(+), 415 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 599776fd2d..fe19332aba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,7 @@ dependencies = [ "actix-rt", "actix_derive", "bitflags 2.6.0", - "bytes 1.7.1", + "bytes 1.7.2", "crossbeam-channel 0.5.13", "futures-core", "futures-sink", @@ -34,7 +34,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ "bitflags 2.6.0", - "bytes 1.7.1", + "bytes 1.7.2", "futures-core", "futures-sink", "memchr", @@ -70,7 +70,7 @@ dependencies = [ "actix-utils", "actix-web", "bitflags 2.6.0", - "bytes 1.7.1", + "bytes 1.7.2", "derive_more", "futures-core", "http-range", @@ -97,7 +97,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.6.0", "brotli", - "bytes 1.7.1", + "bytes 1.7.2", "bytestring", "derive_more", "encoding_rs", @@ -135,7 +135,7 @@ dependencies = [ "actix-tls", "actix-utils", "awc", - "bytes 1.7.1", + "bytes 1.7.2", "futures-core", "http 0.2.12", "log", @@ -286,7 +286,7 @@ dependencies = [ "actix-utils", "actix-web-codegen", "ahash 0.8.11", - "bytes 1.7.1", + "bytes 1.7.2", "bytestring", "cfg-if 1.0.0", "cookie", @@ -322,7 +322,7 @@ dependencies = [ "actix-codec", "actix-http", "actix-web", - "bytes 1.7.1", + "bytes 1.7.2", "bytestring", "futures-core", "pin-project-lite", @@ -578,9 +578,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "appdirs" @@ -759,7 +759,7 @@ dependencies = [ "actix-tls", "actix-utils", "base64 0.22.1", - "bytes 1.7.1", + "bytes 1.7.2", "cfg-if 1.0.0", "cookie", "derive_more", @@ -1113,15 +1113,18 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "bytesize" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +dependencies = [ + "serde", +] [[package]] name = "bytestring" @@ -1129,7 +1132,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", ] [[package]] @@ -1211,9 +1214,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.18" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0" dependencies = [ "jobserver", "libc", @@ -2686,7 +2689,7 @@ dependencies = [ "log", "regex", "thiserror", - "yansi", + "yansi 0.5.1", ] [[package]] @@ -3055,7 +3058,7 @@ dependencies = [ [[package]] name = "golem-certificate" version = "0.1.1" -source = "git+https://github.com/golemfactory/golem-certificate.git?rev=952fdbd47adc57e46b7370935111e046271ef415#952fdbd47adc57e46b7370935111e046271ef415" +source = "git+https://github.com/golemfactory/golem-certificate.git?rev=f2d7514c18fc066e9cfb796090b90f5b27cfe1c6#f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" dependencies = [ "anyhow", "chrono", @@ -3110,7 +3113,7 @@ dependencies = [ "ya-core-model", "ya-provider", "ya-utils-path", - "ya-utils-process 0.3.0", + "ya-utils-process", ] [[package]] @@ -3139,7 +3142,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "fnv", "futures-core", "futures-sink", @@ -3159,7 +3162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", - "bytes 1.7.1", + "bytes 1.7.2", "fnv", "futures-core", "futures-sink", @@ -3225,7 +3228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ "base64 0.21.7", - "bytes 1.7.1", + "bytes 1.7.2", "headers-core", "http 0.2.12", "httpdate", @@ -3405,7 +3408,7 @@ version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "fnv", "itoa", ] @@ -3416,7 +3419,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "fnv", "itoa", ] @@ -3427,7 +3430,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "http 0.2.12", "pin-project-lite", ] @@ -3438,7 +3441,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "http 1.1.0", ] @@ -3448,7 +3451,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-util", "http 1.1.0", "http-body 1.0.1", @@ -3514,7 +3517,7 @@ version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-channel", "futures-core", "futures-util", @@ -3538,7 +3541,7 @@ version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-channel", "futures-util", "h2 0.4.6", @@ -3572,7 +3575,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "hyper 0.14.30", "native-tls", "tokio", @@ -3585,7 +3588,7 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-util", "http 1.1.0", "http-body 1.0.1", @@ -3596,9 +3599,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -4001,7 +4004,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", - "redox_syscall 0.5.3", + "redox_syscall 0.5.4", ] [[package]] @@ -4436,7 +4439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09b34bd91b9e5c5b06338d392463e1318d683cf82ec3d3af4014609be6e2108d" dependencies = [ "assert-json-diff", - "bytes 1.7.1", + "bytes 1.7.2", "colored", "futures-util", "http 1.1.0", @@ -5011,7 +5014,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall 0.5.3", + "redox_syscall 0.5.4", "smallvec", "windows-targets 0.52.6", ] @@ -5246,12 +5249,12 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ "diff", - "yansi", + "yansi 1.0.1", ] [[package]] @@ -5332,7 +5335,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.22.20", + "toml_edit 0.22.21", ] [[package]] @@ -5389,7 +5392,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "prost-derive 0.10.1", ] @@ -5399,7 +5402,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "prost-derive 0.12.6", ] @@ -5409,7 +5412,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae5a4388762d5815a9fc0dea33c56b021cdc8dde0c55e0c9ca57197254b0cab" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "cfg-if 1.0.0", "cmake", "heck 0.4.1", @@ -5431,7 +5434,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "heck 0.5.0", "itertools 0.12.1", "log", @@ -5478,7 +5481,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "prost 0.10.4", ] @@ -5743,9 +5746,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags 2.6.0", ] @@ -5821,7 +5824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64 0.21.7", - "bytes 1.7.1", + "bytes 1.7.2", "encoding_rs", "futures-core", "futures-util", @@ -5920,7 +5923,7 @@ checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec 1.0.1", "bytecheck", - "bytes 1.7.1", + "bytes 1.7.2", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -5947,7 +5950,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb919243f34364b6bd2fc10ef797edbfa75f33c252e7998527479c6d6b47e1ec" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "rustc-hex", ] @@ -5990,7 +5993,7 @@ checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" dependencies = [ "arrayvec 0.7.6", "borsh", - "bytes 1.7.1", + "bytes 1.7.2", "num-traits", "rand 0.8.5", "rkyv", @@ -6021,9 +6024,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno 0.3.9", @@ -6984,7 +6987,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2" dependencies = [ "base64 0.13.1", - "bytes 1.7.1", + "bytes 1.7.2", "futures 0.3.30", "httparse", "log", @@ -7049,7 +7052,7 @@ dependencies = [ "ahash 0.8.11", "atoi", "byteorder", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "crc", "crossbeam-queue 0.3.11", @@ -7130,7 +7133,7 @@ dependencies = [ "base64 0.21.7", "bitflags 2.6.0", "byteorder", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "crc", "digest 0.10.7", @@ -7739,7 +7742,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", - "bytes 1.7.1", + "bytes 1.7.2", "libc", "mio 1.0.2", "parking_lot 0.12.3", @@ -7783,11 +7786,10 @@ dependencies = [ [[package]] name = "tokio-openssl" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ffab79df67727f6acf57f1ff743091873c24c579b1e2ce4d8f53e47ded4d63d" +checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd" dependencies = [ - "futures-util", "openssl", "openssl-sys", "tokio", @@ -7867,7 +7869,7 @@ version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "futures-core", "futures-io", "futures-sink", @@ -7886,7 +7888,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.20", + "toml_edit 0.22.21", ] [[package]] @@ -7911,9 +7913,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "3b072cee73c449a636ffd6f32bd8de3a9f7119139aff882f44943ce2986dc5cf" dependencies = [ "indexmap 2.5.0", "serde", @@ -8110,15 +8112,15 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -8131,9 +8133,9 @@ checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524" [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" @@ -8177,7 +8179,7 @@ dependencies = [ "rustls 0.23.13", "rustls-pki-types", "url", - "webpki-roots 0.26.5", + "webpki-roots 0.26.6", ] [[package]] @@ -8430,7 +8432,7 @@ checksum = "5388522c899d1e1c96a4c307e3797e0f697ba7c77dd8e0e625ecba9dd0342937" dependencies = [ "arrayvec 0.7.6", "base64 0.21.7", - "bytes 1.7.1", + "bytes 1.7.2", "derive_more", "ethabi", "ethereum-types 0.14.1", @@ -8496,9 +8498,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -8521,7 +8523,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.3", + "redox_syscall 0.5.4", "wasite", ] @@ -8864,11 +8866,11 @@ dependencies = [ [[package]] name = "ya-client" version = "0.9.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" +source = "git+https://github.com/golemfactory/ya-client.git?rev=60327ddaaea289ecf45836c1e242e15855d71dc5#60327ddaaea289ecf45836c1e242e15855d71dc5" dependencies = [ "actix-codec", "awc", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "envy", "futures 0.3.30", @@ -8887,10 +8889,11 @@ dependencies = [ [[package]] name = "ya-client-model" -version = "0.7.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=dacad31b5bbd039b8ffc97adb70696655d0872ad#dacad31b5bbd039b8ffc97adb70696655d0872ad" +version = "0.6.0" +source = "git+https://github.com/golemfactory/ya-client.git?rev=60327ddaaea289ecf45836c1e242e15855d71dc5#60327ddaaea289ecf45836c1e242e15855d71dc5" dependencies = [ "bigdecimal 0.2.2", + "bytesize", "chrono", "derive_more", "diesel", @@ -8949,7 +8952,7 @@ dependencies = [ "actix", "actix-rt", "anyhow", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "futures 0.3.30", "lazy_static", @@ -8963,7 +8966,7 @@ dependencies = [ "tokio", "winapi 0.3.9", "ya-agreement-utils", - "ya-utils-process 0.3.0", + "ya-utils-process", ] [[package]] @@ -9054,7 +9057,7 @@ dependencies = [ "anyhow", "async-stream", "async-trait", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "derivative", "derive_more", @@ -9108,8 +9111,8 @@ dependencies = [ "ya-utils-futures", "ya-utils-networking", "ya-utils-path", - "ya-utils-process 0.3.0", - "yansi", + "ya-utils-process", + "yansi 0.5.1", ] [[package]] @@ -9128,7 +9131,7 @@ dependencies = [ "chrono", "flexi_logger 0.17.1", "log", - "yansi", + "yansi 0.5.1", ] [[package]] @@ -9144,7 +9147,7 @@ dependencies = [ "async-compression", "async-trait", "awc", - "bytes 1.7.1", + "bytes 1.7.2", "cargo_metadata 0.18.1", "crossterm 0.26.1", "env_logger 0.7.1", @@ -9248,7 +9251,7 @@ dependencies = [ "anyhow", "awc", "base64 0.21.7", - "bytes 1.7.1", + "bytes 1.7.2", "ctor", "env_logger 0.10.2", "flexbuffers", @@ -9281,7 +9284,7 @@ dependencies = [ "actix-web", "anyhow", "async-stream", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "derive_more", "env_logger 0.10.2", @@ -9352,7 +9355,7 @@ dependencies = [ "ya-service-api-derive", "ya-service-api-interfaces", "ya-service-bus", - "yansi", + "yansi 0.5.1", ] [[package]] @@ -9512,7 +9515,7 @@ dependencies = [ "actix", "anyhow", "async-trait", - "bytes 1.7.1", + "bytes 1.7.2", "env_logger 0.10.2", "futures 0.3.30", "hex", @@ -9542,7 +9545,7 @@ dependencies = [ "actix", "actix-web", "anyhow", - "bytes 1.7.1", + "bytes 1.7.2", "chrono", "env_logger 0.7.1", "ethsign", @@ -9716,7 +9719,7 @@ dependencies = [ "ya-core-model", "ya-service-api", "ya-service-api-interfaces", - "ya-utils-process 0.3.0", + "ya-utils-process", ] [[package]] @@ -9791,14 +9794,14 @@ dependencies = [ "ya-utils-actix", "ya-utils-cli", "ya-utils-path", - "ya-utils-process 0.3.0", - "yansi", + "ya-utils-process", + "yansi 0.5.1", ] [[package]] name = "ya-relay-client" version = "0.6.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" dependencies = [ "anyhow", "async-trait", @@ -9813,7 +9816,6 @@ dependencies = [ "metrics 0.21.1", "num_cpus", "parking_lot 0.12.3", - "rand 0.8.5", "strum 0.25.0", "strum_macros 0.25.3", "thiserror", @@ -9829,7 +9831,7 @@ dependencies = [ [[package]] name = "ya-relay-core" version = "0.4.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" dependencies = [ "anyhow", "chrono", @@ -9859,10 +9861,10 @@ dependencies = [ [[package]] name = "ya-relay-proto" version = "0.4.3" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" dependencies = [ "anyhow", - "bytes 1.7.1", + "bytes 1.7.2", "derive_more", "futures 0.3.30", "hex", @@ -9873,7 +9875,7 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252)", ] [[package]] @@ -9899,7 +9901,7 @@ dependencies = [ [[package]] name = "ya-relay-stack" version = "0.5.1" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" dependencies = [ "derive_more", "futures 0.3.30", @@ -9913,7 +9915,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489)", + "ya-relay-util 0.1.0 (git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252)", ] [[package]] @@ -9922,16 +9924,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "306a73f6ce2286987c9da25bc0c2ef81f4f0b2b58bb8d9aeedc34d27407603ff" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "derive_more", ] [[package]] name = "ya-relay-util" version = "0.1.0" -source = "git+https://github.com/golemfactory/ya-relay.git?rev=e199ee1cfdb22837f9d95f4202378e182d3cb489#e199ee1cfdb22837f9d95f4202378e182d3cb489" +source = "git+https://github.com/golemfactory/ya-relay.git?rev=0588dd1af311ae19c621b04cc2a4cfd9c0483252#0588dd1af311ae19c621b04cc2a4cfd9c0483252" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "derive_more", ] @@ -9942,7 +9944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf0db25811f107d62be6c6ac7444d9c6c3e39714b6f76d72798b66ecce47506f" dependencies = [ "anyhow", - "bytes 1.7.1", + "bytes 1.7.2", "futures 0.3.30", "log", "prost 0.10.4", @@ -9959,7 +9961,7 @@ name = "ya-runtime-api" version = "0.7.1" dependencies = [ "anyhow", - "bytes 1.7.1", + "bytes 1.7.2", "env_logger 0.7.1", "futures 0.3.30", "log", @@ -9979,7 +9981,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69e6b13e3a7d0deab2421046e7fd5b498bd30b9d51d068b3fb4e3278a48bb63f" dependencies = [ - "bytes 1.7.1", + "bytes 1.7.2", "prost 0.10.4", "prost-build 0.10.4", "thiserror", @@ -10179,7 +10181,7 @@ dependencies = [ "tokio", "url", "ya-framework-macro", - "ya-utils-process 0.3.0", + "ya-utils-process", ] [[package]] @@ -10195,7 +10197,7 @@ dependencies = [ "async-compression", "async-trait", "awc", - "bytes 1.7.1", + "bytes 1.7.2", "crossterm 0.26.1", "env_logger 0.7.1", "futures 0.3.30", @@ -10225,7 +10227,6 @@ dependencies = [ "tokio-util", "url", "walkdir", - "ya-client", "ya-client-model", "ya-core-model", "ya-exe-unit", @@ -10296,24 +10297,6 @@ dependencies = [ "log", ] -[[package]] -name = "ya-utils-process" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ae347dbc4362cc24241f96ddb7c4bc46fdc6f4772c2f92d12b40af359b81ef" -dependencies = [ - "actix", - "anyhow", - "derive_more", - "fs2", - "futures 0.3.30", - "futures-util", - "libc", - "nix 0.22.3", - "shared_child", - "tokio", -] - [[package]] name = "ya-utils-process" version = "0.3.0" @@ -10370,7 +10353,7 @@ dependencies = [ "actix-web", "actix-web-actors", "anyhow", - "bytes 1.7.1", + "bytes 1.7.2", "env_logger 0.7.1", "futures 0.3.30", "hex", @@ -10460,7 +10443,7 @@ dependencies = [ "ya-utils-futures", "ya-utils-networking", "ya-utils-path", - "ya-utils-process 0.2.0", + "ya-utils-process", "ya-version", "ya-vpn", ] @@ -10480,6 +10463,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "zerocopy" version = "0.7.35" diff --git a/Cargo.toml b/Cargo.toml index db21e038e8..4bbb252694 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,13 @@ version = "0.16.0" default = ['erc20-driver', 'gftp/bin'] dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] -framework-test = ['ya-exe-unit/system-test', 'ya-payment/framework-test', 'ya-identity/framework-test'] static-openssl = ["openssl/vendored", "openssl-probe"] tos = [] +framework-test = [ + 'ya-exe-unit/framework-test', + 'ya-payment/framework-test', + 'ya-identity/framework-test', +] # Temporary to make goth integration tests work central-net = ['ya-net/central-net'] packet-trace-enable = [ @@ -30,44 +34,45 @@ name = "yagna" path = "core/serv/src/main.rs" [dependencies] -gftp = {workspace = true, optional = true} -ya-activity = "0.4" -ya-client-model.workspace = true -ya-client.workspace = true -ya-compile-time-utils = "0.2" +ya-compile-time-utils.workspace = true +ya-activity.workspace = true ya-core-model.workspace = true -ya-dummy-driver = {version = "0.3", optional = true} -ya-erc20-driver = {version = "0.4", optional = true} -ya-fd-metrics = {path = "utils/fd-metrics"} -ya-file-logging = "0.1" -ya-gsb-api = "0.1" -ya-identity = "0.3" -ya-market = "0.4" -ya-metrics = "0.2" -ya-net = {version = "0.3", features = ["service"]} -ya-payment = "0.3" -ya-persistence = {version = "0.3", features = ["service"]} -ya-sb-proto = {workspace = true} -ya-sb-router = {workspace = true} -ya-service-api = "0.1" -ya-service-api-derive = "0.2" -ya-service-api-interfaces = "0.2" -ya-service-api-web = "0.2" -ya-service-bus = {workspace = true} -ya-sgx = "0.2" +ya-dummy-driver = { workspace = true, optional = true } +ya-file-logging.workspace = true +ya-gsb-api.workspace = true +ya-erc20-driver = { workspace = true, optional = true } +ya-identity.workspace = true +ya-market.workspace = true +ya-metrics.workspace = true +ya-net = { workspace = true, features = ["service"] } +ya-payment.workspace = true +ya-persistence = { path = "core/persistence", features = ["service"] } +ya-sb-proto = { workspace = true } +ya-sb-router = { workspace = true } +ya-service-api.workspace = true +ya-service-api-derive.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-api-web.workspace = true +ya-service-bus = { workspace = true } +ya-sgx.path = "core/sgx" +ya-utils-path.workspace = true ya-utils-futures.workspace = true -ya-utils-networking = "0.2" -ya-utils-path = "0.1" -ya-utils-process = {version = "0.2", features = ["lock"]} -ya-version = "0.2" -ya-vpn = "0.2" +ya-utils-process = { workspace = true, features = ["lock"] } +ya-utils-networking.workspace = true +ya-fd-metrics = { path = "utils/fd-metrics" } +ya-version = { path = "core/version" } +ya-vpn.workspace = true +ya-client.workspace = true +ya-client-model.workspace = true +gftp = { workspace = true, optional = true } # just to enable gftp build for cargo-deb -ya-exe-unit = {version = "0.4", optional = true, path = "exe-unit"}# just to enable conditionally running some tests -ya-provider = {version = "0.3", optional = true}# just to enable conditionally running some tests +ya-provider = { path = "agent/provider", optional = true } # just to enable conditionally running some tests +ya-exe-unit = { version = "0.4", optional = true, path = "exe-unit" } # just to enable conditionally running some tests + +actix-rt.workspace = true +actix-service.workspace = true +actix-web.workspace = true -actix-rt = "2.7" -actix-service = "2" -actix-web = "4" anyhow = "1.0" chrono = "0.4" directories = "2.0.2" @@ -89,9 +94,10 @@ tokio-util = {version = "0.7", features = ["codec"]} url = "2.1.1" [dev-dependencies] -erc20_processor = {workspace = true} -ya-exe-unit = {version = "0.4", path = "exe-unit"} -ya-test-framework = "0.1" +erc20_processor = { workspace = true } +ya-test-framework.path= "test-utils/test-framework" + +ya-exe-unit = { version = "0.4", path = "exe-unit" } [package.metadata.deb] assets = [ @@ -236,6 +242,11 @@ members = [ # this entry is needed to make sqlx version >=0.5.9 work with diesel 1.4.* # diesel 1.4.* supports up to 0.23.0, but sqlx 0.5.9 requires 0.22.0 # sqlx 0.5.10 need 0.23.2, so 0.5.9 is last version possible +actix-rt = "2.7" +actix-service = "2" +actix-web = "4" +actix = { version = "0.13", default-features = false } + derive_more = "0.99.11" erc20_payment_lib = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} erc20_processor = {git = "https://github.com/golemfactory/erc20_payment_lib", rev = "4200567b931af64f4fb1f6b756dd6d051576b64f"} @@ -253,93 +264,65 @@ strum = {version = "0.24", features = ["derive"]} trust-dns-resolver = "0.22" url = "2.3.1" -ya-agreement-utils = {version = "0.6", path = "utils/agreement-utils"} -ya-core-model = {version = "0.10", path = "core/model"} -ya-relay-client = {git = "https://github.com/golemfactory/ya-relay.git", rev = "e199ee1cfdb22837f9d95f4202378e182d3cb489"} -ya-relay-stack = {git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5"} -ya-utils-futures = {path = "utils/futures"} +ya-agreement-utils = { version = "0.6", path = "utils/agreement-utils" } +ya-exe-unit.path = "./exe-unit" +ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "0588dd1af311ae19c621b04cc2a4cfd9c0483252" } +ya-relay-stack = { git = "https://github.com/golemfactory/ya-relay.git", rev = "c92a75b0cf062fcc9dbb3ea2a034d913e5fad8e5" } +ya-utils-futures = { path = "utils/futures" } +ya-utils-networking = { path = "utils/networking", default-features = false } +ya-file-logging.path = "utils/file-logging" +ya-utils-cli.path = "utils/cli" -mime = "0.3.17" +ya-service-bus = { version = "0.7.3", features = ['tls'] } +ya-sb-router = { version = "0.6.4" } +ya-sb-proto = { version = "0.6.2" } +ya-sb-util = { version = "0.5.1" } parking_lot = "0.12.3" -ya-sb-proto = {version = "0.6.2"} -ya-sb-router = {version = "0.6.4"} -ya-sb-util = {version = "0.5.1"} -ya-service-bus = {version = "0.7.3", features = ['tls']} -# true version is given in patches section -ya-client = "0.9" -# true version is given in patches section -ya-client-model = "0.7" - -[patch.crates-io] -## SERVICES -ya-activity = {path = "core/activity"} -ya-dummy-driver = {path = "core/payment-driver/dummy"} -ya-erc20-driver = {path = "core/payment-driver/erc20"} -ya-gsb-api = {path = "core/gsb-api"} -ya-identity = {path = "core/identity"} -ya-market = {path = "core/market"} -ya-market-resolver = {path = "core/market/resolver"} -ya-net = {path = "core/net"} -ya-payment = {path = "core/payment"} -ya-payment-driver = {path = "core/payment-driver/base"} -ya-sgx = {path = "core/sgx"} -ya-version = {path = "core/version"} -ya-vpn = {path = "core/vpn"} - -## CORE UTILS -ya-core-model = {path = "core/model"} -ya-persistence = {path = "core/persistence"} -ya-service-api = {path = "core/serv-api"} -ya-service-api-derive = {path = "core/serv-api/derive"} -ya-service-api-interfaces = {path = "core/serv-api/interfaces"} -ya-service-api-web = {path = "core/serv-api/web"} - -## CLIENT -ya-client = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} -#ya-client = { path = "../ya-client" } -ya-client-model = {git = "https://github.com/golemfactory/ya-client.git", rev = "dacad31b5bbd039b8ffc97adb70696655d0872ad"} -#ya-client-model = { path = "../ya-client/model" } -golem-certificate = {git = "https://github.com/golemfactory/golem-certificate.git", rev = "952fdbd47adc57e46b7370935111e046271ef415"} +mime = "0.3.17" +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "60327ddaaea289ecf45836c1e242e15855d71dc5" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "60327ddaaea289ecf45836c1e242e15855d71dc5" } -## RELAY and networking stack +ya-compile-time-utils.path = "utils/compile-time-utils" +ya-manifest-utils = { path = "utils/manifest-utils" } +ya-std-utils = { path = "utils/std-utils" } +ya-diesel-utils.path = "utils/diesel-utils" +ya-utils-actix.path = "utils/actix_utils" +ya-core-model = { path = "core/model" } +ya-utils-path.path = "utils/path" +ya-utils-process.path = "utils/process" -#ya-relay-stack = { path = "../ya-relay/crates/stack" } -#ya-relay-client = { path = "../ya-relay/client" } -#ya-relay-core = { path = "../ya-relay/crates/core" } -#ya-relay-proto = { path = "../ya-relay/crates/proto" } +ya-identity.path = "core/identity" +ya-market.path="core/market" +ya-activity.path = "core/activity" +ya-net.path = "core/net" +ya-persistence.path = "core/persistence" +ya-payment.path = "core/payment" +ya-metrics.path = "core/metrics" +ya-manifest-test-utils.path = "utils/manifest-utils/test-utils" +ya-vpn.path = "core/vpn" +ya-gsb-api.path = "core/gsb-api" -## OTHERS -gftp = {path = "core/gftp"} -tokio-process-ns = {path = "exe-unit/tokio-process-ns"} -ya-agreement-utils = {path = "utils/agreement-utils"} -ya-compile-time-utils = {path = "utils/compile-time-utils"} -ya-counters = {path = "exe-unit/components/counters"} -ya-diesel-utils = {path = "utils/diesel-utils"} -ya-exe-unit = {path = "exe-unit"} -ya-file-logging = {path = "utils/file-logging"} -ya-gsb-http-proxy = {path = "exe-unit/components/gsb-http-proxy"} -ya-manifest-utils = {path = "utils/manifest-utils"} -ya-metrics = {path = "core/metrics"} -ya-provider = {path = "agent/provider"} -ya-std-utils = {path = "utils/std-utils"} -ya-transfer = {path = "exe-unit/components/transfer"} -ya-utils-actix = {path = "utils/actix_utils"} -ya-utils-cli = {path = "utils/cli"} -ya-utils-networking = {path = "utils/networking"} -ya-utils-path = {path = "utils/path"} -ya-utils-process = {path = "utils/process"} +ya-payment-driver.path = "core/payment-driver/base" +ya-dummy-driver.path= "core/payment-driver/dummy" +ya-erc20-driver.path = "core/payment-driver/erc20" -## TEST UTILS -ya-framework-basic = {path = "test-utils/test-framework/framework-basic"} -ya-framework-macro = {path = "test-utils/test-framework/framework-macro"} -ya-framework-mocks = {path = "test-utils/test-framework/framework-mocks"} -ya-manifest-test-utils = {path = "utils/manifest-utils/test-utils"} -ya-test-framework = {path = "test-utils/test-framework"} +ya-service-api.path = "core/serv-api" +ya-service-api-derive.path = "core/serv-api/derive" +ya-service-api-interfaces.path = "core/serv-api/interfaces" +ya-service-api-web.path = "core/serv-api/web" -ethereum-tx-sign = {git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195"} -graphene-sgx = {git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407"} +ya-framework-macro.path = "test-utils/test-framework/framework-macro" +ya-framework-basic.path = "test-utils/test-framework/framework-basic" +ya-framework-mocks.path = "test-utils/test-framework/framework-mocks" -diesel = {git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563"} +[patch.crates-io] +## CLIENT +#ya-client = { git = "https://github.com/golemfactory/ya-client.git", branch = "kek/deploy-volumes" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "60327ddaaea289ecf45836c1e242e15855d71dc5" } +golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } +ethereum-tx-sign = { git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195" } +graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407" } +diesel = { git = "https://github.com/golemfactory/yagna-diesel-patch.git", rev = "a512c66d520a9066dd9a4d1416f9109019b39563" } # Speed up builds on macOS (will be default in next rust version probably) # https://jakedeichert.com/blog/reducing-rust-incremental-compilation-times-on-macos-by-70-percent/ diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml index ee7dd15a2f..2ee25e9480 100644 --- a/agent/provider/Cargo.toml +++ b/agent/provider/Cargo.toml @@ -14,17 +14,17 @@ path = "src/main.rs" [dependencies] ya-agreement-utils = { workspace = true } -ya-manifest-utils = { version = "0.2" } +ya-manifest-utils.workspace = true ya-client = { workspace = true, features = ['cli'] } ya-client-model.workspace = true -ya-compile-time-utils = "0.2" +ya-compile-time-utils.workspace = true ya-core-model = { workspace = true, features = ['activity', 'payment'] } -ya-file-logging = "0.1" -ya-utils-actix = "0.2" -ya-utils-cli = "0.1" -ya-utils-path = "0.1" -ya-utils-process = { version = "0.3", features = ['lock'] } -ya-std-utils = "0.1" +ya-file-logging.workspace = true +ya-utils-actix.workspace = true +ya-utils-cli.workspace = true +ya-utils-path.workspace = true +ya-utils-process = { workspace = true, features = ['lock'] } +ya-std-utils.workspace = true golem-certificate = "0.1.1" actix = { version = "0.13", default-features = false } @@ -91,8 +91,8 @@ tempdir = "0.3" tempfile = "3.5.0" pretty_assertions = "1.3" -ya-manifest-test-utils = "0.1" -ya-framework-basic = "0.1" +ya-manifest-test-utils.workspace = true +ya-framework-basic.workspace = true [lints] workspace = true diff --git a/core/activity/Cargo.toml b/core/activity/Cargo.toml index f3164c23d7..0c6ec40e56 100644 --- a/core/activity/Cargo.toml +++ b/core/activity/Cargo.toml @@ -7,11 +7,11 @@ edition = "2018" [dependencies] ya-core-model = { workspace = true, features = ["activity", "market"] } ya-client-model = { workspace = true, features = ["sgx"] } -ya-net = "0.3" -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" -ya-service-api-web = "0.2" +ya-net.workspace = true +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-api-web.workspace = true ya-service-bus = { workspace = true } ya-gsb-http-proxy = { path = "../../exe-unit/components/gsb-http-proxy" } diff --git a/core/gftp/Cargo.toml b/core/gftp/Cargo.toml index cdacdc72a7..e38790e51f 100644 --- a/core/gftp/Cargo.toml +++ b/core/gftp/Cargo.toml @@ -18,7 +18,7 @@ name = "gftp" required-features = ['bin'] [dependencies] -ya-compile-time-utils = "0.2" +ya-compile-time-utils.workspace = true ya-core-model = { workspace = true, features = ["gftp", "identity", "net"] } ya-service-bus = { workspace = true } diff --git a/core/gsb-api/Cargo.toml b/core/gsb-api/Cargo.toml index dd8c889710..17c5cf625b 100644 --- a/core/gsb-api/Cargo.toml +++ b/core/gsb-api/Cargo.toml @@ -7,10 +7,10 @@ edition = "2018" [dependencies] ya-client-model.workspace = true -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" -ya-service-api-web = "0.2" +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-api-web.workspace = true ya-service-bus = { workspace = true } actix = "0" diff --git a/core/identity/Cargo.toml b/core/identity/Cargo.toml index 10238f7299..773f85559e 100644 --- a/core/identity/Cargo.toml +++ b/core/identity/Cargo.toml @@ -12,9 +12,9 @@ framework-test = [] [dependencies] ya-client-model = { workspace = true, features = ["with-diesel"] } ya-core-model = { workspace = true, features = ["identity", "appkey"] } -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true ya-service-bus = { workspace = true } anyhow = "1.0" @@ -41,10 +41,10 @@ rustc-hex = "2.1.0" yansi = "0.5.0" [dev-dependencies] -ya-service-api-derive = "0.2" +ya-service-api-derive.workspace = true ya-sb-router = { workspace = true } -ya-framework-basic = { version = "0.1" } -ya-framework-mocks = { version = "0.1" } +ya-framework-basic.workspace = true +ya-framework-mocks.workspace = true actix-rt = "2.7" actix-service = "2" @@ -57,4 +57,4 @@ serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "act test-context = "0.1.4" [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/core/market/Cargo.toml b/core/market/Cargo.toml index 8036bd3c3d..76d1fb5cd9 100644 --- a/core/market/Cargo.toml +++ b/core/market/Cargo.toml @@ -12,17 +12,17 @@ test-suite = [] ya-agreement-utils = { workspace = true } ya-client.workspace = true ya-core-model = { workspace = true, features = ["market", "net"] } -ya-diesel-utils = { version = "0.1" } -ya-framework-basic = "0.1" -ya-market-resolver = "0.2" -ya-net = "0.3" -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" -ya-service-api-web = "0.2" +ya-diesel-utils.workspace = true +ya-framework-basic.workspace = true +ya-market-resolver.path = "./resolver" +ya-net.workspace = true +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-api-web.workspace = true ya-service-bus = { workspace = true } -ya-std-utils = "0.1" -ya-utils-actix = "0.2" +ya-std-utils.workspace = true +ya-utils-actix.workspace = true actix = { version = "0.13", default-features = false } actix-http = "3" @@ -73,7 +73,7 @@ serial_test = { git = "https://github.com/golemfactory/serial_test.git", branch structopt = "0.3" tokio = { version = "1", features = ["macros", "rt"] } -ya-framework-mocks = "0.1" +ya-framework-mocks.workspace = true [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/core/metrics/Cargo.toml b/core/metrics/Cargo.toml index 7d3a2dcd8f..d54fdff916 100644 --- a/core/metrics/Cargo.toml +++ b/core/metrics/Cargo.toml @@ -11,9 +11,9 @@ license = "LGPL-3.0" [dependencies] ya-core-model = { workspace = true, features = ["identity"] } -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" -ya-service-bus = { workspace = true } +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-bus = { workspace = true } awc = "3" actix-web = { version = "4", features = ["openssl"] } diff --git a/core/net/Cargo.toml b/core/net/Cargo.toml index af0fc30efd..dce7433912 100644 --- a/core/net/Cargo.toml +++ b/core/net/Cargo.toml @@ -22,14 +22,14 @@ ya-relay-client = { workspace = true } ya-sb-proto = { workspace = true } ya-sb-util = { workspace = true } -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true ya-service-bus = { workspace = true, features = ["tls"] } -ya-utils-networking = "0.2" +ya-utils-networking.workspace = true ya-packet-trace = { git = "https://github.com/golemfactory/ya-packet-trace" } -actix = "0.13" -actix-web = "4" +actix.workspace = true +actix-web.workspace = true anyhow = "1.0" chrono = "0.4" futures = "0.3" diff --git a/core/payment-driver/base/Cargo.toml b/core/payment-driver/base/Cargo.toml index 885c92fd90..8dcc3e3572 100644 --- a/core/payment-driver/base/Cargo.toml +++ b/core/payment-driver/base/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" default = [] [dependencies] -actix = { version = "0.13", default-features = false } +actix = { workspace = true, default-features = false } anyhow = "1.0" async-trait = "0.1" bigdecimal = { version = "0.2" } @@ -38,7 +38,7 @@ ya-core-model = { workspace = true, features = [ "identity", "payment", ] } -ya-persistence = "0.3" +ya-persistence.workspace = true ya-service-bus = { workspace = true } diff --git a/core/payment-driver/dummy/Cargo.toml b/core/payment-driver/dummy/Cargo.toml index 3633a76df9..ab3b430e57 100644 --- a/core/payment-driver/dummy/Cargo.toml +++ b/core/payment-driver/dummy/Cargo.toml @@ -14,9 +14,9 @@ ya-core-model = { workspace = true, features = [ "payment", ] } ya-client-model.workspace = true -ya-payment-driver = "0.3" -ya-persistence = "0.3" -ya-service-api-interfaces = "0.2" +ya-payment-driver.workspace = true +ya-persistence.workspace = true +ya-service-api-interfaces.workspace = true ya-service-bus = { workspace = true } anyhow = "1.0" diff --git a/core/payment-driver/erc20/Cargo.toml b/core/payment-driver/erc20/Cargo.toml index 94563eefe1..1b2406e3ef 100644 --- a/core/payment-driver/erc20/Cargo.toml +++ b/core/payment-driver/erc20/Cargo.toml @@ -42,13 +42,13 @@ web3 = { version = "0.19.0", default-features = false, features = [ rust_decimal = "1" ## yagna dependencies -ya-payment-driver = "0.3" -ya-core-model = { workspace = true } +ya-payment-driver.workspace = true +ya-core-model.workspace = true ya-client-model.workspace = true -ya-service-api-interfaces = "0.2" +ya-service-api-interfaces.workspace = true ya-utils-futures.workspace = true -ya-utils-networking = "0.2" -erc20_payment_lib = { workspace = true } +ya-utils-networking.workspace = true +erc20_payment_lib.workspace = true [dev-dependencies] actix-rt = "2.7" diff --git a/core/payment/Cargo.toml b/core/payment/Cargo.toml index 01898cddd7..19a47772a1 100644 --- a/core/payment/Cargo.toml +++ b/core/payment/Cargo.toml @@ -18,14 +18,14 @@ ya-core-model = { workspace = true, features = [ "market", "payment", ] } -ya-net = "0.3" -ya-metrics = "0.2" -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" -ya-service-api-web = "0.2" +ya-net.workspace = true +ya-metrics.workspace = true +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-api-web.workspace = true ya-service-bus = { workspace = true } -ya-std-utils = "0.1" +ya-std-utils.workspace = true actix-web = "4" anyhow = "1.0" @@ -66,16 +66,16 @@ uint = "0.7" uuid = { version = "0.8", features = ["v4"] } [dev-dependencies] -ya-client = { workspace = true } -ya-dummy-driver = "0.3" -ya-erc20-driver = "0.4" -ya-net = { version = "0.3", features = ["service"] } -ya-identity = "0.3" -ya-market = "0.4" +ya-client.workspace = true +ya-dummy-driver.workspace = true +ya-erc20-driver.workspace = true +ya-net = { workspace = true, features = ["service"] } +ya-identity.workspace = true +ya-market.workspace = true ya-sb-router = { workspace = true } -ya-framework-basic = { version = "0.1" } -ya-framework-mocks = { version = "0.1" } -ya-payment-driver = "0.3" +ya-framework-basic.workspace = true +ya-framework-mocks.workspace = true +ya-payment-driver.workspace = true actix-rt = "2.7" rand = "0.8" diff --git a/core/persistence/Cargo.toml b/core/persistence/Cargo.toml index e05d6a3931..b2b649272d 100644 --- a/core/persistence/Cargo.toml +++ b/core/persistence/Cargo.toml @@ -15,10 +15,10 @@ service = [ [dependencies] ya-client-model = { workspace = true, features = ["with-diesel"] } -ya-core-model = { workspace = true } -ya-service-api = { version = "0.1", optional = true } -ya-service-api-interfaces = { version = "0.2", optional = true } -ya-utils-process = { version = "0.3", features = ["lock"], optional = true } +ya-core-model.workspace = true +ya-service-api = { workspace = true, optional = true } +ya-service-api-interfaces = { workspace = true, optional = true } +ya-utils-process = { workspace = true, features = ["lock"], optional = true } anyhow = "1.0.26" bigdecimal = "0.2" diff --git a/core/serv-api/Cargo.toml b/core/serv-api/Cargo.toml index 3f6d97e568..190f329a46 100644 --- a/core/serv-api/Cargo.toml +++ b/core/serv-api/Cargo.toml @@ -11,5 +11,5 @@ lazy_static = "1.4" serde = { version = "1.0", features = ["derive"] } url = "2.1.1" -ya-utils-cli = "0.1" -ya-core-model = { workspace = true } +ya-utils-cli.workspace = true +ya-core-model.workspace = true diff --git a/core/serv-api/derive/Cargo.toml b/core/serv-api/derive/Cargo.toml index 1f399fa945..911ee34a4c 100644 --- a/core/serv-api/derive/Cargo.toml +++ b/core/serv-api/derive/Cargo.toml @@ -17,11 +17,11 @@ strum_macros = "0.24" syn = { version = "1.0", features = ["full"] } [dev-dependencies] -ya-service-api-interfaces = "0.2" +ya-service-api-interfaces.workspace = true -actix-rt = "2.7" -actix-service = "2" -actix-web = "4" +actix-rt.workspace = true +actix-service.workspace = true +actix-web.workspace = true anyhow = "1.0" log = "0.4" structopt = "0.3" diff --git a/core/serv-api/web/Cargo.toml b/core/serv-api/web/Cargo.toml index 7683913e3a..54d3e350cd 100644 --- a/core/serv-api/web/Cargo.toml +++ b/core/serv-api/web/Cargo.toml @@ -8,8 +8,8 @@ edition = "2018" [dependencies] ya-client.workspace = true ya-core-model = { workspace = true, features = ["appkey"] } -ya-service-api = "0.1" -ya-service-bus = { workspace = true } +ya-service-api.workspace = true +ya-service-bus = { workspace = true } actix-cors = "0.6" actix-service = "2" @@ -23,11 +23,11 @@ structopt = "0.3" url = "2.1.1" [dev-dependencies] -ya-identity = "0.3" -ya-persistence = "0.3" +ya-identity.workspace = true +ya-persistence.workspace = true ya-sb-router = { workspace = true } -ya-service-api-derive = "0.2" -ya-service-api-interfaces = "0.2" +ya-service-api-derive.workspace = true +ya-service-api-interfaces.workspace = true actix-rt = "2.7" anyhow = "1.0" diff --git a/core/version/Cargo.toml b/core/version/Cargo.toml index 68ca971a8d..9519d06a0a 100644 --- a/core/version/Cargo.toml +++ b/core/version/Cargo.toml @@ -7,12 +7,12 @@ edition = "2018" [dependencies] ya-client.workspace = true -ya-compile-time-utils = "0.2" +ya-compile-time-utils.workspace = true ya-core-model = { workspace = true, features = ["version"] } -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" -ya-service-bus = { workspace = true } +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-bus = { workspace = true } actix-web = "4" anyhow = "1.0" diff --git a/core/vpn/Cargo.toml b/core/vpn/Cargo.toml index ac2ffa89ef..7fdbe451a8 100644 --- a/core/vpn/Cargo.toml +++ b/core/vpn/Cargo.toml @@ -7,15 +7,13 @@ edition = "2018" [dependencies] ya-core-model = { workspace = true, features = ["activity", "market"] } ya-client-model = { workspace = true, features = ["sgx"] } -ya-net = "0.3" -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-interfaces = "0.2" -ya-service-api-web = "0.2" +ya-net.workspace = true +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-interfaces.workspace = true +ya-service-api-web.workspace = true ya-service-bus = { workspace = true } -ya-utils-networking = { version = "0.2", default-features = false, features = [ - "vpn", -] } +ya-utils-networking = { workspace = true, default-features = false, features = [ "vpn", ] } ya-packet-trace = { git = "https://github.com/golemfactory/ya-packet-trace" } actix = "0.13" diff --git a/exe-unit/Cargo.toml b/exe-unit/Cargo.toml index ff1eb42248..d697ee198d 100644 --- a/exe-unit/Cargo.toml +++ b/exe-unit/Cargo.toml @@ -24,30 +24,32 @@ sgx = [ 'ya-core-model/sgx', 'ya-transfer/sgx', ] -system-test = ["ya-transfer/system-test"] +framework-test = [] [dependencies] ya-agreement-utils = {workspace = true} ya-client-model.workspace = true -ya-compile-time-utils = "0.2" +ya-compile-time-utils.workspace = true ya-core-model = {workspace = true, features = ["activity", "appkey"]} -ya-counters = {path = "../exe-unit/components/counters", features = ["os"]} +ya-counters = {path = "./components/counters", features = ["os"]} ya-gsb-http-proxy = {path = "../exe-unit/components/gsb-http-proxy"} -ya-manifest-utils = {version = "0.2"} +ya-manifest-utils.workspace = true ya-packet-trace = {git = "https://github.com/golemfactory/ya-packet-trace"} ya-runtime-api = {version = "0.7", path = "runtime-api", features = [ "server", ]} ya-service-bus = {workspace = true} -ya-std-utils = "0.1" -ya-transfer = "0.3" +ya-std-utils.workspace = true +ya-transfer.path = "components/transfer" ya-utils-futures.workspace = true -ya-utils-networking = {version = "0.2", default-features = false, features = [ - "dns", - "vpn", -]} -ya-utils-path = "0.1" -ya-utils-process = "0.3" +ya-utils-networking = { workspace = true, default-features = false, features = [ + "dns", + "vpn", +] } +ya-utils-process.workspace = true +ya-utils-path.workspace = true + + actix = {version = "0.13", default-features = false} actix-rt = "2.7" @@ -110,7 +112,7 @@ shell-words = "1.0.0" tempdir = "0.3.7" test-context = "0.1.4" -ya-framework-basic = {version = "0.1"} +ya-framework-basic.workspace = true ya-mock-runtime = {path = "components/mock-runtime"} [lints] diff --git a/exe-unit/components/counters/Cargo.toml b/exe-unit/components/counters/Cargo.toml index 585fcb827a..f6cff54e44 100644 --- a/exe-unit/components/counters/Cargo.toml +++ b/exe-unit/components/counters/Cargo.toml @@ -6,7 +6,7 @@ description = "Runtime counters generating data used to collect activity metrics [dependencies] ya-agreement-utils = { workspace = true } -ya-utils-process = { version = "0.3", optional = true } +ya-utils-process = { workspace = true, optional = true } anyhow = "1.0" actix = "0.13" diff --git a/exe-unit/components/gsb-http-proxy/Cargo.toml b/exe-unit/components/gsb-http-proxy/Cargo.toml index ced2888965..d625aae187 100644 --- a/exe-unit/components/gsb-http-proxy/Cargo.toml +++ b/exe-unit/components/gsb-http-proxy/Cargo.toml @@ -39,5 +39,5 @@ mockito = "1.2" serial_test = { git = "https://github.com/tworec/serial_test.git", branch = "actix_rt_test", features = ["actix-rt2"] } test-context = "0.1.4" -ya-framework-basic = { version = "0.1" } +ya-framework-basic.workspace = true ya-sb-router = "0.6.2" diff --git a/exe-unit/components/mock-runtime/Cargo.toml b/exe-unit/components/mock-runtime/Cargo.toml index 564a6e5e67..e35a2ab04a 100644 --- a/exe-unit/components/mock-runtime/Cargo.toml +++ b/exe-unit/components/mock-runtime/Cargo.toml @@ -30,9 +30,9 @@ ya-runtime-api = "0.7" # Dependancies for ExeUnit testing utils ya-client-model.workspace = true ya-core-model = {workspace = true, features = ["activity", "appkey"]} -ya-exe-unit = "0.4" -ya-framework-basic = {version = "0.1"} -ya-sb-router = "0.6" +ya-exe-unit.workspace = true +ya-framework-basic.workspace = true +ya-sb-router.workspace = true ya-service-bus = {workspace = true} actix = {version = "0.13", default-features = false} diff --git a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs index 8559a20c27..cebfc03cda 100644 --- a/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs +++ b/exe-unit/components/mock-runtime/src/testing/exe_unit_ext.rs @@ -181,7 +181,7 @@ impl ExeUnitExt for ExeUnitHandle { env: Default::default(), hosts: Default::default(), hostname: None, - volumes: vec![], + volumes: None, }], ) .await diff --git a/exe-unit/components/transfer/Cargo.toml b/exe-unit/components/transfer/Cargo.toml index ad4f965ef5..d7b30beba1 100644 --- a/exe-unit/components/transfer/Cargo.toml +++ b/exe-unit/components/transfer/Cargo.toml @@ -6,9 +6,8 @@ authors = ["Golem Factory "] edition = "2018" [dependencies] -ya-client = { workspace = true } -ya-client-model = { workspace = true } -ya-core-model = { workspace = true, features = ["activity", "gftp"] } +ya-client-model.workspace = true +ya-core-model.workspace = true ya-service-bus = { workspace = true } ya-utils-path = { version = "0.1", path = "../../../utils/path" } ya-utils-futures.workspace = true @@ -82,5 +81,5 @@ structopt = "0.3.15" test-context = "0.1.4" test-case = "3" -ya-framework-basic = { version = "0.1" } +ya-framework-basic.workspace = true ya-exe-unit = { version = "0.4", path = "../../../exe-unit" } diff --git a/exe-unit/src/exe_unit.rs b/exe-unit/src/exe_unit.rs index 5b8936ace3..ed44fc16b8 100644 --- a/exe-unit/src/exe_unit.rs +++ b/exe-unit/src/exe_unit.rs @@ -14,6 +14,7 @@ use ya_counters::message::GetCounters; use ya_counters::service::CountersService; use ya_agreement_utils::OfferTemplate; +use ya_client_model::activity::exe_script_command::VolumeMount; use ya_client_model::activity::{ActivityUsage, CommandOutput, ExeScriptCommand, State, StatePair}; use ya_core_model::activity; use ya_core_model::activity::local::Credentials; @@ -320,14 +321,26 @@ impl RuntimeRef { volumes, .. } => { - let volumes = volumes - .iter() - .enumerate() - .map(|(idx, vol)| ContainerVolume { - name: format!("vol-custom-{idx}"), - path: vol.to_string(), - }) - .collect::>(); + let volumes = if let Some(v) = &volumes { + v.clone() + .as_volumes() + .iter() + .enumerate() + .filter_map(|(idx, (path, info))| { + if matches!(info, VolumeMount::Host { .. }) { + Some(ContainerVolume { + name: format!("vol-custom-{idx}"), + path: path.clone(), + }) + } else { + None + } + }) + .collect::>() + } else { + Vec::new() + }; + transfer_service.send(AddVolumes::new(volumes)).await??; // TODO: We should pass `task_package` here not in `TransferService` initialization. diff --git a/exe-unit/src/manifest.rs b/exe-unit/src/manifest.rs index 9d085f948c..8c1df47382 100644 --- a/exe-unit/src/manifest.rs +++ b/exe-unit/src/manifest.rs @@ -504,7 +504,7 @@ mod tests { net: Default::default(), hosts: Default::default(), hostname: None, - volumes: vec![], + volumes: Default::default(), env: Default::default(), progress: None, }, diff --git a/exe-unit/src/runtime/process.rs b/exe-unit/src/runtime/process.rs index 7c0fd1c9f7..773f9cf502 100644 --- a/exe-unit/src/runtime/process.rs +++ b/exe-unit/src/runtime/process.rs @@ -205,7 +205,16 @@ impl RuntimeProcess { let (cmd, ctx) = cmd.split(); match cmd { - ExeScriptCommand::Deploy { .. } => rt_args.args(["deploy", "--"]), + ExeScriptCommand::Deploy { volumes, .. } => { + if let Some(volumes) = volumes { + let vol_override_json = serde_json::to_string(&volumes.as_volumes()) + .expect("failed to serialize volume info"); + + rt_args.arg("--volume-override"); + rt_args.arg(vol_override_json); + } + rt_args.args(["deploy", "--"]) + } ExeScriptCommand::Start { args } => rt_args.args(["start", "--"]).args(args), ExeScriptCommand::Run { entry_point, args, .. diff --git a/exe-unit/tests/test_exe_unit_basic.rs b/exe-unit/tests/test_exe_unit_basic.rs index 5c354e4b6c..18444b5c6c 100644 --- a/exe-unit/tests/test_exe_unit_basic.rs +++ b/exe-unit/tests/test_exe_unit_basic.rs @@ -9,7 +9,6 @@ use ya_framework_basic::test_dirs::cargo_binary; use ya_framework_basic::{resource, temp_dir}; use ya_mock_runtime::testing::{create_exe_unit, exe_unit_config, ExeUnitExt}; -#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_exe_unit_start_terminate(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { diff --git a/exe-unit/tests/test_progress.rs b/exe-unit/tests/test_progress.rs index c87bbe4394..3183f23091 100644 --- a/exe-unit/tests/test_progress.rs +++ b/exe-unit/tests/test_progress.rs @@ -2,7 +2,7 @@ use futures::StreamExt; use std::time::Duration; use test_context::test_context; -use ya_client_model::activity::exe_script_command::ProgressArgs; +use ya_client_model::activity::exe_script_command::{ProgressArgs, Volumes}; use ya_client_model::activity::{ExeScriptCommand, RuntimeEventKind, TransferArgs}; use ya_core_model::activity; use ya_framework_basic::async_drop::DroppableTestContext; @@ -17,7 +17,6 @@ use ya_service_bus::typed as bus; /// Test if progress reporting mechanisms work on gsb level /// with full ExeUnit setup. -#[cfg_attr(not(feature = "system-test"), ignore)] #[test_context(DroppableTestContext)] #[serial_test::serial] async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Result<()> { @@ -67,7 +66,7 @@ async fn test_progress_reporting(ctx: &mut DroppableTestContext) -> anyhow::Resu env: Default::default(), hosts: Default::default(), hostname: None, - volumes: vec!["/input".to_owned()], + volumes: Some(Volumes::Simple(vec!["/input".to_owned()])), }], ) .await diff --git a/golem_cli/Cargo.toml b/golem_cli/Cargo.toml index 2a5da6bb02..5259abcc5e 100644 --- a/golem_cli/Cargo.toml +++ b/golem_cli/Cargo.toml @@ -7,11 +7,11 @@ edition = "2018" [dependencies] ya-client = { workspace = true, features = ['cli'] } -ya-compile-time-utils = "0.2" +ya-compile-time-utils.workspace = true ya-core-model = { workspace = true, features = ["payment", "version"] } -ya-provider = "0.3" -ya-utils-path = "0.1.0" -ya-utils-process = { version = "0.3", features = ["lock"] } +ya-provider.path = "../agent/provider" +ya-utils-path.workspace = true +ya-utils-process = { workspace = true, features = ["lock"] } actix-rt = "2.7" ansi_term = "0.12.1" diff --git a/test-utils/test-framework/Cargo.toml b/test-utils/test-framework/Cargo.toml index 74f4b2a24f..c743a8bef4 100644 --- a/test-utils/test-framework/Cargo.toml +++ b/test-utils/test-framework/Cargo.toml @@ -17,8 +17,8 @@ serial_test = { git = "https://github.com/golemfactory/serial_test.git", branch tokio = { version = "1", features = ["rt"] } url = "2.3" -ya-utils-process = "0.3" -ya-framework-macro = "0.1" +ya-utils-process.workspace = true +ya-framework-macro = { path = "./framework-macro" } [target.'cfg(target_family = "unix")'.dependencies] libc = "0.2.73" diff --git a/test-utils/test-framework/framework-basic/Cargo.toml b/test-utils/test-framework/framework-basic/Cargo.toml index 1c884aef97..67e3440bda 100644 --- a/test-utils/test-framework/framework-basic/Cargo.toml +++ b/test-utils/test-framework/framework-basic/Cargo.toml @@ -11,13 +11,13 @@ bcast-singleton = [] [dependencies] ya-client-model.workspace = true ya-client.workspace = true -ya-core-model = { workspace = true, features = ["net"] } +ya-core-model.workspace = true ya-service-bus = { workspace = true } ya-utils-path = { version = "0.1", path = "../../../utils/path" } ya-utils-futures.workspace = true gftp = { workspace = true } -ya-net = "0.3" +ya-net.workspace = true actix = "0.13" actix-files = "0.6" diff --git a/test-utils/test-framework/framework-mocks/Cargo.toml b/test-utils/test-framework/framework-mocks/Cargo.toml index a9a33e1491..963783a18e 100644 --- a/test-utils/test-framework/framework-mocks/Cargo.toml +++ b/test-utils/test-framework/framework-mocks/Cargo.toml @@ -6,27 +6,27 @@ edition = "2021" [dependencies] ya-agreement-utils = { workspace = true } -ya-client-model = { workspace = true } -ya-client = { workspace = true } -ya-core-model = { workspace = true } -ya-dummy-driver = "0.3" -ya-erc20-driver = "0.4" -ya-identity = "0.3" -ya-market = "0.4" -ya-net = { version = "0.3", features = ["service"] } -ya-payment = "0.3" -ya-persistence = "0.3" -ya-service-api = "0.1" -ya-service-api-web = "0.2" +ya-client-model.workspace = true +ya-client.workspace = true +ya-core-model.workspace = true +ya-dummy-driver.workspace = true +ya-erc20-driver.workspace = true +ya-identity.workspace = true +ya-market.workspace = true +ya-net = { workspace = true, features = ["service"] } +ya-payment.workspace = true +ya-persistence.workspace = true +ya-service-api.workspace = true +ya-service-api-web.workspace = true ya-service-bus = { workspace = true } ya-sb-router = { workspace = true } -ya-utils-path = { version = "0.1", path = "../../../utils/path" } -ya-std-utils = "0.1" +ya-utils-path.workspace = true +ya-std-utils.workspace = true -ya-framework-basic = { version = "0.1" } +ya-framework-basic.workspace = true -actix-rt = "2.7" -actix-web = "4" +actix-rt.workspace = true +actix-web.workspace = true actix-http = "3" actix-service = "2" anyhow = "1.0" diff --git a/utils/manifest-utils/Cargo.toml b/utils/manifest-utils/Cargo.toml index 4bb6e1707f..86db7ef3c3 100644 --- a/utils/manifest-utils/Cargo.toml +++ b/utils/manifest-utils/Cargo.toml @@ -21,7 +21,7 @@ schema = ["schemars"] [dependencies] ya-agreement-utils = { workspace = true } -ya-utils-path = "0.1" +ya-utils-path.workspace = true ya-client-model.workspace = true golem-certificate = "0.1.1" @@ -57,8 +57,8 @@ serial_test = "2" shlex = "1.1" tar = "0.4" test-case = "3.1" -ya-manifest-test-utils = "0.1" +ya-manifest-test-utils.workspace = true openssl.workspace = true [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/utils/manifest-utils/test-utils/Cargo.toml b/utils/manifest-utils/test-utils/Cargo.toml index 27aac3c1ba..449d74c03c 100644 --- a/utils/manifest-utils/test-utils/Cargo.toml +++ b/utils/manifest-utils/test-utils/Cargo.toml @@ -13,5 +13,5 @@ base64 = "0.13" tar = "0.4" openssl = "0.10" -ya-manifest-utils = "0.2" +ya-manifest-utils.workspace = true ya-agreement-utils = { workspace = true } From b78a4f8e088a25b9735ca9fd34a019402caf8f69 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 20 Sep 2024 00:20:54 +0200 Subject: [PATCH 119/125] ya-client points to master now --- Cargo.lock | 4 ++-- Cargo.toml | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe19332aba..d014ad0897 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8866,7 +8866,7 @@ dependencies = [ [[package]] name = "ya-client" version = "0.9.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=60327ddaaea289ecf45836c1e242e15855d71dc5#60327ddaaea289ecf45836c1e242e15855d71dc5" +source = "git+https://github.com/golemfactory/ya-client.git?rev=653e7ed3ff8836837b660a76e604055e167b1f2e#653e7ed3ff8836837b660a76e604055e167b1f2e" dependencies = [ "actix-codec", "awc", @@ -8890,7 +8890,7 @@ dependencies = [ [[package]] name = "ya-client-model" version = "0.6.0" -source = "git+https://github.com/golemfactory/ya-client.git?rev=60327ddaaea289ecf45836c1e242e15855d71dc5#60327ddaaea289ecf45836c1e242e15855d71dc5" +source = "git+https://github.com/golemfactory/ya-client.git?rev=653e7ed3ff8836837b660a76e604055e167b1f2e#653e7ed3ff8836837b660a76e604055e167b1f2e" dependencies = [ "bigdecimal 0.2.2", "bytesize", diff --git a/Cargo.toml b/Cargo.toml index 4bbb252694..69c4bd1e31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -279,8 +279,8 @@ ya-sb-proto = { version = "0.6.2" } ya-sb-util = { version = "0.5.1" } parking_lot = "0.12.3" mime = "0.3.17" -ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "60327ddaaea289ecf45836c1e242e15855d71dc5" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "60327ddaaea289ecf45836c1e242e15855d71dc5" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "653e7ed3ff8836837b660a76e604055e167b1f2e" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "653e7ed3ff8836837b660a76e604055e167b1f2e" } ya-compile-time-utils.path = "utils/compile-time-utils" ya-manifest-utils = { path = "utils/manifest-utils" } @@ -316,9 +316,8 @@ ya-framework-basic.path = "test-utils/test-framework/framework-basic" ya-framework-mocks.path = "test-utils/test-framework/framework-mocks" [patch.crates-io] -## CLIENT -#ya-client = { git = "https://github.com/golemfactory/ya-client.git", branch = "kek/deploy-volumes" } -ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "60327ddaaea289ecf45836c1e242e15855d71dc5" } +ya-client = { git = "https://github.com/golemfactory/ya-client.git", rev = "653e7ed3ff8836837b660a76e604055e167b1f2e" } +ya-client-model = { git = "https://github.com/golemfactory/ya-client.git", rev = "653e7ed3ff8836837b660a76e604055e167b1f2e" } golem-certificate = { git = "https://github.com/golemfactory/golem-certificate.git", rev = "f2d7514c18fc066e9cfb796090b90f5b27cfe1c6" } ethereum-tx-sign = { git = "https://github.com/golemfactory/ethereum-tx-sign.git", rev = "1164c74187a9e2947faeaea7dde104c3cdec4195" } graphene-sgx = { git = " https://github.com/golemfactory/graphene-rust.git", rev = "dbd993ebad7f9190410ea390a589348479af6407" } From 12c65934300348a0f8c793e56f160904628ee0b2 Mon Sep 17 00:00:00 2001 From: scx1332 Date: Fri, 20 Sep 2024 14:09:17 +0200 Subject: [PATCH 120/125] Placeholder for serving future dashboard. Can be enabled by adding feature dashboard to build. --- .github/workflows/fast-win-build.yml | 3 +- Cargo.lock | 36 ++++++++++++++++ Cargo.toml | 3 ++ core/serv-api/web/src/middleware/auth/mod.rs | 13 +++--- core/serv/src/main.rs | 44 +++++++++++++++++++- dashboard/index.html | 11 +++++ 6 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 dashboard/index.html diff --git a/.github/workflows/fast-win-build.yml b/.github/workflows/fast-win-build.yml index cc0d729897..38e3db39ea 100644 --- a/.github/workflows/fast-win-build.yml +++ b/.github/workflows/fast-win-build.yml @@ -51,7 +51,6 @@ jobs: - name: Checkout frontend run: | - mkdir -p dashboard git clone https://github.com/scx1332/yagna-dashboard.git yagna-dashboard cd yagna-dashboard git checkout e52bb7b51d7a644acc407479332f1f4b4cda263e @@ -67,7 +66,7 @@ jobs: - name: Build run: | - cargo build --target x86_64-pc-windows-gnu --profile release-fast --features static-openssl + cargo build --target x86_64-pc-windows-gnu --profile release-fast --features dashboard,static-openssl cp target/x86_64-pc-windows-gnu/release-fast/yagna.exe yagna.exe tar -czf yagna.tar.gz yagna.exe diff --git a/Cargo.lock b/Cargo.lock index d014ad0897..12f588b7c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5985,6 +5985,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-embed" +version = "8.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa66af4a4fdd5e7ebc276f115e895611a34739a9c1c01028383d612d550953c0" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6125dbc8867951125eec87294137f4e9c2c96566e61bf72c45095a7c77761478" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.77", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5347777e9aacb56039b0e1f28785929a8a3b709e87482e7442c72e7c12529d" +dependencies = [ + "sha2 0.10.8", + "walkdir", +] + [[package]] name = "rust_decimal" version = "1.36.0" @@ -10403,9 +10437,11 @@ dependencies = [ "libsqlite3-sys", "log", "metrics 0.12.1", + "mime_guess", "num_cpus", "openssl", "openssl-probe", + "rust-embed", "serde", "serde_json", "structopt", diff --git a/Cargo.toml b/Cargo.toml index 69c4bd1e31..000b02d45a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ version = "0.16.0" [features] default = ['erc20-driver', 'gftp/bin'] +dashboard = ['rust-embed', 'mime_guess'] dummy-driver = ['ya-dummy-driver'] erc20-driver = ['ya-erc20-driver'] static-openssl = ["openssl/vendored", "openssl-probe"] @@ -82,9 +83,11 @@ lazy_static = "1.4" libsqlite3-sys = {workspace = true} log = "0.4" metrics = "0.12" +mime_guess = { version = "2.0", optional = true } num_cpus = "1" openssl-probe = {version = "0.1", optional = true} openssl.workspace = true +rust-embed = { version = "8.5", optional = true } serde = "1.0" serde_json = "1.0" structopt = "0.3" diff --git a/core/serv-api/web/src/middleware/auth/mod.rs b/core/serv-api/web/src/middleware/auth/mod.rs index 56533ae887..3542186f87 100644 --- a/core/serv-api/web/src/middleware/auth/mod.rs +++ b/core/serv-api/web/src/middleware/auth/mod.rs @@ -84,12 +84,13 @@ where let cache = self.cache.clone(); let service = self.service.clone(); - // TODO: remove this hack; possibly by enabling creation of arbitrary appkey from CLI - if req.uri().to_string().starts_with("/metrics-api") - || req.uri().to_string().starts_with("/version") - { - log::debug!("skipping authorization for uri={}", req.uri()); - return Box::pin(service.borrow_mut().call(req)); + let allowed_uris = vec!["/metrics-api", "/version", "/dashboard"]; + + for uri in allowed_uris { + if req.uri().to_string().starts_with(uri) { + log::debug!("skipping authorization for uri={}", req.uri()); + return Box::pin(service.borrow_mut().call(req)); + } } Box::pin(async move { diff --git a/core/serv/src/main.rs b/core/serv/src/main.rs index 9960bed5c5..05d842ee20 100644 --- a/core/serv/src/main.rs +++ b/core/serv/src/main.rs @@ -1,6 +1,6 @@ #![allow(clippy::obfuscated_if_else)] -use actix_web::{middleware, web, App, HttpServer, Responder}; +use actix_web::{middleware, web, App, HttpResponse, HttpServer, Responder}; use anyhow::{Context, Result}; use futures::prelude::*; use metrics::{counter, gauge}; @@ -568,6 +568,8 @@ impl ServiceCommand { .wrap(middleware::Logger::default()) .wrap(auth::Auth::new(cors.cache())) .wrap(cors.cors()) + .route("/dashboard", web::get().to(redirect_to_dashboard)) + .route("/dashboard/{_:.*}", web::get().to(dashboard_serve)) .route("/me", web::get().to(me)) .service(forward_gsb); let rest = Services::rest(app, &context); @@ -710,6 +712,46 @@ async fn forward_gsb( Ok::<_, actix_web::Error>(web::Json(json_resp)) } +#[cfg(feature = "dashboard")] +#[derive(rust_embed::RustEmbed)] +#[folder = "dashboard"] +struct Asset; + +pub async fn redirect_to_dashboard() -> impl Responder { + #[cfg(feature = "dashboard")] + { + let target = "/dashboard/"; + log::debug!("Redirecting to endpoint: {target}"); + HttpResponse::Ok() + .status(actix_web::http::StatusCode::PERMANENT_REDIRECT) + .append_header((actix_web::http::header::LOCATION, target)) + .finish() + } + #[cfg(not(feature = "dashboard"))] + HttpResponse::NotFound().body("404 Not Found") +} + +pub async fn dashboard_serve(path: web::Path) -> impl Responder { + #[cfg(feature = "dashboard")] + { + let mut path = path.as_str(); + let mut content = Asset::get(path); + if content.is_none() && !path.contains('.') { + path = "index.html"; + content = Asset::get(path); + } + log::debug!("Serving frontend file: {path}"); + match content { + Some(content) => HttpResponse::Ok() + .content_type(mime_guess::from_path(path).first_or_octet_stream().as_ref()) + .body(content.data.into_owned()), + None => HttpResponse::NotFound().body("404 Not Found"), + } + } + #[cfg(not(feature = "dashboard"))] + HttpResponse::NotFound().body(format!("404 Not Found: {}", path)) +} + #[actix_rt::main] async fn main() -> Result<()> { dotenv::dotenv().ok(); diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 0000000000..5bdc8c3c01 --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,11 @@ + + + + + Yagna Dashboard + + +

Yagna Dashboard

+

This is placeholder, place your files into ./dashboard directory to serve them from yagna

+ + From d5d42e7fdc08e771b818f9af4ca9d3d86a1bc95b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Krzysztof=20Rekucki?= Date: Mon, 23 Sep 2024 09:42:44 +0200 Subject: [PATCH 121/125] release {{crate_name}} {{version}} --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12f588b7c8..b68d33147d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10421,7 +10421,7 @@ dependencies = [ [[package]] name = "yagna" -version = "0.16.0" +version = "0.17.0" dependencies = [ "actix-rt", "actix-service", diff --git a/Cargo.toml b/Cargo.toml index 000b02d45a..9265a79f4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "GPL-3.0" name = "yagna" readme = "README.md" repository = "https://github.com/golemfactory/yagna" -version = "0.16.0" +version = "0.17.0" [features] default = ['erc20-driver', 'gftp/bin'] From fc12b94b4919c9e13e24f35cde5761cac3491375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Krzysztof=20Rekucki?= Date: Mon, 23 Sep 2024 09:57:14 +0200 Subject: [PATCH 122/125] fix buid deb --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad1c22d796..0acf2d63f4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -372,7 +372,7 @@ jobs: console.log(release.data.upload_url); return release.data.upload_url - - uses: golemfactory/build-deb-action@5 + - uses: golemfactory/build-deb-action@v7 id: deb with: debVersion: ${{ steps.version.outputs.version-ext }} From 298ed6e9d98b64e574a13f3448d343cf7c39795c Mon Sep 17 00:00:00 2001 From: Kamil Koczurek Date: Tue, 24 Sep 2024 11:33:27 +0200 Subject: [PATCH 123/125] exe-unit: fix counters --- exe-unit/src/lib.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/exe-unit/src/lib.rs b/exe-unit/src/lib.rs index d76bac574a..cc4ee1520f 100644 --- a/exe-unit/src/lib.rs +++ b/exe-unit/src/lib.rs @@ -6,7 +6,6 @@ use anyhow::{bail, Context}; use std::convert::TryFrom; use std::path::PathBuf; use structopt::clap; -use ya_counters::service::CountersServiceBuilder; use ya_client_model::activity::ExeScriptCommand; use ya_core_model::activity; @@ -369,12 +368,10 @@ pub async fn exe_unit(mut config: ExeUnitConfig) -> anyhow::Result Date: Tue, 24 Sep 2024 15:58:17 +0200 Subject: [PATCH 124/125] Rename MATIC token name to POL --- core/payment-driver/erc20/config-payments.toml | 6 +++--- core/payment-driver/erc20/src/lib.rs | 10 +++++----- extra/payments/multi_test/config-payments.toml | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/payment-driver/erc20/config-payments.toml b/core/payment-driver/erc20/config-payments.toml index 4a30be038f..b16cc47d2d 100644 --- a/core/payment-driver/erc20/config-payments.toml +++ b/core/payment-driver/erc20/config-payments.toml @@ -112,7 +112,7 @@ dns-source = "holesky.rpc-node.dev.golem.network." [chain.mumbai] chain-name = "Mumbai testnet" chain-id = 80001 -currency-symbol = "tMATIC" +currency-symbol = "POL" priority-fee = 1.0 max-fee-per-gas = 14.0 transaction-timeout = 60 @@ -144,7 +144,7 @@ dns-source = "mumbai.rpc-node.dev.golem.network." [chain.polygon] chain-name = "Polygon mainnet" chain-id = 137 -currency-symbol = "MATIC" +currency-symbol = "POL" priority-fee = 30.111 max-fee-per-gas = 500.0 transaction-timeout = 100 @@ -165,7 +165,7 @@ dns-source = "polygon.rpc-node.dev.golem.network." [chain.amoy] chain-name = "Amoy testnet" chain-id = 80002 -currency-symbol = "tMATIC" +currency-symbol = "POL" priority-fee = 30.111 max-fee-per-gas = 500.0 transaction-timeout = 100 diff --git a/core/payment-driver/erc20/src/lib.rs b/core/payment-driver/erc20/src/lib.rs index 758c1c2109..06f86dc99c 100644 --- a/core/payment-driver/erc20/src/lib.rs +++ b/core/payment-driver/erc20/src/lib.rs @@ -28,14 +28,14 @@ pub const HOLESKY_CURRENCY_LONG: &str = "Holesky Ether"; pub const MUMBAI_NETWORK: &str = "mumbai"; pub const MUMBAI_TOKEN: &str = "tGLM"; pub const MUMBAI_PLATFORM: &str = "erc20-mumbai-tglm"; -pub const MUMBAI_CURRENCY_SHORT: &str = "tMATIC"; -pub const MUMBAI_CURRENCY_LONG: &str = "Test MATIC"; +pub const MUMBAI_CURRENCY_SHORT: &str = "POL"; +pub const MUMBAI_CURRENCY_LONG: &str = "Test POL"; pub const AMOY_NETWORK: &str = "amoy"; pub const AMOY_TOKEN: &str = "tGLM"; pub const AMOY_PLATFORM: &str = "erc20-amoy-tglm"; -pub const AMOY_CURRENCY_SHORT: &str = "tMATIC"; -pub const AMOY_CURRENCY_LONG: &str = "Test MATIC"; +pub const AMOY_CURRENCY_SHORT: &str = "POL"; +pub const AMOY_CURRENCY_LONG: &str = "Test POL"; pub const SEPOLIA_NETWORK: &str = "sepolia"; pub const SEPOLIA_TOKEN: &str = "tGLM"; @@ -52,7 +52,7 @@ pub const MAINNET_CURRENCY_LONG: &str = "Ether"; pub const POLYGON_MAINNET_NETWORK: &str = "polygon"; pub const POLYGON_MAINNET_TOKEN: &str = "GLM"; pub const POLYGON_MAINNET_PLATFORM: &str = "erc20-polygon-glm"; -pub const POLYGON_MAINNET_CURRENCY_SHORT: &str = "MATIC"; +pub const POLYGON_MAINNET_CURRENCY_SHORT: &str = "POL"; pub const POLYGON_MAINNET_CURRENCY_LONG: &str = "Polygon"; pub use service::Erc20Service as PaymentDriverService; diff --git a/extra/payments/multi_test/config-payments.toml b/extra/payments/multi_test/config-payments.toml index 6982e940de..f1eb90591c 100644 --- a/extra/payments/multi_test/config-payments.toml +++ b/extra/payments/multi_test/config-payments.toml @@ -135,7 +135,7 @@ dns-source = "holesky.rpc-node.dev.golem.network." [chain.mumbai] chain-name = "Mumbai testnet" chain-id = 80001 -currency-symbol = "tMATIC" +currency-symbol = "POL" priority-fee = 1.0 max-fee-per-gas = 14.0 transaction-timeout = 60 @@ -167,7 +167,7 @@ dns-source = "mumbai.rpc-node.dev.golem.network." [chain.polygon] chain-name = "Polygon mainnet" chain-id = 137 -currency-symbol = "MATIC" +currency-symbol = "POL" priority-fee = 30.111 max-fee-per-gas = 500.0 transaction-timeout = 100 From 1418de5788ec2c1615d21fa7ac51ff2f76cbd9dc Mon Sep 17 00:00:00 2001 From: scx1332 Date: Tue, 1 Oct 2024 15:59:43 +0200 Subject: [PATCH 125/125] Add options for ignoring legacy network when listing drivers (#3337) --- core/model/src/payment.rs | 4 +- core/payment/src/cli.rs | 72 +++++++++++++++++++++-------------- core/payment/src/processor.rs | 21 ++++++++-- core/payment/src/service.rs | 9 ++++- 4 files changed, 71 insertions(+), 35 deletions(-) diff --git a/core/model/src/payment.rs b/core/model/src/payment.rs index 45c3e0ec7f..be4f2a21b8 100644 --- a/core/model/src/payment.rs +++ b/core/model/src/payment.rs @@ -469,7 +469,9 @@ pub mod local { } #[derive(Clone, Debug, Serialize, Deserialize)] - pub struct GetDrivers {} + pub struct GetDrivers { + pub ignore_legacy_networks: bool, + } #[derive(Clone, Debug, Serialize, Deserialize, thiserror::Error)] pub enum GetDriversError { diff --git a/core/payment/src/cli.rs b/core/payment/src/cli.rs index cae16ce052..70a0984bd5 100644 --- a/core/payment/src/cli.rs +++ b/core/payment/src/cli.rs @@ -1,13 +1,14 @@ mod rpc; +use std::collections::HashMap; // External crates use bigdecimal::BigDecimal; use chrono::{DateTime, Utc}; -use serde_json::to_value; +use serde_json::{to_value, Value}; use std::str::FromStr; use std::time::UNIX_EPOCH; use structopt::*; -use ya_client_model::payment::DriverStatusProperty; +use ya_client_model::payment::{DriverDetails, DriverStatusProperty}; use ya_core_model::payment::local::NetworkName; // Workspace uses @@ -553,41 +554,56 @@ Typically operation should take less than 1 minute. ))) } DriverSubcommand::List => { - let drivers = bus::service(pay::BUS_ID).call(pay::GetDrivers {}).await??; + let drivers: HashMap = bus::service(pay::BUS_ID) + .call(pay::GetDrivers { + ignore_legacy_networks: false, + }) + .await??; if ctx.json_output { return CommandOutput::object(drivers); } - Ok(ResponseTable { - columns: vec![ - "driver".to_owned(), - "network".to_owned(), - "default?".to_owned(), - "token".to_owned(), - "platform".to_owned(), - ], - values: drivers - .iter() - .flat_map(|(driver, dd)| { - dd.networks - .iter() - .flat_map(|(network, n)| { - n.tokens - .iter() - .map(|(token, platform)| - serde_json::json! {[ + + let mut values: Vec = drivers + .iter() + .flat_map(|(driver, dd)| { + dd.networks + .iter() + .flat_map(|(network, n)| { + n.tokens + .iter() + .map(|(token, platform)| + serde_json::json! {[ driver, network, if &dd.default_network == network { "X" } else { "" }, token, platform, ]} - ) - .collect::>() - }) - .collect::>() - }) - .collect(), - }.into()) + ) + .collect::>() + }) + .collect::>() + }) + .collect(); + + values.sort_by(|a, b| { + //sort by index 4 (which means platform, be careful when changing these values) + let left_str = a.as_array().unwrap()[4].as_str().unwrap(); + let right_str = b.as_array().unwrap()[4].as_str().unwrap(); + left_str.cmp(right_str) + }); + + Ok(ResponseTable { + columns: vec![ + "driver".to_owned(), //index 0 + "network".to_owned(), //index 1 + "default?".to_owned(), //index 2 + "token".to_owned(), //index 3 + "platform".to_owned(), //index 4 - we are sorting by platform, be careful when changing these values + ], + values, + } + .into()) } }, PaymentCli::ReleaseAllocations => { diff --git a/core/payment/src/processor.rs b/core/payment/src/processor.rs index 17b74ccca9..275734b9f8 100644 --- a/core/payment/src/processor.rs +++ b/core/payment/src/processor.rs @@ -230,8 +230,18 @@ impl DriverRegistry { } } - pub fn get_drivers(&self) -> HashMap { - self.drivers.clone() + pub fn get_drivers(&self, ignore_legacy_networks: bool) -> HashMap { + let mut drivers = self.drivers.clone(); + + let legacy_networks = ["mumbai", "goerli", "rinkeby"]; + if ignore_legacy_networks { + drivers.values_mut().for_each(|driver| { + driver + .networks + .retain(|name, _| !legacy_networks.contains(&name.as_str())) + }) + } + drivers } pub fn get_network( @@ -381,11 +391,14 @@ impl PaymentProcessor { .map_err(|_| GetAccountsError::InternalTimeout) } - pub async fn get_drivers(&self) -> Result, GetDriversError> { + pub async fn get_drivers( + &self, + ignore_legacy_networks: bool, + ) -> Result, GetDriversError> { self.registry .timeout_read(REGISTRY_LOCK_TIMEOUT) .await - .map(|registry| registry.get_drivers()) + .map(|registry| registry.get_drivers(ignore_legacy_networks)) .map_err(|_| GetDriversError::InternalTimeout) } diff --git a/core/payment/src/service.rs b/core/payment/src/service.rs index aa7d3c7269..ebf5428b7c 100644 --- a/core/payment/src/service.rs +++ b/core/payment/src/service.rs @@ -492,7 +492,7 @@ mod local { _caller: String, msg: GetDrivers, ) -> Result, GetDriversError> { - processor.get_drivers().await + processor.get_drivers(msg.ignore_legacy_networks).await } async fn payment_driver_status( @@ -506,7 +506,12 @@ mod local { None => { #[allow(clippy::iter_kv_map)] // Unwrap is provably safe because NoError can't be instanciated - match service(PAYMENT_BUS_ID).call(GetDrivers {}).await { + match service(PAYMENT_BUS_ID) + .call(GetDrivers { + ignore_legacy_networks: false, + }) + .await + { Ok(drivers) => drivers, Err(e) => return Err(PaymentDriverStatusError::Internal(e.to_string())), }