From 64eb84800a2b7d38433b2d0e260ed2530ff60485 Mon Sep 17 00:00:00 2001 From: Daniel Knopik <107140945+dknopik@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:56:29 +0100 Subject: [PATCH] Anchor pre-PR: Modularize validator store (#6771) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * pass slots_per_epoch at runtime * remove generic E from unrequired types * move `validator_store` to `lighthouse_validator_store` * make validator_store into a trait * further reduce dependencies * remove `environment` dependency on `beacon_node_fallback` * Manually pull in some changes from tracing-integration (thanks sayan!) Co-authored-by: ThreeHrSleep * remove `environment` from `validator_services` * unify boost factor accessors * add builder for DutiesService * Manually merge tracing PR for beacon_node_fallback Co-authored-by: ThreeHrSleep * Fix chain_spec for BlockService * address review * remove PhantomData from SyncDutiesMap * fix tests * correct test * Add `E` to `ValidatorStore` as associated type * fix tests * derive Clone for ValidatorStore's Error and required sub-errors * switch to enum for block signing to allow differing types --------- Co-authored-by: João Oliveira Co-authored-by: ThreeHrSleep Co-authored-by: Jimmy Chen --- Cargo.lock | 45 +- Cargo.toml | 3 +- common/logging/src/lib.rs | 1 + common/logging/src/macros.rs | 6 + consensus/types/src/attestation.rs | 2 +- consensus/types/src/payload.rs | 13 +- testing/web3signer_tests/Cargo.toml | 1 + testing/web3signer_tests/src/lib.rs | 15 +- validator_client/Cargo.toml | 1 + .../beacon_node_fallback/Cargo.toml | 4 +- .../src/beacon_node_health.rs | 8 +- .../beacon_node_fallback/src/lib.rs | 157 +-- .../doppelganger_service/Cargo.toml | 1 + .../doppelganger_service/src/lib.rs | 81 +- validator_client/graffiti_file/src/lib.rs | 12 +- validator_client/http_api/Cargo.toml | 31 +- .../src/create_signed_voluntary_exit.rs | 3 +- .../http_api/src/create_validator.rs | 9 +- validator_client/http_api/src/graffiti.rs | 8 +- validator_client/http_api/src/keystores.rs | 14 +- validator_client/http_api/src/lib.rs | 108 +- validator_client/http_api/src/remotekeys.rs | 18 +- validator_client/http_api/src/test_utils.rs | 13 +- validator_client/http_api/src/tests.rs | 17 +- .../http_api/src/tests/keystores.rs | 3 +- validator_client/http_metrics/Cargo.toml | 2 +- validator_client/http_metrics/src/lib.rs | 14 +- .../lighthouse_validator_store/Cargo.toml | 20 + .../lighthouse_validator_store/src/lib.rs | 1109 ++++++++++++++++ validator_client/signing_method/src/lib.rs | 17 +- .../slashing_protection/src/lib.rs | 2 +- .../src/signed_attestation.rs | 2 +- .../slashing_protection/src/signed_block.rs | 2 +- validator_client/src/config.rs | 2 +- validator_client/src/latency.rs | 2 +- validator_client/src/lib.rs | 88 +- validator_client/src/notifier.rs | 5 +- .../validator_services/Cargo.toml | 8 +- .../src/attestation_service.rs | 263 ++-- .../validator_services/src/block_service.rs | 266 ++-- .../validator_services/src/duties_service.rs | 416 +++--- .../src/preparation_service.rs | 141 +- .../validator_services/src/sync.rs | 172 ++- .../src/sync_committee_service.rs | 231 ++-- validator_client/validator_store/Cargo.toml | 14 - validator_client/validator_store/src/lib.rs | 1130 +++-------------- 46 files changed, 2295 insertions(+), 2185 deletions(-) create mode 100644 common/logging/src/macros.rs create mode 100644 validator_client/lighthouse_validator_store/Cargo.toml create mode 100644 validator_client/lighthouse_validator_store/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 42e9df4e796..ef81e062fc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -868,15 +868,15 @@ dependencies = [ name = "beacon_node_fallback" version = "0.1.0" dependencies = [ - "environment", "eth2", "futures", "itertools 0.10.5", "serde", - "slog", "slot_clock", "strum", + "task_executor", "tokio", + "tracing", "types", "validator_metrics", ] @@ -2250,6 +2250,7 @@ dependencies = [ "task_executor", "tokio", "types", + "validator_store", ] [[package]] @@ -5357,6 +5358,25 @@ dependencies = [ "void", ] +[[package]] +name = "lighthouse_validator_store" +version = "0.1.0" +dependencies = [ + "account_utils", + "doppelganger_service", + "initialized_validators", + "parking_lot 0.12.3", + "serde", + "signing_method", + "slashing_protection", + "slog", + "slot_clock", + "task_executor", + "types", + "validator_metrics", + "validator_store", +] + [[package]] name = "lighthouse_version" version = "0.1.0" @@ -9537,6 +9557,7 @@ dependencies = [ "graffiti_file", "hyper 1.5.1", "initialized_validators", + "lighthouse_validator_store", "metrics", "monitoring_api", "parking_lot 0.12.3", @@ -9592,6 +9613,7 @@ dependencies = [ "graffiti_file", "initialized_validators", "itertools 0.10.5", + "lighthouse_validator_store", "lighthouse_version", "logging", "parking_lot 0.12.3", @@ -9622,6 +9644,7 @@ dependencies = [ name = "validator_http_metrics" version = "0.1.0" dependencies = [ + "lighthouse_validator_store", "lighthouse_version", "malloc_utils", "metrics", @@ -9632,7 +9655,6 @@ dependencies = [ "types", "validator_metrics", "validator_services", - "validator_store", "warp", "warp_utils", ] @@ -9675,16 +9697,16 @@ version = "0.1.0" dependencies = [ "beacon_node_fallback", "bls", - "doppelganger_service", - "environment", "eth2", "futures", "graffiti_file", + "logging", "parking_lot 0.12.3", "safe_arith", - "slog", "slot_clock", + "task_executor", "tokio", + "tracing", "tree_hash", "types", "validator_metrics", @@ -9695,18 +9717,8 @@ dependencies = [ name = "validator_store" version = "0.1.0" dependencies = [ - "account_utils", - "doppelganger_service", - "initialized_validators", - "parking_lot 0.12.3", - "serde", - "signing_method", "slashing_protection", - "slog", - "slot_clock", - "task_executor", "types", - "validator_metrics", ] [[package]] @@ -9987,6 +9999,7 @@ dependencies = [ "eth2_network_config", "futures", "initialized_validators", + "lighthouse_validator_store", "logging", "parking_lot 0.12.3", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 23e52a306b6..0fcd13009eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,11 +89,11 @@ members = [ "validator_client/http_api", "validator_client/http_metrics", "validator_client/initialized_validators", + "validator_client/lighthouse_validator_store", "validator_client/signing_method", "validator_client/slashing_protection", "validator_client/validator_metrics", "validator_client/validator_services", - "validator_client/validator_store", "validator_manager", @@ -253,6 +253,7 @@ int_to_bytes = { path = "consensus/int_to_bytes" } kzg = { path = "crypto/kzg" } metrics = { path = "common/metrics" } lighthouse_network = { path = "beacon_node/lighthouse_network" } +lighthouse_validator_store = { path = "validator_client/lighthouse_validator_store" } lighthouse_version = { path = "common/lighthouse_version" } lockfile = { path = "common/lockfile" } logging = { path = "common/logging" } diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index 7fe7f79506c..a1a81ba889d 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -13,6 +13,7 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; pub const MAX_MESSAGE_WIDTH: usize = 40; pub mod async_record; +pub mod macros; mod sse_logging_components; mod tracing_logging_layer; mod tracing_metrics_layer; diff --git a/common/logging/src/macros.rs b/common/logging/src/macros.rs new file mode 100644 index 00000000000..eb25eba56ce --- /dev/null +++ b/common/logging/src/macros.rs @@ -0,0 +1,6 @@ +#[macro_export] +macro_rules! crit { + ($($arg:tt)*) => { + tracing::error!(error_type = "crit", $($arg)*); + }; +} diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 190964736fe..052b196719a 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -16,7 +16,7 @@ use super::{ Signature, SignedRoot, }; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub enum Error { SszTypesError(ssz_types::Error), AlreadySigned(usize), diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index e68801840af..c832df059f7 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -84,6 +84,7 @@ pub trait AbstractExecPayload: + TryInto + TryInto + TryInto + + Sync { type Ref<'a>: ExecPayload + Copy @@ -95,19 +96,23 @@ pub trait AbstractExecPayload: type Bellatrix: OwnedExecPayload + Into + for<'a> From>> - + TryFrom>; + + TryFrom> + + Sync; type Capella: OwnedExecPayload + Into + for<'a> From>> - + TryFrom>; + + TryFrom> + + Sync; type Deneb: OwnedExecPayload + Into + for<'a> From>> - + TryFrom>; + + TryFrom> + + Sync; type Electra: OwnedExecPayload + Into + for<'a> From>> - + TryFrom>; + + TryFrom> + + Sync; } #[superstruct( diff --git a/testing/web3signer_tests/Cargo.toml b/testing/web3signer_tests/Cargo.toml index 376aa13406e..f68fa56e16d 100644 --- a/testing/web3signer_tests/Cargo.toml +++ b/testing/web3signer_tests/Cargo.toml @@ -14,6 +14,7 @@ eth2_keystore = { workspace = true } eth2_network_config = { workspace = true } futures = { workspace = true } initialized_validators = { workspace = true } +lighthouse_validator_store = { workspace = true } logging = { workspace = true } parking_lot = { workspace = true } reqwest = { workspace = true } diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index e0dee9ceb4b..87c809f8c89 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -25,6 +25,7 @@ mod tests { use initialized_validators::{ load_pem_certificate, load_pkcs12_identity, InitializedValidators, }; + use lighthouse_validator_store::LighthouseValidatorStore; use logging::test_logger; use parking_lot::Mutex; use reqwest::Client; @@ -45,7 +46,7 @@ mod tests { use tokio::time::sleep; use types::{attestation::AttestationBase, *}; use url::Url; - use validator_store::{Error as ValidatorStoreError, ValidatorStore}; + use validator_store::{Error as ValidatorStoreError, SignBlock, ValidatorStore}; /// If the we are unable to reach the Web3Signer HTTP API within this time out then we will /// assume it failed to start. @@ -309,7 +310,7 @@ mod tests { /// A testing rig which holds a `ValidatorStore`. struct ValidatorStoreRig { - validator_store: Arc>, + validator_store: Arc>, _validator_dir: TempDir, runtime: Arc, _runtime_shutdown: async_channel::Sender<()>, @@ -358,12 +359,12 @@ mod tests { let slot_clock = TestingSlotClock::new(Slot::new(0), Duration::from_secs(0), Duration::from_secs(1)); - let config = validator_store::Config { + let config = lighthouse_validator_store::Config { enable_web3signer_slashing_protection: slashing_protection_config.local, ..Default::default() }; - let validator_store = ValidatorStore::<_, E>::new( + let validator_store = LighthouseValidatorStore::new( initialized_validators, slashing_protection, Hash256::repeat_byte(42), @@ -488,7 +489,7 @@ mod tests { generate_sig: F, ) -> Self where - F: Fn(PublicKeyBytes, Arc>) -> R, + F: Fn(PublicKeyBytes, Arc>) -> R, R: Future, // We use the `SignedObject` trait to white-list objects for comparison. This avoids // accidentally comparing something meaningless like a `()`. @@ -523,8 +524,8 @@ mod tests { web3signer_should_sign: bool, ) -> Self where - F: Fn(PublicKeyBytes, Arc>) -> R, - R: Future>, + F: Fn(PublicKeyBytes, Arc>) -> R, + R: Future>, { for validator_rig in &self.validator_rigs { let result = diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 504d96ae1c1..8f945dc0d28 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -25,6 +25,7 @@ fdlimit = "0.3.0" graffiti_file = { workspace = true } hyper = { workspace = true } initialized_validators = { workspace = true } +lighthouse_validator_store = { workspace = true } metrics = { workspace = true } monitoring_api = { workspace = true } parking_lot = { workspace = true } diff --git a/validator_client/beacon_node_fallback/Cargo.toml b/validator_client/beacon_node_fallback/Cargo.toml index c15ded43d78..a871beb03ba 100644 --- a/validator_client/beacon_node_fallback/Cargo.toml +++ b/validator_client/beacon_node_fallback/Cargo.toml @@ -9,14 +9,14 @@ name = "beacon_node_fallback" path = "src/lib.rs" [dependencies] -environment = { workspace = true } eth2 = { workspace = true } futures = { workspace = true } itertools = { workspace = true } serde = { workspace = true } -slog = { workspace = true } slot_clock = { workspace = true } strum = { workspace = true } +task_executor = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } types = { workspace = true } validator_metrics = { workspace = true } diff --git a/validator_client/beacon_node_fallback/src/beacon_node_health.rs b/validator_client/beacon_node_fallback/src/beacon_node_health.rs index e5b04876560..33ebc659b3b 100644 --- a/validator_client/beacon_node_fallback/src/beacon_node_health.rs +++ b/validator_client/beacon_node_fallback/src/beacon_node_health.rs @@ -2,10 +2,10 @@ use super::CandidateError; use eth2::BeaconNodeHttpClient; use itertools::Itertools; use serde::{Deserialize, Serialize}; -use slog::{warn, Logger}; use std::cmp::Ordering; use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; +use tracing::warn; use types::Slot; /// Sync distances between 0 and DEFAULT_SYNC_TOLERANCE are considered `synced`. @@ -290,15 +290,13 @@ impl BeaconNodeHealth { pub async fn check_node_health( beacon_node: &BeaconNodeHttpClient, - log: &Logger, ) -> Result<(Slot, bool, bool), CandidateError> { let resp = match beacon_node.get_node_syncing().await { Ok(resp) => resp, Err(e) => { warn!( - log, - "Unable connect to beacon node"; - "error" => %e + error = %e, + "Unable connect to beacon node" ); return Err(CandidateError::Offline); diff --git a/validator_client/beacon_node_fallback/src/lib.rs b/validator_client/beacon_node_fallback/src/lib.rs index 95a221f1897..1201020a812 100644 --- a/validator_client/beacon_node_fallback/src/lib.rs +++ b/validator_client/beacon_node_fallback/src/lib.rs @@ -7,21 +7,20 @@ use beacon_node_health::{ check_node_health, BeaconNodeHealth, BeaconNodeSyncDistanceTiers, ExecutionEngineHealth, IsOptimistic, SyncDistanceTier, }; -use environment::RuntimeContext; use eth2::BeaconNodeHttpClient; use futures::future; use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer}; -use slog::{debug, error, warn, Logger}; use slot_clock::SlotClock; use std::cmp::Ordering; use std::fmt; use std::fmt::Debug; use std::future::Future; -use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; use strum::{EnumString, EnumVariantNames}; +use task_executor::TaskExecutor; use tokio::{sync::RwLock, time::sleep}; +use tracing::{debug, error, warn}; use types::{ChainSpec, Config as ConfigSpec, EthSpec, Slot}; use validator_metrics::{inc_counter_vec, ENDPOINT_ERRORS, ENDPOINT_REQUESTS}; @@ -59,17 +58,16 @@ pub struct LatencyMeasurement { /// /// See `SLOT_LOOKAHEAD` for information about when this should run. pub fn start_fallback_updater_service( - context: RuntimeContext, - beacon_nodes: Arc>, + executor: TaskExecutor, + beacon_nodes: Arc>, ) -> Result<(), &'static str> { - let executor = context.executor; if beacon_nodes.slot_clock.is_none() { return Err("Cannot start fallback updater without slot clock"); } let future = async move { loop { - beacon_nodes.update_all_candidates().await; + beacon_nodes.update_all_candidates::().await; let sleep_time = beacon_nodes .slot_clock @@ -184,29 +182,27 @@ impl Serialize for CandidateInfo { /// Represents a `BeaconNodeHttpClient` inside a `BeaconNodeFallback` that may or may not be used /// for a query. #[derive(Clone, Debug)] -pub struct CandidateBeaconNode { +pub struct CandidateBeaconNode { pub index: usize, pub beacon_node: BeaconNodeHttpClient, pub health: Arc>>, - _phantom: PhantomData, } -impl PartialEq for CandidateBeaconNode { +impl PartialEq for CandidateBeaconNode { fn eq(&self, other: &Self) -> bool { self.index == other.index && self.beacon_node == other.beacon_node } } -impl Eq for CandidateBeaconNode {} +impl Eq for CandidateBeaconNode {} -impl CandidateBeaconNode { +impl CandidateBeaconNode { /// Instantiate a new node. pub fn new(beacon_node: BeaconNodeHttpClient, index: usize) -> Self { Self { index, beacon_node, health: Arc::new(RwLock::new(Err(CandidateError::Uninitialized))), - _phantom: PhantomData, } } @@ -215,20 +211,19 @@ impl CandidateBeaconNode { *self.health.read().await } - pub async fn refresh_health( + pub async fn refresh_health( &self, distance_tiers: &BeaconNodeSyncDistanceTiers, slot_clock: Option<&T>, spec: &ChainSpec, - log: &Logger, ) -> Result<(), CandidateError> { - if let Err(e) = self.is_compatible(spec, log).await { + if let Err(e) = self.is_compatible::(spec).await { *self.health.write().await = Err(e); return Err(e); } if let Some(slot_clock) = slot_clock { - match check_node_health(&self.beacon_node, log).await { + match check_node_health(&self.beacon_node).await { Ok((head, is_optimistic, el_offline)) => { let Some(slot_clock_head) = slot_clock.now() else { let e = match slot_clock.is_prior_to_genesis() { @@ -286,17 +281,16 @@ impl CandidateBeaconNode { } /// Checks if the node has the correct specification. - async fn is_compatible(&self, spec: &ChainSpec, log: &Logger) -> Result<(), CandidateError> { + async fn is_compatible(&self, spec: &ChainSpec) -> Result<(), CandidateError> { let config = self .beacon_node .get_config_spec::() .await .map_err(|e| { error!( - log, - "Unable to read spec from beacon node"; - "error" => %e, - "endpoint" => %self.beacon_node, + error = %e, + endpoint = %self.beacon_node, + "Unable to read spec from beacon node" ); CandidateError::Offline })? @@ -304,62 +298,56 @@ impl CandidateBeaconNode { let beacon_node_spec = ChainSpec::from_config::(&config).ok_or_else(|| { error!( - log, + endpoint = %self.beacon_node, "The minimal/mainnet spec type of the beacon node does not match the validator \ - client. See the --network command."; - "endpoint" => %self.beacon_node, + client. See the --network command." + ); CandidateError::Incompatible })?; if beacon_node_spec.genesis_fork_version != spec.genesis_fork_version { error!( - log, - "Beacon node is configured for a different network"; - "endpoint" => %self.beacon_node, - "bn_genesis_fork" => ?beacon_node_spec.genesis_fork_version, - "our_genesis_fork" => ?spec.genesis_fork_version, + endpoint = %self.beacon_node, + bn_genesis_fork = ?beacon_node_spec.genesis_fork_version, + our_genesis_fork = ?spec.genesis_fork_version, + "Beacon node is configured for a different network" ); return Err(CandidateError::Incompatible); } else if beacon_node_spec.altair_fork_epoch != spec.altair_fork_epoch { warn!( - log, - "Beacon node has mismatched Altair fork epoch"; - "endpoint" => %self.beacon_node, - "endpoint_altair_fork_epoch" => ?beacon_node_spec.altair_fork_epoch, - "hint" => UPDATE_REQUIRED_LOG_HINT, + endpoint = %self.beacon_node, + endpoint_altair_fork_epoch = ?beacon_node_spec.altair_fork_epoch, + hint = UPDATE_REQUIRED_LOG_HINT, + "Beacon node has mismatched Altair fork epoch" ); } else if beacon_node_spec.bellatrix_fork_epoch != spec.bellatrix_fork_epoch { warn!( - log, - "Beacon node has mismatched Bellatrix fork epoch"; - "endpoint" => %self.beacon_node, - "endpoint_bellatrix_fork_epoch" => ?beacon_node_spec.bellatrix_fork_epoch, - "hint" => UPDATE_REQUIRED_LOG_HINT, + endpoint = %self.beacon_node, + endpoint_bellatrix_fork_epoch = ?beacon_node_spec.bellatrix_fork_epoch, + hint = UPDATE_REQUIRED_LOG_HINT, + "Beacon node has mismatched Bellatrix fork epoch" ); } else if beacon_node_spec.capella_fork_epoch != spec.capella_fork_epoch { warn!( - log, - "Beacon node has mismatched Capella fork epoch"; - "endpoint" => %self.beacon_node, - "endpoint_capella_fork_epoch" => ?beacon_node_spec.capella_fork_epoch, - "hint" => UPDATE_REQUIRED_LOG_HINT, + endpoint = %self.beacon_node, + endpoint_capella_fork_epoch = ?beacon_node_spec.capella_fork_epoch, + hint = UPDATE_REQUIRED_LOG_HINT, + "Beacon node has mismatched Capella fork epoch" ); } else if beacon_node_spec.deneb_fork_epoch != spec.deneb_fork_epoch { warn!( - log, - "Beacon node has mismatched Deneb fork epoch"; - "endpoint" => %self.beacon_node, - "endpoint_deneb_fork_epoch" => ?beacon_node_spec.deneb_fork_epoch, - "hint" => UPDATE_REQUIRED_LOG_HINT, + endpoint = %self.beacon_node, + endpoint_deneb_fork_epoch = ?beacon_node_spec.deneb_fork_epoch, + hint = UPDATE_REQUIRED_LOG_HINT, + "Beacon node has mismatched Deneb fork epoch" ); } else if beacon_node_spec.electra_fork_epoch != spec.electra_fork_epoch { warn!( - log, - "Beacon node has mismatched Electra fork epoch"; - "endpoint" => %self.beacon_node, - "endpoint_electra_fork_epoch" => ?beacon_node_spec.electra_fork_epoch, - "hint" => UPDATE_REQUIRED_LOG_HINT, + endpoint = %self.beacon_node, + endpoint_electra_fork_epoch = ?beacon_node_spec.electra_fork_epoch, + hint = UPDATE_REQUIRED_LOG_HINT, + "Beacon node has mismatched Electra fork epoch" ); } @@ -371,22 +359,20 @@ impl CandidateBeaconNode { /// behaviour, where the failure of one candidate results in the next candidate receiving an /// identical query. #[derive(Clone, Debug)] -pub struct BeaconNodeFallback { - pub candidates: Arc>>>, +pub struct BeaconNodeFallback { + pub candidates: Arc>>, distance_tiers: BeaconNodeSyncDistanceTiers, slot_clock: Option, broadcast_topics: Vec, spec: Arc, - log: Logger, } -impl BeaconNodeFallback { +impl BeaconNodeFallback { pub fn new( - candidates: Vec>, + candidates: Vec, config: Config, broadcast_topics: Vec, spec: Arc, - log: Logger, ) -> Self { let distance_tiers = config.sync_tolerances; Self { @@ -395,7 +381,6 @@ impl BeaconNodeFallback { slot_clock: None, broadcast_topics, spec, - log, } } @@ -466,7 +451,7 @@ impl BeaconNodeFallback { /// It is possible for a node to return an unsynced status while continuing to serve /// low quality responses. To route around this it's best to poll all connected beacon nodes. /// A previous implementation of this function polled only the unavailable BNs. - pub async fn update_all_candidates(&self) { + pub async fn update_all_candidates(&self) { // Clone the vec, so we release the read lock immediately. // `candidate.health` is behind an Arc, so this would still allow us to mutate the values. let candidates = self.candidates.read().await.clone(); @@ -474,11 +459,10 @@ impl BeaconNodeFallback { let mut nodes = Vec::with_capacity(candidates.len()); for candidate in candidates.iter() { - futures.push(candidate.refresh_health( + futures.push(candidate.refresh_health::( &self.distance_tiers, self.slot_clock.as_ref(), &self.spec, - &self.log, )); nodes.push(candidate.beacon_node.to_string()); } @@ -491,10 +475,9 @@ impl BeaconNodeFallback { if let Err(e) = result { if *e != CandidateError::PreGenesis { warn!( - self.log, - "A connected beacon node errored during routine health check"; - "error" => ?e, - "endpoint" => node, + error = ?e, + endpoint = %node, + "A connected beacon node errored during routine health check" ); } } @@ -566,11 +549,7 @@ impl BeaconNodeFallback { // Run `func` using a `candidate`, returning the value or capturing errors. for candidate in candidates.iter() { - futures.push(Self::run_on_candidate( - candidate.beacon_node.clone(), - &func, - &self.log, - )); + futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func)); } drop(candidates); @@ -588,11 +567,7 @@ impl BeaconNodeFallback { // Run `func` using a `candidate`, returning the value or capturing errors. for candidate in candidates.iter() { - futures.push(Self::run_on_candidate( - candidate.beacon_node.clone(), - &func, - &self.log, - )); + futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func)); } drop(candidates); @@ -611,7 +586,6 @@ impl BeaconNodeFallback { async fn run_on_candidate( candidate: BeaconNodeHttpClient, func: F, - log: &Logger, ) -> Result)> where F: Fn(BeaconNodeHttpClient) -> R, @@ -626,10 +600,9 @@ impl BeaconNodeFallback { Ok(val) => Ok(val), Err(e) => { debug!( - log, - "Request to beacon node failed"; - "node" => %candidate, - "error" => ?e, + node = %candidate, + error = ?e, + "Request to beacon node failed" ); inc_counter_vec(&ENDPOINT_ERRORS, &[candidate.as_ref()]); Err((candidate.to_string(), Error::RequestFailed(e))) @@ -656,11 +629,7 @@ impl BeaconNodeFallback { // Run `func` using a `candidate`, returning the value or capturing errors. for candidate in candidates.iter() { - futures.push(Self::run_on_candidate( - candidate.beacon_node.clone(), - &func, - &self.log, - )); + futures.push(Self::run_on_candidate(candidate.beacon_node.clone(), &func)); } drop(candidates); @@ -693,7 +662,7 @@ impl BeaconNodeFallback { } /// Helper functions to allow sorting candidate nodes by health. -async fn sort_nodes_by_health(nodes: &mut Vec>) { +async fn sort_nodes_by_health(nodes: &mut Vec) { // Fetch all health values. let health_results: Vec> = future::join_all(nodes.iter().map(|node| node.health())).await; @@ -711,7 +680,7 @@ async fn sort_nodes_by_health(nodes: &mut Vec }); // Reorder candidates based on the sorted indices. - let sorted_nodes: Vec> = indices_with_health + let sorted_nodes: Vec = indices_with_health .into_iter() .map(|(index, _)| nodes[index].clone()) .collect(); @@ -743,9 +712,7 @@ mod tests { use eth2::Timeouts; use std::str::FromStr; use strum::VariantNames; - use types::{MainnetEthSpec, Slot}; - - type E = MainnetEthSpec; + use types::Slot; #[test] fn api_topic_all() { @@ -764,7 +731,7 @@ mod tests { let optimistic_status = IsOptimistic::No; let execution_status = ExecutionEngineHealth::Healthy; - fn new_candidate(index: usize) -> CandidateBeaconNode { + fn new_candidate(index: usize) -> CandidateBeaconNode { let beacon_node = BeaconNodeHttpClient::new( SensitiveUrl::parse(&format!("http://example_{index}.com")).unwrap(), Timeouts::set_all(Duration::from_secs(index as u64)), diff --git a/validator_client/doppelganger_service/Cargo.toml b/validator_client/doppelganger_service/Cargo.toml index 66b61a411b6..a37dbbf3033 100644 --- a/validator_client/doppelganger_service/Cargo.toml +++ b/validator_client/doppelganger_service/Cargo.toml @@ -14,6 +14,7 @@ slot_clock = { workspace = true } task_executor = { workspace = true } tokio = { workspace = true } types = { workspace = true } +validator_store = { workspace = true } [dev-dependencies] futures = { workspace = true } diff --git a/validator_client/doppelganger_service/src/lib.rs b/validator_client/doppelganger_service/src/lib.rs index 35228fe3546..8412e9b1746 100644 --- a/validator_client/doppelganger_service/src/lib.rs +++ b/validator_client/doppelganger_service/src/lib.rs @@ -41,68 +41,7 @@ use std::sync::Arc; use task_executor::ShutdownReason; use tokio::time::sleep; use types::{Epoch, EthSpec, PublicKeyBytes, Slot}; - -/// A wrapper around `PublicKeyBytes` which encodes information about the status of a validator -/// pubkey with regards to doppelganger protection. -#[derive(Debug, PartialEq)] -pub enum DoppelgangerStatus { - /// Doppelganger protection has approved this for signing. - /// - /// This is because the service has waited some period of time to - /// detect other instances of this key on the network. - SigningEnabled(PublicKeyBytes), - /// Doppelganger protection is still waiting to detect other instances. - /// - /// Do not use this pubkey for signing slashable messages!! - /// - /// However, it can safely be used for other non-slashable operations (e.g., collecting duties - /// or subscribing to subnets). - SigningDisabled(PublicKeyBytes), - /// This pubkey is unknown to the doppelganger service. - /// - /// This represents a serious internal error in the program. This validator will be permanently - /// disabled! - UnknownToDoppelganger(PublicKeyBytes), -} - -impl DoppelgangerStatus { - /// Only return a pubkey if it is explicitly safe for doppelganger protection. - /// - /// If `Some(pubkey)` is returned, doppelganger has declared it safe for signing. - /// - /// ## Note - /// - /// "Safe" is only best-effort by doppelganger. There is no guarantee that a doppelganger - /// doesn't exist. - pub fn only_safe(self) -> Option { - match self { - DoppelgangerStatus::SigningEnabled(pubkey) => Some(pubkey), - DoppelgangerStatus::SigningDisabled(_) => None, - DoppelgangerStatus::UnknownToDoppelganger(_) => None, - } - } - - /// Returns a key regardless of whether or not doppelganger has approved it. Such a key might be - /// used for signing non-slashable messages, duties collection or other activities. - /// - /// If the validator is unknown to doppelganger then `None` will be returned. - pub fn ignored(self) -> Option { - match self { - DoppelgangerStatus::SigningEnabled(pubkey) => Some(pubkey), - DoppelgangerStatus::SigningDisabled(pubkey) => Some(pubkey), - DoppelgangerStatus::UnknownToDoppelganger(_) => None, - } - } - - /// Only return a pubkey if it will not be used for signing due to doppelganger detection. - pub fn only_unsafe(self) -> Option { - match self { - DoppelgangerStatus::SigningEnabled(_) => None, - DoppelgangerStatus::SigningDisabled(pubkey) => Some(pubkey), - DoppelgangerStatus::UnknownToDoppelganger(pubkey) => Some(pubkey), - } - } -} +use validator_store::{DoppelgangerStatus, ValidatorStore}; struct LivenessResponses { current_epoch_responses: Vec, @@ -113,13 +52,6 @@ struct LivenessResponses { /// validators on the network. pub const DEFAULT_REMAINING_DETECTION_EPOCHS: u64 = 1; -/// This crate cannot depend on ValidatorStore as validator_store depends on this crate and -/// initialises the doppelganger protection. For this reason, we abstract the validator store -/// functions this service needs through the following trait -pub trait DoppelgangerValidatorStore { - fn get_validator_index(&self, pubkey: &PublicKeyBytes) -> Option; -} - /// Store the per-validator status of doppelganger checking. #[derive(Debug, PartialEq)] pub struct DoppelgangerState { @@ -162,8 +94,8 @@ impl DoppelgangerState { /// If the BN fails to respond to either of these requests, simply return an empty response. /// This behaviour is to help prevent spurious failures on the BN from needlessly preventing /// doppelganger progression. -async fn beacon_node_liveness<'a, T: 'static + SlotClock, E: EthSpec>( - beacon_nodes: Arc>, +async fn beacon_node_liveness<'a, T: 'static + SlotClock>( + beacon_nodes: Arc>, log: Logger, current_epoch: Epoch, validator_indices: Vec, @@ -290,16 +222,16 @@ impl DoppelgangerService { service: Arc, context: RuntimeContext, validator_store: Arc, - beacon_nodes: Arc>, + beacon_nodes: Arc>, slot_clock: T, ) -> Result<(), String> where E: EthSpec, T: 'static + SlotClock, - V: DoppelgangerValidatorStore + Send + Sync + 'static, + V: ValidatorStore + Send + Sync + 'static, { // Define the `get_index` function as one that uses the validator store. - let get_index = move |pubkey| validator_store.get_validator_index(&pubkey); + let get_index = move |pubkey| validator_store.validator_index(&pubkey); // Define the `get_liveness` function as one that queries the beacon node API. let log = service.log.clone(); @@ -704,6 +636,7 @@ mod test { test_utils::{SeedableRng, TestRandom, XorShiftRng}, MainnetEthSpec, }; + use validator_store::DoppelgangerStatus; const DEFAULT_VALIDATORS: usize = 8; diff --git a/validator_client/graffiti_file/src/lib.rs b/validator_client/graffiti_file/src/lib.rs index 9dab2e78272..7cab504bf85 100644 --- a/validator_client/graffiti_file/src/lib.rs +++ b/validator_client/graffiti_file/src/lib.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; -use slog::warn; use std::collections::HashMap; use std::fs::File; use std::io::{prelude::*, BufReader}; @@ -108,19 +107,14 @@ fn read_line(line: &str) -> Result<(Option, Graffiti), Error> { // the next block produced by the validator with the given public key. pub fn determine_graffiti( validator_pubkey: &PublicKeyBytes, - log: &slog::Logger, graffiti_file: Option, validator_definition_graffiti: Option, graffiti_flag: Option, ) -> Option { + // TODO when merging make sure logging on failure is back: + // warn!(log, "Failed to read graffiti file"; "error" => ?e); graffiti_file - .and_then(|mut g| match g.load_graffiti(validator_pubkey) { - Ok(g) => g, - Err(e) => { - warn!(log, "Failed to read graffiti file"; "error" => ?e); - None - } - }) + .and_then(|mut g| g.load_graffiti(validator_pubkey).unwrap_or(None)) .or(validator_definition_graffiti) .or(graffiti_flag) } diff --git a/validator_client/http_api/Cargo.toml b/validator_client/http_api/Cargo.toml index 76a021ab8c3..e1968c6b29f 100644 --- a/validator_client/http_api/Cargo.toml +++ b/validator_client/http_api/Cargo.toml @@ -16,12 +16,13 @@ deposit_contract = { workspace = true } directory = { workspace = true } dirs = { workspace = true } doppelganger_service = { workspace = true } -eth2 = { workspace = true } -eth2_keystore = { workspace = true } +eth2 = { workspace = true } +eth2_keystore = { workspace = true } ethereum_serde_utils = { workspace = true } filesystem = { workspace = true } graffiti_file = { workspace = true } initialized_validators = { workspace = true } +lighthouse_validator_store = { workspace = true } lighthouse_version = { workspace = true } logging = { workspace = true } parking_lot = { workspace = true } @@ -30,19 +31,19 @@ sensitive_url = { workspace = true } serde = { workspace = true } signing_method = { workspace = true } slashing_protection = { workspace = true } -slog = { workspace = true } -slot_clock = { workspace = true } -sysinfo = { workspace = true } -system_health = { workspace = true } -task_executor = { workspace = true } -tempfile = { workspace = true } -tokio = { workspace = true } -tokio-stream = { workspace = true } -types = { workspace = true } -url = { workspace = true } -validator_dir = { workspace = true } -validator_services = { workspace = true } -validator_store = { workspace = true } +slog = { workspace = true } +slot_clock = { workspace = true } +sysinfo = { workspace = true } +system_health = { workspace = true } +task_executor = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +types = { workspace = true } +url = { workspace = true } +validator_dir = { workspace = true } +validator_services = { workspace = true } +validator_store = { workspace = true } warp = { workspace = true } warp_utils = { workspace = true } zeroize = { workspace = true } diff --git a/validator_client/http_api/src/create_signed_voluntary_exit.rs b/validator_client/http_api/src/create_signed_voluntary_exit.rs index 32269b202b0..0c714715a71 100644 --- a/validator_client/http_api/src/create_signed_voluntary_exit.rs +++ b/validator_client/http_api/src/create_signed_voluntary_exit.rs @@ -1,5 +1,6 @@ use bls::{PublicKey, PublicKeyBytes}; use eth2::types::GenericResponse; +use lighthouse_validator_store::LighthouseValidatorStore; use slog::{info, Logger}; use slot_clock::SlotClock; use std::sync::Arc; @@ -9,7 +10,7 @@ use validator_store::ValidatorStore; pub async fn create_signed_voluntary_exit( pubkey: PublicKey, maybe_epoch: Option, - validator_store: Arc>, + validator_store: Arc>, slot_clock: T, log: Logger, ) -> Result, warp::Rejection> { diff --git a/validator_client/http_api/src/create_validator.rs b/validator_client/http_api/src/create_validator.rs index f90a1057a43..278274198d5 100644 --- a/validator_client/http_api/src/create_validator.rs +++ b/validator_client/http_api/src/create_validator.rs @@ -5,12 +5,11 @@ use account_utils::{ random_mnemonic, random_password, }; use eth2::lighthouse_vc::types::{self as api_types}; +use lighthouse_validator_store::LighthouseValidatorStore; use slot_clock::SlotClock; use std::path::{Path, PathBuf}; -use types::ChainSpec; -use types::EthSpec; +use types::{ChainSpec, EthSpec}; use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder}; -use validator_store::ValidatorStore; use zeroize::Zeroizing; /// Create some validator EIP-2335 keystores and store them on disk. Then, enroll the validators in @@ -30,7 +29,7 @@ pub async fn create_validators_mnemonic, T: 'static + SlotClock, validator_requests: &[api_types::ValidatorRequest], validator_dir: P, secrets_dir: Option, - validator_store: &ValidatorStore, + validator_store: &LighthouseValidatorStore, spec: &ChainSpec, ) -> Result<(Vec, Mnemonic), warp::Rejection> { let mnemonic = mnemonic_opt.unwrap_or_else(random_mnemonic); @@ -178,7 +177,7 @@ pub async fn create_validators_mnemonic, T: 'static + SlotClock, pub async fn create_validators_web3signer( validators: Vec, - validator_store: &ValidatorStore, + validator_store: &LighthouseValidatorStore, ) -> Result<(), warp::Rejection> { for validator in validators { validator_store diff --git a/validator_client/http_api/src/graffiti.rs b/validator_client/http_api/src/graffiti.rs index 86238a697c6..4372b14b04a 100644 --- a/validator_client/http_api/src/graffiti.rs +++ b/validator_client/http_api/src/graffiti.rs @@ -1,12 +1,12 @@ use bls::PublicKey; +use lighthouse_validator_store::LighthouseValidatorStore; use slot_clock::SlotClock; use std::sync::Arc; use types::{graffiti::GraffitiString, EthSpec, Graffiti}; -use validator_store::ValidatorStore; pub fn get_graffiti( validator_pubkey: PublicKey, - validator_store: Arc>, + validator_store: Arc>, graffiti_flag: Option, ) -> Result { let initialized_validators_rw_lock = validator_store.initialized_validators(); @@ -29,7 +29,7 @@ pub fn get_graffiti( pub fn set_graffiti( validator_pubkey: PublicKey, graffiti: GraffitiString, - validator_store: Arc>, + validator_store: Arc>, ) -> Result<(), warp::Rejection> { let initialized_validators_rw_lock = validator_store.initialized_validators(); let mut initialized_validators = initialized_validators_rw_lock.write(); @@ -55,7 +55,7 @@ pub fn set_graffiti( pub fn delete_graffiti( validator_pubkey: PublicKey, - validator_store: Arc>, + validator_store: Arc>, ) -> Result<(), warp::Rejection> { let initialized_validators_rw_lock = validator_store.initialized_validators(); let mut initialized_validators = initialized_validators_rw_lock.write(); diff --git a/validator_client/http_api/src/keystores.rs b/validator_client/http_api/src/keystores.rs index fd6b4fdae51..9b57bbd5577 100644 --- a/validator_client/http_api/src/keystores.rs +++ b/validator_client/http_api/src/keystores.rs @@ -10,6 +10,7 @@ use eth2::lighthouse_vc::{ }; use eth2_keystore::Keystore; use initialized_validators::{Error, InitializedValidators}; +use lighthouse_validator_store::LighthouseValidatorStore; use signing_method::SigningMethod; use slog::{info, warn, Logger}; use slot_clock::SlotClock; @@ -19,13 +20,12 @@ use task_executor::TaskExecutor; use tokio::runtime::Handle; use types::{EthSpec, PublicKeyBytes}; use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder}; -use validator_store::ValidatorStore; use warp::Rejection; use warp_utils::reject::{custom_bad_request, custom_server_error}; use zeroize::Zeroizing; pub fn list( - validator_store: Arc>, + validator_store: Arc>, ) -> ListKeystoresResponse { let initialized_validators_rwlock = validator_store.initialized_validators(); let initialized_validators = initialized_validators_rwlock.read(); @@ -62,7 +62,7 @@ pub fn import( request: ImportKeystoresRequest, validator_dir: PathBuf, secrets_dir: Option, - validator_store: Arc>, + validator_store: Arc>, task_executor: TaskExecutor, log: Logger, ) -> Result { @@ -122,7 +122,7 @@ pub fn import( ) } else if let Some(handle) = task_executor.handle() { // Import the keystore. - match import_single_keystore( + match import_single_keystore::<_, E>( keystore, password, validator_dir.clone(), @@ -171,7 +171,7 @@ fn import_single_keystore( password: Zeroizing, validator_dir_path: PathBuf, secrets_dir: Option, - validator_store: &ValidatorStore, + validator_store: &LighthouseValidatorStore, handle: Handle, ) -> Result { // Check if the validator key already exists, erroring if it is a remote signer validator. @@ -241,7 +241,7 @@ fn import_single_keystore( pub fn delete( request: DeleteKeystoresRequest, - validator_store: Arc>, + validator_store: Arc>, task_executor: TaskExecutor, log: Logger, ) -> Result { @@ -274,7 +274,7 @@ pub fn delete( pub fn export( request: DeleteKeystoresRequest, - validator_store: Arc>, + validator_store: Arc>, task_executor: TaskExecutor, log: Logger, ) -> Result { diff --git a/validator_client/http_api/src/lib.rs b/validator_client/http_api/src/lib.rs index 73ebe717af3..f499562f09c 100644 --- a/validator_client/http_api/src/lib.rs +++ b/validator_client/http_api/src/lib.rs @@ -13,6 +13,7 @@ use graffiti::{delete_graffiti, get_graffiti, set_graffiti}; use create_signed_voluntary_exit::create_signed_voluntary_exit; use graffiti_file::{determine_graffiti, GraffitiFile}; +use lighthouse_validator_store::LighthouseValidatorStore; use validator_store::ValidatorStore; use account_utils::{ @@ -40,7 +41,6 @@ use slog::{crit, info, warn, Logger}; use slot_clock::SlotClock; use std::collections::HashMap; use std::future::Future; -use std::marker::PhantomData; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; @@ -75,11 +75,11 @@ impl From for Error { /// A wrapper around all the items required to spawn the HTTP server. /// /// The server will gracefully handle the case where any fields are `None`. -pub struct Context { +pub struct Context { pub task_executor: TaskExecutor, pub api_secret: ApiSecret, - pub block_service: Option>, - pub validator_store: Option>>, + pub block_service: Option, T>>, + pub validator_store: Option>>, pub validator_dir: Option, pub secrets_dir: Option, pub graffiti_file: Option, @@ -89,7 +89,6 @@ pub struct Context { pub log: Logger, pub sse_logging_components: Option, pub slot_clock: T, - pub _phantom: PhantomData, } /// Configuration for the HTTP server. @@ -324,7 +323,7 @@ pub fn serve( .and(warp::path("validators")) .and(warp::path::end()) .and(validator_store_filter.clone()) - .then(|validator_store: Arc>| { + .then(|validator_store: Arc>| { blocking_json_task(move || { let validators = validator_store .initialized_validators() @@ -349,7 +348,7 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .then( - |validator_pubkey: PublicKey, validator_store: Arc>| { + |validator_pubkey: PublicKey, validator_store: Arc>| { blocking_json_task(move || { let validator = validator_store .initialized_validators() @@ -400,10 +399,10 @@ pub fn serve( .and(graffiti_flag_filter) .and(log_filter.clone()) .then( - |validator_store: Arc>, + |validator_store: Arc>, graffiti_file: Option, graffiti_flag: Option, - log| { + _log| { blocking_json_task(move || { let mut result = HashMap::new(); for (key, graffiti_definition) in validator_store @@ -413,7 +412,6 @@ pub fn serve( { let graffiti = determine_graffiti( key, - &log, graffiti_file.clone(), graffiti_definition, graffiti_flag, @@ -431,33 +429,35 @@ pub fn serve( .and(warp::path("fallback_health")) .and(warp::path::end()) .and(block_service_filter.clone()) - .then(|block_filter: BlockService| async move { - let mut result: HashMap> = HashMap::new(); - - let mut beacon_nodes = Vec::new(); - for node in &*block_filter.beacon_nodes.candidates.read().await { - beacon_nodes.push(CandidateInfo { - index: node.index, - endpoint: node.beacon_node.to_string(), - health: *node.health.read().await, - }); - } - result.insert("beacon_nodes".to_string(), beacon_nodes); - - if let Some(proposer_nodes_list) = &block_filter.proposer_nodes { - let mut proposer_nodes = Vec::new(); - for node in &*proposer_nodes_list.candidates.read().await { - proposer_nodes.push(CandidateInfo { + .then( + |block_filter: BlockService, T>| async move { + let mut result: HashMap> = HashMap::new(); + + let mut beacon_nodes = Vec::new(); + for node in &*block_filter.beacon_nodes.candidates.read().await { + beacon_nodes.push(CandidateInfo { index: node.index, endpoint: node.beacon_node.to_string(), health: *node.health.read().await, }); } - result.insert("proposer_nodes".to_string(), proposer_nodes); - } + result.insert("beacon_nodes".to_string(), beacon_nodes); + + if let Some(proposer_nodes_list) = &block_filter.proposer_nodes { + let mut proposer_nodes = Vec::new(); + for node in &*proposer_nodes_list.candidates.read().await { + proposer_nodes.push(CandidateInfo { + index: node.index, + endpoint: node.beacon_node.to_string(), + health: *node.health.read().await, + }); + } + result.insert("proposer_nodes".to_string(), proposer_nodes); + } - blocking_json_task(move || Ok(api_types::GenericResponse::from(result))).await - }); + blocking_json_task(move || Ok(api_types::GenericResponse::from(result))).await + }, + ); // POST lighthouse/validators/ let post_validators = warp::path("lighthouse") @@ -473,14 +473,14 @@ pub fn serve( move |body: Vec, validator_dir: PathBuf, secrets_dir: PathBuf, - validator_store: Arc>, + validator_store: Arc>, spec: Arc, task_executor: TaskExecutor| { blocking_json_task(move || { let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir); if let Some(handle) = task_executor.handle() { let (validators, mnemonic) = - handle.block_on(create_validators_mnemonic( + handle.block_on(create_validators_mnemonic::<_, _, E>( None, None, &body, @@ -518,7 +518,7 @@ pub fn serve( move |body: api_types::CreateValidatorsMnemonicRequest, validator_dir: PathBuf, secrets_dir: PathBuf, - validator_store: Arc>, + validator_store: Arc>, spec: Arc, task_executor: TaskExecutor| { blocking_json_task(move || { @@ -532,7 +532,7 @@ pub fn serve( )) })?; let (validators, _mnemonic) = - handle.block_on(create_validators_mnemonic( + handle.block_on(create_validators_mnemonic::<_, _, E>( Some(mnemonic), Some(body.key_derivation_path_offset), &body.validators, @@ -565,7 +565,7 @@ pub fn serve( move |body: api_types::KeystoreValidatorsPostRequest, validator_dir: PathBuf, secrets_dir: PathBuf, - validator_store: Arc>, + validator_store: Arc>, task_executor: TaskExecutor| { blocking_json_task(move || { // Check to ensure the password is correct. @@ -651,7 +651,7 @@ pub fn serve( .and(task_executor_filter.clone()) .then( |body: Vec, - validator_store: Arc>, + validator_store: Arc>, task_executor: TaskExecutor| { blocking_json_task(move || { if let Some(handle) = task_executor.handle() { @@ -679,7 +679,7 @@ pub fn serve( ), }) .collect(); - handle.block_on(create_validators_web3signer( + handle.block_on(create_validators_web3signer::<_, E>( web3signers, &validator_store, ))?; @@ -705,7 +705,7 @@ pub fn serve( .then( |validator_pubkey: PublicKey, body: api_types::ValidatorPatchRequest, - validator_store: Arc>, + validator_store: Arc>, graffiti_file: Option, task_executor: TaskExecutor| { blocking_json_task(move || { @@ -859,7 +859,7 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .then( - |validator_pubkey: PublicKey, validator_store: Arc>| { + |validator_pubkey: PublicKey, validator_store: Arc>| { blocking_json_task(move || { if validator_store .initialized_validators() @@ -900,7 +900,7 @@ pub fn serve( .then( |validator_pubkey: PublicKey, request: api_types::UpdateFeeRecipientRequest, - validator_store: Arc>| { + validator_store: Arc>| { blocking_json_task(move || { if validator_store .initialized_validators() @@ -936,7 +936,7 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .then( - |validator_pubkey: PublicKey, validator_store: Arc>| { + |validator_pubkey: PublicKey, validator_store: Arc>| { blocking_json_task(move || { if validator_store .initialized_validators() @@ -972,7 +972,7 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .then( - |validator_pubkey: PublicKey, validator_store: Arc>| { + |validator_pubkey: PublicKey, validator_store: Arc>| { blocking_json_task(move || { if validator_store .initialized_validators() @@ -1005,7 +1005,7 @@ pub fn serve( .then( |validator_pubkey: PublicKey, request: api_types::UpdateGasLimitRequest, - validator_store: Arc>| { + validator_store: Arc>| { blocking_json_task(move || { if validator_store .initialized_validators() @@ -1041,7 +1041,7 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .then( - |validator_pubkey: PublicKey, validator_store: Arc>| { + |validator_pubkey: PublicKey, validator_store: Arc>| { blocking_json_task(move || { if validator_store .initialized_validators() @@ -1083,14 +1083,14 @@ pub fn serve( .then( |pubkey: PublicKey, query: api_types::VoluntaryExitQuery, - validator_store: Arc>, + validator_store: Arc>, slot_clock: T, log, task_executor: TaskExecutor| { blocking_json_task(move || { if let Some(handle) = task_executor.handle() { let signed_voluntary_exit = - handle.block_on(create_signed_voluntary_exit( + handle.block_on(create_signed_voluntary_exit::( pubkey, query.epoch, validator_store, @@ -1117,7 +1117,7 @@ pub fn serve( .and(graffiti_flag_filter) .then( |pubkey: PublicKey, - validator_store: Arc>, + validator_store: Arc>, graffiti_flag: Option| { blocking_json_task(move || { let graffiti = get_graffiti(pubkey.clone(), validator_store, graffiti_flag)?; @@ -1141,7 +1141,7 @@ pub fn serve( .then( |pubkey: PublicKey, query: SetGraffitiRequest, - validator_store: Arc>, + validator_store: Arc>, graffiti_file: Option| { blocking_json_task(move || { if graffiti_file.is_some() { @@ -1166,7 +1166,7 @@ pub fn serve( .and(graffiti_file_filter.clone()) .then( |pubkey: PublicKey, - validator_store: Arc>, + validator_store: Arc>, graffiti_file: Option| { blocking_json_task(move || { if graffiti_file.is_some() { @@ -1183,7 +1183,7 @@ pub fn serve( // GET /eth/v1/keystores let get_std_keystores = std_keystores.and(validator_store_filter.clone()).then( - |validator_store: Arc>| { + |validator_store: Arc>| { blocking_json_task(move || Ok(keystores::list(validator_store))) }, ); @@ -1200,7 +1200,7 @@ pub fn serve( move |request, validator_dir, secrets_dir, validator_store, task_executor, log| { let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir); blocking_json_task(move || { - keystores::import( + keystores::import::<_, E>( request, validator_dir, secrets_dir, @@ -1226,7 +1226,7 @@ pub fn serve( // GET /eth/v1/remotekeys let get_std_remotekeys = std_remotekeys.and(validator_store_filter.clone()).then( - |validator_store: Arc>| { + |validator_store: Arc>| { blocking_json_task(move || Ok(remotekeys::list(validator_store))) }, ); @@ -1239,7 +1239,7 @@ pub fn serve( .and(log_filter.clone()) .then(|request, validator_store, task_executor, log| { blocking_json_task(move || { - remotekeys::import(request, validator_store, task_executor, log) + remotekeys::import::<_, E>(request, validator_store, task_executor, log) }) }); diff --git a/validator_client/http_api/src/remotekeys.rs b/validator_client/http_api/src/remotekeys.rs index 289be571825..802871ea5a9 100644 --- a/validator_client/http_api/src/remotekeys.rs +++ b/validator_client/http_api/src/remotekeys.rs @@ -8,6 +8,7 @@ use eth2::lighthouse_vc::std_types::{ ListRemotekeysResponse, SingleListRemotekeysResponse, Status, }; use initialized_validators::{Error, InitializedValidators}; +use lighthouse_validator_store::LighthouseValidatorStore; use slog::{info, warn, Logger}; use slot_clock::SlotClock; use std::sync::Arc; @@ -15,12 +16,11 @@ use task_executor::TaskExecutor; use tokio::runtime::Handle; use types::{EthSpec, PublicKeyBytes}; use url::Url; -use validator_store::ValidatorStore; use warp::Rejection; use warp_utils::reject::custom_server_error; pub fn list( - validator_store: Arc>, + validator_store: Arc>, ) -> ListRemotekeysResponse { let initialized_validators_rwlock = validator_store.initialized_validators(); let initialized_validators = initialized_validators_rwlock.read(); @@ -50,7 +50,7 @@ pub fn list( pub fn import( request: ImportRemotekeysRequest, - validator_store: Arc>, + validator_store: Arc>, task_executor: TaskExecutor, log: Logger, ) -> Result { @@ -65,8 +65,12 @@ pub fn import( for remotekey in request.remote_keys { let status = if let Some(handle) = task_executor.handle() { // Import the keystore. - match import_single_remotekey(remotekey.pubkey, remotekey.url, &validator_store, handle) - { + match import_single_remotekey::<_, E>( + remotekey.pubkey, + remotekey.url, + &validator_store, + handle, + ) { Ok(status) => Status::ok(status), Err(e) => { warn!( @@ -92,7 +96,7 @@ pub fn import( fn import_single_remotekey( pubkey: PublicKeyBytes, url: String, - validator_store: &ValidatorStore, + validator_store: &LighthouseValidatorStore, handle: Handle, ) -> Result { if let Err(url_err) = Url::parse(&url) { @@ -146,7 +150,7 @@ fn import_single_remotekey( pub fn delete( request: DeleteRemotekeysRequest, - validator_store: Arc>, + validator_store: Arc>, task_executor: TaskExecutor, log: Logger, ) -> Result { diff --git a/validator_client/http_api/src/test_utils.rs b/validator_client/http_api/src/test_utils.rs index 390095eec73..7145e27e2cb 100644 --- a/validator_client/http_api/src/test_utils.rs +++ b/validator_client/http_api/src/test_utils.rs @@ -14,20 +14,20 @@ use eth2::{ use eth2_keystore::KeystoreBuilder; use initialized_validators::key_cache::{KeyCache, CACHE_FILENAME}; use initialized_validators::{InitializedValidators, OnDecryptFailure}; +use lighthouse_validator_store::{Config as ValidatorStoreConfig, LighthouseValidatorStore}; use logging::test_logger; use parking_lot::RwLock; use sensitive_url::SensitiveUrl; use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME}; use slot_clock::{SlotClock, TestingSlotClock}; use std::future::Future; -use std::marker::PhantomData; use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; use std::time::Duration; use task_executor::test_utils::TestRuntime; use tempfile::{tempdir, TempDir}; use tokio::sync::oneshot; -use validator_store::{Config as ValidatorStoreConfig, ValidatorStore}; +use validator_services::block_service::BlockService; use zeroize::Zeroizing; pub const PASSWORD_BYTES: &[u8] = &[42, 50, 37]; @@ -55,7 +55,7 @@ pub struct Web3SignerValidatorScenario { pub struct ApiTester { pub client: ValidatorClientHttpClient, pub initialized_validators: Arc>, - pub validator_store: Arc>, + pub validator_store: Arc>, pub url: SensitiveUrl, pub api_token: String, pub test_runtime: TestRuntime, @@ -105,7 +105,7 @@ impl ApiTester { let test_runtime = TestRuntime::default(); - let validator_store = Arc::new(ValidatorStore::<_, E>::new( + let validator_store = Arc::new(LighthouseValidatorStore::new( initialized_validators, slashing_protection, Hash256::repeat_byte(42), @@ -126,7 +126,7 @@ impl ApiTester { let context = Arc::new(Context { task_executor: test_runtime.task_executor.clone(), api_secret, - block_service: None, + block_service: None::, _>>, validator_dir: Some(validator_dir.path().into()), secrets_dir: Some(secrets_dir.path().into()), validator_store: Some(validator_store.clone()), @@ -137,7 +137,6 @@ impl ApiTester { log, sse_logging_components: None, slot_clock, - _phantom: PhantomData, }); let ctx = context; let (shutdown_tx, shutdown_rx) = oneshot::channel(); @@ -145,7 +144,7 @@ impl ApiTester { // It's not really interesting why this triggered, just that it happened. let _ = shutdown_rx.await; }; - let (listening_socket, server) = super::serve(ctx, server_shutdown).unwrap(); + let (listening_socket, server) = super::serve::<_, E>(ctx, server_shutdown).unwrap(); tokio::spawn(server); diff --git a/validator_client/http_api/src/tests.rs b/validator_client/http_api/src/tests.rs index 7ea3d7ebaab..b650f49c08d 100644 --- a/validator_client/http_api/src/tests.rs +++ b/validator_client/http_api/src/tests.rs @@ -18,13 +18,13 @@ use eth2::{ Error as ApiError, }; use eth2_keystore::KeystoreBuilder; +use lighthouse_validator_store::{Config as ValidatorStoreConfig, LighthouseValidatorStore}; use logging::test_logger; use parking_lot::RwLock; use sensitive_url::SensitiveUrl; use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME}; use slot_clock::{SlotClock, TestingSlotClock}; use std::future::Future; -use std::marker::PhantomData; use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; use std::sync::Arc; @@ -32,7 +32,7 @@ use std::time::Duration; use task_executor::test_utils::TestRuntime; use tempfile::{tempdir, TempDir}; use types::graffiti::GraffitiString; -use validator_store::{Config as ValidatorStoreConfig, ValidatorStore}; +use validator_store::ValidatorStore; use zeroize::Zeroizing; const PASSWORD_BYTES: &[u8] = &[42, 50, 37]; @@ -43,7 +43,7 @@ type E = MainnetEthSpec; struct ApiTester { client: ValidatorClientHttpClient, initialized_validators: Arc>, - validator_store: Arc>, + validator_store: Arc>, url: SensitiveUrl, slot_clock: TestingSlotClock, _validator_dir: TempDir, @@ -95,7 +95,7 @@ impl ApiTester { let test_runtime = TestRuntime::default(); - let validator_store = Arc::new(ValidatorStore::<_, E>::new( + let validator_store = Arc::new(LighthouseValidatorStore::new( initialized_validators, slashing_protection, Hash256::repeat_byte(42), @@ -135,11 +135,10 @@ impl ApiTester { sse_logging_components: None, log, slot_clock: slot_clock.clone(), - _phantom: PhantomData, }); let ctx = context.clone(); let (listening_socket, server) = - super::serve(ctx, test_runtime.task_executor.exit()).unwrap(); + super::serve::<_, E>(ctx, test_runtime.task_executor.exit()).unwrap(); tokio::spawn(server); @@ -708,7 +707,7 @@ impl ApiTester { assert_eq!( self.validator_store - .determine_validator_builder_boost_factor(&validator.voting_pubkey), + .determine_builder_boost_factor(&validator.voting_pubkey), builder_boost_factor ); @@ -718,7 +717,7 @@ impl ApiTester { pub fn assert_default_builder_boost_factor(self, builder_boost_factor: Option) -> Self { assert_eq!( self.validator_store - .determine_default_builder_boost_factor(), + .determine_builder_boost_factor(&PublicKeyBytes::empty()), builder_boost_factor ); @@ -1165,7 +1164,7 @@ async fn validator_derived_builder_boost_factor_with_process_defaults() { }) .await .assert_default_builder_boost_factor(Some(80)) - .assert_validator_derived_builder_boost_factor(0, None) + .assert_validator_derived_builder_boost_factor(0, Some(80)) .await .set_builder_proposals(0, false) .await diff --git a/validator_client/http_api/src/tests/keystores.rs b/validator_client/http_api/src/tests/keystores.rs index 6559a2bb9e5..5a4cfc677f5 100644 --- a/validator_client/http_api/src/tests/keystores.rs +++ b/validator_client/http_api/src/tests/keystores.rs @@ -8,12 +8,13 @@ use eth2::lighthouse_vc::{ types::Web3SignerValidatorRequest, }; use itertools::Itertools; +use lighthouse_validator_store::DEFAULT_GAS_LIMIT; use rand::{rngs::SmallRng, Rng, SeedableRng}; use slashing_protection::interchange::{Interchange, InterchangeMetadata}; use std::{collections::HashMap, path::Path}; use tokio::runtime::Handle; use types::{attestation::AttestationBase, Address}; -use validator_store::DEFAULT_GAS_LIMIT; +use validator_store::ValidatorStore; use zeroize::Zeroizing; fn new_keystore(password: Zeroizing) -> Keystore { diff --git a/validator_client/http_metrics/Cargo.toml b/validator_client/http_metrics/Cargo.toml index c29a4d18fa0..11dd443ab1d 100644 --- a/validator_client/http_metrics/Cargo.toml +++ b/validator_client/http_metrics/Cargo.toml @@ -5,6 +5,7 @@ edition = { workspace = true } authors = ["Sigma Prime "] [dependencies] +lighthouse_validator_store = { workspace = true } lighthouse_version = { workspace = true } malloc_utils = { workspace = true } metrics = { workspace = true } @@ -15,6 +16,5 @@ slot_clock = { workspace = true } types = { workspace = true } validator_metrics = { workspace = true } validator_services = { workspace = true } -validator_store = { workspace = true } warp = { workspace = true } warp_utils = { workspace = true } diff --git a/validator_client/http_metrics/src/lib.rs b/validator_client/http_metrics/src/lib.rs index 984b752e5a5..e37d198ad9a 100644 --- a/validator_client/http_metrics/src/lib.rs +++ b/validator_client/http_metrics/src/lib.rs @@ -2,6 +2,7 @@ //! //! For other endpoints, see the `http_api` crate. +use lighthouse_validator_store::LighthouseValidatorStore; use lighthouse_version::version_with_platform; use malloc_utils::scrape_allocator_metrics; use parking_lot::RwLock; @@ -14,7 +15,6 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use types::EthSpec; use validator_services::duties_service::DutiesService; -use validator_store::ValidatorStore; use warp::{http::Response, Filter}; #[derive(Debug)] @@ -35,17 +35,19 @@ impl From for Error { } } +type ValidatorStore = LighthouseValidatorStore; + /// Contains objects which have shared access from inside/outside of the metrics server. -pub struct Shared { - pub validator_store: Option>>, - pub duties_service: Option>>, +pub struct Shared { + pub validator_store: Option>>, + pub duties_service: Option, SystemTimeSlotClock>>>, pub genesis_time: Option, } /// A wrapper around all the items required to spawn the HTTP server. /// /// The server will gracefully handle the case where any fields are `None`. -pub struct Context { +pub struct Context { pub config: Config, pub shared: RwLock>, pub log: Logger, @@ -122,7 +124,7 @@ pub fn serve( .map(move || inner_ctx.clone()) .and_then(|ctx: Arc>| async move { Ok::<_, warp::Rejection>( - gather_prometheus_metrics(&ctx) + gather_prometheus_metrics::(&ctx) .map(|body| { Response::builder() .status(200) diff --git a/validator_client/lighthouse_validator_store/Cargo.toml b/validator_client/lighthouse_validator_store/Cargo.toml new file mode 100644 index 00000000000..3997453e2ce --- /dev/null +++ b/validator_client/lighthouse_validator_store/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "lighthouse_validator_store" +version = "0.1.0" +edition = { workspace = true } +authors = ["Sigma Prime "] + +[dependencies] +account_utils = { workspace = true } +doppelganger_service = { workspace = true } +initialized_validators = { workspace = true } +parking_lot = { workspace = true } +serde = { workspace = true } +signing_method = { workspace = true } +slashing_protection = { workspace = true } +slog = { workspace = true } +slot_clock = { workspace = true } +task_executor = { workspace = true } +types = { workspace = true } +validator_metrics = { workspace = true } +validator_store = { workspace = true } diff --git a/validator_client/lighthouse_validator_store/src/lib.rs b/validator_client/lighthouse_validator_store/src/lib.rs new file mode 100644 index 00000000000..5d438e50c04 --- /dev/null +++ b/validator_client/lighthouse_validator_store/src/lib.rs @@ -0,0 +1,1109 @@ +use account_utils::validator_definitions::{PasswordStorage, ValidatorDefinition}; +use doppelganger_service::DoppelgangerService; +use initialized_validators::InitializedValidators; +use parking_lot::{Mutex, RwLock}; +use serde::{Deserialize, Serialize}; +use signing_method::Error as SigningError; +use signing_method::{SignableMessage, SigningContext, SigningMethod}; +use slashing_protection::{ + interchange::Interchange, InterchangeError, NotSafe, Safe, SlashingDatabase, +}; +use slog::{crit, error, info, warn, Logger}; +use slot_clock::SlotClock; +use std::marker::PhantomData; +use std::path::Path; +use std::sync::Arc; +use task_executor::TaskExecutor; +use types::{ + graffiti::GraffitiString, AbstractExecPayload, Address, AggregateAndProof, Attestation, + BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, Domain, Epoch, EthSpec, Fork, + Graffiti, Hash256, PublicKeyBytes, SelectionProof, Signature, SignedAggregateAndProof, + SignedBeaconBlock, SignedContributionAndProof, SignedRoot, SignedValidatorRegistrationData, + SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, + SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, + VoluntaryExit, +}; +use validator_store::{ + DoppelgangerStatus, Error as ValidatorStoreError, ProposalData, SignedBlock, UnsignedBlock, + ValidatorStore, +}; + +pub type Error = ValidatorStoreError; + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct Config { + /// Fallback fee recipient address. + pub fee_recipient: Option
, + /// Fallback gas limit. + pub gas_limit: Option, + /// Enable use of the blinded block endpoints during proposals. + pub builder_proposals: bool, + /// Enable slashing protection even while using web3signer keys. + pub enable_web3signer_slashing_protection: bool, + /// If true, Lighthouse will prefer builder proposals, if available. + pub prefer_builder_proposals: bool, + /// Specifies the boost factor, a percentage multiplier to apply to the builder's payload value. + pub builder_boost_factor: Option, +} + +/// Number of epochs of slashing protection history to keep. +/// +/// This acts as a maximum safe-guard against clock drift. +const SLASHING_PROTECTION_HISTORY_EPOCHS: u64 = 512; + +/// Currently used as the default gas limit in execution clients. +/// +/// https://github.com/ethereum/builder-specs/issues/17 +pub const DEFAULT_GAS_LIMIT: u64 = 30_000_000; + +pub struct LighthouseValidatorStore { + validators: Arc>, + slashing_protection: SlashingDatabase, + slashing_protection_last_prune: Arc>, + genesis_validators_root: Hash256, + spec: Arc, + log: Logger, + doppelganger_service: Option>, + slot_clock: T, + fee_recipient_process: Option
, + gas_limit: Option, + builder_proposals: bool, + enable_web3signer_slashing_protection: bool, + prefer_builder_proposals: bool, + builder_boost_factor: Option, + task_executor: TaskExecutor, + _phantom: PhantomData, +} + +impl LighthouseValidatorStore { + // All arguments are different types. Making the fields `pub` is undesired. A builder seems + // unnecessary. + #[allow(clippy::too_many_arguments)] + pub fn new( + validators: InitializedValidators, + slashing_protection: SlashingDatabase, + genesis_validators_root: Hash256, + spec: Arc, + doppelganger_service: Option>, + slot_clock: T, + config: &Config, + task_executor: TaskExecutor, + log: Logger, + ) -> Self { + Self { + validators: Arc::new(RwLock::new(validators)), + slashing_protection, + slashing_protection_last_prune: Arc::new(Mutex::new(Epoch::new(0))), + genesis_validators_root, + spec, + log, + doppelganger_service, + slot_clock, + fee_recipient_process: config.fee_recipient, + gas_limit: config.gas_limit, + builder_proposals: config.builder_proposals, + enable_web3signer_slashing_protection: config.enable_web3signer_slashing_protection, + prefer_builder_proposals: config.prefer_builder_proposals, + builder_boost_factor: config.builder_boost_factor, + task_executor, + _phantom: PhantomData, + } + } + + /// Register all local validators in doppelganger protection to try and prevent instances of + /// duplicate validators operating on the network at the same time. + /// + /// This function has no effect if doppelganger protection is disabled. + pub fn register_all_in_doppelganger_protection_if_enabled(&self) -> Result<(), String> { + if let Some(doppelganger_service) = &self.doppelganger_service { + for pubkey in self.validators.read().iter_voting_pubkeys() { + doppelganger_service.register_new_validator::(*pubkey, &self.slot_clock)? + } + } + + Ok(()) + } + + /// Returns `true` if doppelganger protection is enabled, or else `false`. + pub fn doppelganger_protection_enabled(&self) -> bool { + self.doppelganger_service.is_some() + } + + pub fn initialized_validators(&self) -> Arc> { + self.validators.clone() + } + + /// Indicates if the `voting_public_key` exists in self and is enabled. + pub fn has_validator(&self, voting_public_key: &PublicKeyBytes) -> bool { + self.validators + .read() + .validator(voting_public_key) + .is_some() + } + + /// Insert a new validator to `self`, where the validator is represented by an EIP-2335 + /// keystore on the filesystem. + #[allow(clippy::too_many_arguments)] + pub async fn add_validator_keystore>( + &self, + voting_keystore_path: P, + password_storage: PasswordStorage, + enable: bool, + graffiti: Option, + suggested_fee_recipient: Option
, + gas_limit: Option, + builder_proposals: Option, + builder_boost_factor: Option, + prefer_builder_proposals: Option, + ) -> Result { + let mut validator_def = ValidatorDefinition::new_keystore_with_password( + voting_keystore_path, + password_storage, + graffiti.map(Into::into), + suggested_fee_recipient, + gas_limit, + builder_proposals, + builder_boost_factor, + prefer_builder_proposals, + ) + .map_err(|e| format!("failed to create validator definitions: {:?}", e))?; + + validator_def.enabled = enable; + + self.add_validator(validator_def).await + } + + /// Insert a new validator to `self`. + /// + /// This function includes: + /// + /// - Adding the validator definition to the YAML file, saving it to the filesystem. + /// - Enabling the validator with the slashing protection database. + /// - If `enable == true`, starting to perform duties for the validator. + // FIXME: ignore this clippy lint until the validator store is refactored to use async locks + #[allow(clippy::await_holding_lock)] + pub async fn add_validator( + &self, + validator_def: ValidatorDefinition, + ) -> Result { + let validator_pubkey = validator_def.voting_public_key.compress(); + + self.slashing_protection + .register_validator(validator_pubkey) + .map_err(|e| format!("failed to register validator: {:?}", e))?; + + if let Some(doppelganger_service) = &self.doppelganger_service { + doppelganger_service + .register_new_validator::(validator_pubkey, &self.slot_clock)?; + } + + self.validators + .write() + .add_definition_replace_disabled(validator_def.clone()) + .await + .map_err(|e| format!("Unable to add definition: {:?}", e))?; + + Ok(validator_def) + } + + /// Returns doppelganger statuses for all enabled validators. + #[allow(clippy::needless_collect)] // Collect is required to avoid holding a lock. + pub fn doppelganger_statuses(&self) -> Vec { + // Collect all the pubkeys first to avoid interleaving locks on `self.validators` and + // `self.doppelganger_service`. + let pubkeys = self + .validators + .read() + .iter_voting_pubkeys() + .cloned() + .collect::>(); + + pubkeys + .into_iter() + .map(|pubkey| { + self.doppelganger_service + .as_ref() + .map(|doppelganger_service| doppelganger_service.validator_status(pubkey)) + // Allow signing on all pubkeys if doppelganger protection is disabled. + .unwrap_or_else(|| DoppelgangerStatus::SigningEnabled(pubkey)) + }) + .collect() + } + + fn fork(&self, epoch: Epoch) -> Fork { + self.spec.fork_at_epoch(epoch) + } + + /// Returns a `SigningMethod` for `validator_pubkey` *only if* that validator is considered safe + /// by doppelganger protection. + fn doppelganger_checked_signing_method( + &self, + validator_pubkey: PublicKeyBytes, + ) -> Result, Error> { + if self.doppelganger_protection_allows_signing(validator_pubkey) { + self.validators + .read() + .signing_method(&validator_pubkey) + .ok_or(Error::UnknownPubkey(validator_pubkey)) + } else { + Err(Error::DoppelgangerProtected(validator_pubkey)) + } + } + + /// Returns a `SigningMethod` for `validator_pubkey` regardless of that validators doppelganger + /// protection status. + /// + /// ## Warning + /// + /// This method should only be used for signing non-slashable messages. + fn doppelganger_bypassed_signing_method( + &self, + validator_pubkey: PublicKeyBytes, + ) -> Result, Error> { + self.validators + .read() + .signing_method(&validator_pubkey) + .ok_or(Error::UnknownPubkey(validator_pubkey)) + } + + fn signing_context(&self, domain: Domain, signing_epoch: Epoch) -> SigningContext { + if domain == Domain::VoluntaryExit { + if self.spec.fork_name_at_epoch(signing_epoch).deneb_enabled() { + // EIP-7044 + SigningContext { + domain, + epoch: signing_epoch, + fork: Fork { + previous_version: self.spec.capella_fork_version, + current_version: self.spec.capella_fork_version, + epoch: signing_epoch, + }, + genesis_validators_root: self.genesis_validators_root, + } + } else { + SigningContext { + domain, + epoch: signing_epoch, + fork: self.fork(signing_epoch), + genesis_validators_root: self.genesis_validators_root, + } + } + } else { + SigningContext { + domain, + epoch: signing_epoch, + fork: self.fork(signing_epoch), + genesis_validators_root: self.genesis_validators_root, + } + } + } + + pub fn get_fee_recipient_defaulting(&self, fee_recipient: Option
) -> Option
{ + // If there's nothing in the file, try the process-level default value. + fee_recipient.or(self.fee_recipient_process) + } + + /// Returns the suggested_fee_recipient from `validator_definitions.yml` if any. + /// This has been pulled into a private function so the read lock is dropped easily + fn suggested_fee_recipient(&self, validator_pubkey: &PublicKeyBytes) -> Option
{ + self.validators + .read() + .suggested_fee_recipient(validator_pubkey) + } + + /// Returns the gas limit for the given public key. The priority order for fetching + /// the gas limit is: + /// + /// 1. validator_definitions.yml + /// 2. process level gas limit + /// 3. `DEFAULT_GAS_LIMIT` + pub fn get_gas_limit(&self, validator_pubkey: &PublicKeyBytes) -> u64 { + self.get_gas_limit_defaulting(self.validators.read().gas_limit(validator_pubkey)) + } + + fn get_gas_limit_defaulting(&self, gas_limit: Option) -> u64 { + // If there is a `gas_limit` in the validator definitions yaml + // file, use that value. + gas_limit + // If there's nothing in the file, try the process-level default value. + .or(self.gas_limit) + // If there's no process-level default, use the `DEFAULT_GAS_LIMIT`. + .unwrap_or(DEFAULT_GAS_LIMIT) + } + + /// Returns a `bool` for the given public key that denotes whether this validator should use the + /// builder API. The priority order for fetching this value is: + /// + /// 1. validator_definitions.yml + /// 2. process level flag + pub fn get_builder_proposals(&self, validator_pubkey: &PublicKeyBytes) -> bool { + // If there is a `suggested_fee_recipient` in the validator definitions yaml + // file, use that value. + self.get_builder_proposals_defaulting( + self.validators.read().builder_proposals(validator_pubkey), + ) + } + + /// Returns a `u64` for the given public key that denotes the builder boost factor. The priority order for fetching this value is: + /// + /// 1. validator_definitions.yml + /// 2. process level flag + pub fn get_builder_boost_factor(&self, validator_pubkey: &PublicKeyBytes) -> Option { + self.validators + .read() + .builder_boost_factor(validator_pubkey) + .or(self.builder_boost_factor) + } + + /// Returns a `bool` for the given public key that denotes whether this validator should prefer a + /// builder payload. The priority order for fetching this value is: + /// + /// 1. validator_definitions.yml + /// 2. process level flag + pub fn get_prefer_builder_proposals(&self, validator_pubkey: &PublicKeyBytes) -> bool { + self.validators + .read() + .prefer_builder_proposals(validator_pubkey) + .unwrap_or(self.prefer_builder_proposals) + } + + fn get_builder_proposals_defaulting(&self, builder_proposals: Option) -> bool { + builder_proposals + // If there's nothing in the file, try the process-level default value. + .unwrap_or(self.builder_proposals) + } + + pub fn import_slashing_protection( + &self, + interchange: Interchange, + ) -> Result<(), InterchangeError> { + self.slashing_protection + .import_interchange_info(interchange, self.genesis_validators_root)?; + Ok(()) + } + + /// Export slashing protection data while also disabling the given keys in the database. + /// + /// If any key is unknown to the slashing protection database it will be silently omitted + /// from the result. It is the caller's responsibility to check whether all keys provided + /// had data returned for them. + pub fn export_slashing_protection_for_keys( + &self, + pubkeys: &[PublicKeyBytes], + ) -> Result { + self.slashing_protection.with_transaction(|txn| { + let known_pubkeys = pubkeys + .iter() + .filter_map(|pubkey| { + let validator_id = self + .slashing_protection + .get_validator_id_ignoring_status(txn, pubkey) + .ok()?; + + Some( + self.slashing_protection + .update_validator_status(txn, validator_id, false) + .map(|()| *pubkey), + ) + }) + .collect::, _>>()?; + self.slashing_protection.export_interchange_info_in_txn( + self.genesis_validators_root, + Some(&known_pubkeys), + txn, + ) + }) + } + + async fn sign_abstract_block>( + &self, + validator_pubkey: PublicKeyBytes, + block: BeaconBlock, + current_slot: Slot, + ) -> Result, Error> { + // Make sure the block slot is not higher than the current slot to avoid potential attacks. + if block.slot() > current_slot { + warn!( + self.log, + "Not signing block with slot greater than current slot"; + "block_slot" => block.slot().as_u64(), + "current_slot" => current_slot.as_u64() + ); + return Err(Error::GreaterThanCurrentSlot { + slot: block.slot(), + current_slot, + }); + } + + let signing_epoch = block.epoch(); + let signing_context = self.signing_context(Domain::BeaconProposer, signing_epoch); + let domain_hash = signing_context.domain_hash(&self.spec); + + let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; + + // Check for slashing conditions. + let slashing_status = if signing_method + .requires_local_slashing_protection(self.enable_web3signer_slashing_protection) + { + self.slashing_protection.check_and_insert_block_proposal( + &validator_pubkey, + &block.block_header(), + domain_hash, + ) + } else { + Ok(Safe::Valid) + }; + + match slashing_status { + // We can safely sign this block without slashing. + Ok(Safe::Valid) => { + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_BLOCKS_TOTAL, + &[validator_metrics::SUCCESS], + ); + + let signature = signing_method + .get_signature( + SignableMessage::BeaconBlock(&block), + signing_context, + &self.spec, + &self.task_executor, + ) + .await?; + Ok(SignedBeaconBlock::from_block(block, signature)) + } + Ok(Safe::SameData) => { + warn!( + self.log, + "Skipping signing of previously signed block"; + ); + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_BLOCKS_TOTAL, + &[validator_metrics::SAME_DATA], + ); + Err(Error::SameData) + } + Err(NotSafe::UnregisteredValidator(pk)) => { + warn!( + self.log, + "Not signing block for unregistered validator"; + "msg" => "Carefully consider running with --init-slashing-protection (see --help)", + "public_key" => format!("{:?}", pk) + ); + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_BLOCKS_TOTAL, + &[validator_metrics::UNREGISTERED], + ); + Err(Error::Slashable(NotSafe::UnregisteredValidator(pk))) + } + Err(e) => { + crit!( + self.log, + "Not signing slashable block"; + "error" => format!("{:?}", e) + ); + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_BLOCKS_TOTAL, + &[validator_metrics::SLASHABLE], + ); + Err(Error::Slashable(e)) + } + } + } +} + +impl ValidatorStore for LighthouseValidatorStore { + type Error = SigningError; + type E = E; + + /// Attempts to resolve the pubkey to a validator index. + /// + /// It may return `None` if the `pubkey` is: + /// + /// - Unknown. + /// - Known, but with an unknown index. + fn validator_index(&self, pubkey: &PublicKeyBytes) -> Option { + self.validators.read().get_index(pubkey) + } + + /// Returns all voting pubkeys for all enabled validators. + /// + /// The `filter_func` allows for filtering pubkeys based upon their `DoppelgangerStatus`. There + /// are two primary functions used here: + /// + /// - `DoppelgangerStatus::only_safe`: only returns pubkeys which have passed doppelganger + /// protection and are safe-enough to sign messages. + /// - `DoppelgangerStatus::ignored`: returns all the pubkeys from `only_safe` *plus* those still + /// undergoing protection. This is useful for collecting duties or other non-signing tasks. + #[allow(clippy::needless_collect)] // Collect is required to avoid holding a lock. + fn voting_pubkeys(&self, filter_func: F) -> I + where + I: FromIterator, + F: Fn(DoppelgangerStatus) -> Option, + { + // Collect all the pubkeys first to avoid interleaving locks on `self.validators` and + // `self.doppelganger_service()`. + let pubkeys = self + .validators + .read() + .iter_voting_pubkeys() + .cloned() + .collect::>(); + + pubkeys + .into_iter() + .map(|pubkey| { + self.doppelganger_service + .as_ref() + .map(|doppelganger_service| doppelganger_service.validator_status(pubkey)) + // Allow signing on all pubkeys if doppelganger protection is disabled. + .unwrap_or_else(|| DoppelgangerStatus::SigningEnabled(pubkey)) + }) + .filter_map(filter_func) + .collect() + } + + /// Check if the `validator_pubkey` is permitted by the doppleganger protection to sign + /// messages. + fn doppelganger_protection_allows_signing(&self, validator_pubkey: PublicKeyBytes) -> bool { + self.doppelganger_service + .as_ref() + // If there's no doppelganger service then we assume it is purposefully disabled and + // declare that all keys are safe with regard to it. + .map_or(true, |doppelganger_service| { + doppelganger_service + .validator_status(validator_pubkey) + .only_safe() + .is_some() + }) + } + + fn num_voting_validators(&self) -> usize { + self.validators.read().num_enabled() + } + + fn graffiti(&self, validator_pubkey: &PublicKeyBytes) -> Option { + self.validators.read().graffiti(validator_pubkey) + } + + /// Returns the fee recipient for the given public key. The priority order for fetching + /// the fee recipient is: + /// 1. validator_definitions.yml + /// 2. process level fee recipient + fn get_fee_recipient(&self, validator_pubkey: &PublicKeyBytes) -> Option
{ + // If there is a `suggested_fee_recipient` in the validator definitions yaml + // file, use that value. + self.get_fee_recipient_defaulting(self.suggested_fee_recipient(validator_pubkey)) + } + + /// Translate the per validator `builder_proposals`, `builder_boost_factor` and + /// `prefer_builder_proposals` to a boost factor, if available. + /// - If `prefer_builder_proposals` is true, set boost factor to `u64::MAX` to indicate a + /// preference for builder payloads. + /// - If `builder_boost_factor` is a value other than None, return its value as the boost factor. + /// - If `builder_proposals` is set to false, set boost factor to 0 to indicate a preference for + /// local payloads. + /// - Else return `None` to indicate no preference between builder and local payloads. + fn determine_builder_boost_factor(&self, validator_pubkey: &PublicKeyBytes) -> Option { + let validator_prefer_builder_proposals = self + .validators + .read() + .prefer_builder_proposals(validator_pubkey); + + if matches!(validator_prefer_builder_proposals, Some(true)) { + return Some(u64::MAX); + } + + let factor = self + .validators + .read() + .builder_boost_factor(validator_pubkey) + .or_else(|| { + if matches!( + self.validators.read().builder_proposals(validator_pubkey), + Some(false) + ) { + return Some(0); + } + None + }); + + factor.or_else(|| { + if self.prefer_builder_proposals { + return Some(u64::MAX); + } + self.builder_boost_factor.or({ + if !self.builder_proposals { + Some(0) + } else { + None + } + }) + }) + } + + async fn randao_reveal( + &self, + validator_pubkey: PublicKeyBytes, + signing_epoch: Epoch, + ) -> Result { + let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; + let signing_context = self.signing_context(Domain::Randao, signing_epoch); + + let signature = signing_method + .get_signature::>( + SignableMessage::RandaoReveal(signing_epoch), + signing_context, + &self.spec, + &self.task_executor, + ) + .await?; + + Ok(signature) + } + + fn set_validator_index(&self, validator_pubkey: &PublicKeyBytes, index: u64) { + self.initialized_validators() + .write() + .set_index(validator_pubkey, index); + } + + async fn sign_block( + &self, + validator_pubkey: PublicKeyBytes, + block: UnsignedBlock, + current_slot: Slot, + ) -> Result, Error> { + match block { + UnsignedBlock::Full(block) => self + .sign_abstract_block(validator_pubkey, block, current_slot) + .await + .map(SignedBlock::Full), + UnsignedBlock::Blinded(block) => self + .sign_abstract_block(validator_pubkey, block, current_slot) + .await + .map(SignedBlock::Blinded), + } + } + + async fn sign_attestation( + &self, + validator_pubkey: PublicKeyBytes, + validator_committee_position: usize, + attestation: &mut Attestation, + current_epoch: Epoch, + ) -> Result<(), Error> { + // Make sure the target epoch is not higher than the current epoch to avoid potential attacks. + if attestation.data().target.epoch > current_epoch { + return Err(Error::GreaterThanCurrentEpoch { + epoch: attestation.data().target.epoch, + current_epoch, + }); + } + + // Get the signing method and check doppelganger protection. + let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; + + // Checking for slashing conditions. + let signing_epoch = attestation.data().target.epoch; + let signing_context = self.signing_context(Domain::BeaconAttester, signing_epoch); + let domain_hash = signing_context.domain_hash(&self.spec); + let slashing_status = if signing_method + .requires_local_slashing_protection(self.enable_web3signer_slashing_protection) + { + self.slashing_protection.check_and_insert_attestation( + &validator_pubkey, + attestation.data(), + domain_hash, + ) + } else { + Ok(Safe::Valid) + }; + + match slashing_status { + // We can safely sign this attestation. + Ok(Safe::Valid) => { + let signature = signing_method + .get_signature::>( + SignableMessage::AttestationData(attestation.data()), + signing_context, + &self.spec, + &self.task_executor, + ) + .await?; + attestation + .add_signature(&signature, validator_committee_position) + .map_err(Error::UnableToSignAttestation)?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, + &[validator_metrics::SUCCESS], + ); + + Ok(()) + } + Ok(Safe::SameData) => { + warn!( + self.log, + "Skipping signing of previously signed attestation" + ); + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, + &[validator_metrics::SAME_DATA], + ); + Err(Error::SameData) + } + Err(NotSafe::UnregisteredValidator(pk)) => { + warn!( + self.log, + "Not signing attestation for unregistered validator"; + "msg" => "Carefully consider running with --init-slashing-protection (see --help)", + "public_key" => format!("{:?}", pk) + ); + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, + &[validator_metrics::UNREGISTERED], + ); + Err(Error::Slashable(NotSafe::UnregisteredValidator(pk))) + } + Err(e) => { + crit!( + self.log, + "Not signing slashable attestation"; + "attestation" => format!("{:?}", attestation.data()), + "error" => format!("{:?}", e) + ); + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, + &[validator_metrics::SLASHABLE], + ); + Err(Error::Slashable(e)) + } + } + } + + async fn sign_voluntary_exit( + &self, + validator_pubkey: PublicKeyBytes, + voluntary_exit: VoluntaryExit, + ) -> Result { + let signing_epoch = voluntary_exit.epoch; + let signing_context = self.signing_context(Domain::VoluntaryExit, signing_epoch); + let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; + + let signature = signing_method + .get_signature::>( + SignableMessage::VoluntaryExit(&voluntary_exit), + signing_context, + &self.spec, + &self.task_executor, + ) + .await?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_VOLUNTARY_EXITS_TOTAL, + &[validator_metrics::SUCCESS], + ); + + Ok(SignedVoluntaryExit { + message: voluntary_exit, + signature, + }) + } + + async fn sign_validator_registration_data( + &self, + validator_registration_data: ValidatorRegistrationData, + ) -> Result { + let domain_hash = self.spec.get_builder_domain(); + let signing_root = validator_registration_data.signing_root(domain_hash); + + let signing_method = + self.doppelganger_bypassed_signing_method(validator_registration_data.pubkey)?; + let signature = signing_method + .get_signature_from_root::>( + SignableMessage::ValidatorRegistration(&validator_registration_data), + signing_root, + &self.task_executor, + None, + ) + .await?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_VALIDATOR_REGISTRATIONS_TOTAL, + &[validator_metrics::SUCCESS], + ); + + Ok(SignedValidatorRegistrationData { + message: validator_registration_data, + signature, + }) + } + + /// Signs an `AggregateAndProof` for a given validator. + /// + /// The resulting `SignedAggregateAndProof` is sent on the aggregation channel and cannot be + /// modified by actors other than the signing validator. + async fn produce_signed_aggregate_and_proof( + &self, + validator_pubkey: PublicKeyBytes, + aggregator_index: u64, + aggregate: Attestation, + selection_proof: SelectionProof, + ) -> Result, Error> { + let signing_epoch = aggregate.data().target.epoch; + let signing_context = self.signing_context(Domain::AggregateAndProof, signing_epoch); + + let message = + AggregateAndProof::from_attestation(aggregator_index, aggregate, selection_proof); + + let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; + let signature = signing_method + .get_signature::>( + SignableMessage::SignedAggregateAndProof(message.to_ref()), + signing_context, + &self.spec, + &self.task_executor, + ) + .await?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_AGGREGATES_TOTAL, + &[validator_metrics::SUCCESS], + ); + + Ok(SignedAggregateAndProof::from_aggregate_and_proof( + message, signature, + )) + } + + /// Produces a `SelectionProof` for the `slot`, signed by with corresponding secret key to + /// `validator_pubkey`. + async fn produce_selection_proof( + &self, + validator_pubkey: PublicKeyBytes, + slot: Slot, + ) -> Result { + let signing_epoch = slot.epoch(E::slots_per_epoch()); + let signing_context = self.signing_context(Domain::SelectionProof, signing_epoch); + + // Bypass the `with_validator_signing_method` function. + // + // This is because we don't care about doppelganger protection when it comes to selection + // proofs. They are not slashable and we need them to subscribe to subnets on the BN. + // + // As long as we disallow `SignedAggregateAndProof` then these selection proofs will never + // be published on the network. + let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; + + let signature = signing_method + .get_signature::>( + SignableMessage::SelectionProof(slot), + signing_context, + &self.spec, + &self.task_executor, + ) + .await + .map_err(Error::SpecificError)?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_SELECTION_PROOFS_TOTAL, + &[validator_metrics::SUCCESS], + ); + + Ok(signature.into()) + } + + /// Produce a `SyncSelectionProof` for `slot` signed by the secret key of `validator_pubkey`. + async fn produce_sync_selection_proof( + &self, + validator_pubkey: &PublicKeyBytes, + slot: Slot, + subnet_id: SyncSubnetId, + ) -> Result { + let signing_epoch = slot.epoch(E::slots_per_epoch()); + let signing_context = + self.signing_context(Domain::SyncCommitteeSelectionProof, signing_epoch); + + // Bypass `with_validator_signing_method`: sync committee messages are not slashable. + let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_SYNC_SELECTION_PROOFS_TOTAL, + &[validator_metrics::SUCCESS], + ); + + let message = SyncAggregatorSelectionData { + slot, + subcommittee_index: subnet_id.into(), + }; + + let signature = signing_method + .get_signature::>( + SignableMessage::SyncSelectionProof(&message), + signing_context, + &self.spec, + &self.task_executor, + ) + .await + .map_err(Error::SpecificError)?; + + Ok(signature.into()) + } + + async fn produce_sync_committee_signature( + &self, + slot: Slot, + beacon_block_root: Hash256, + validator_index: u64, + validator_pubkey: &PublicKeyBytes, + ) -> Result { + let signing_epoch = slot.epoch(E::slots_per_epoch()); + let signing_context = self.signing_context(Domain::SyncCommittee, signing_epoch); + + // Bypass `with_validator_signing_method`: sync committee messages are not slashable. + let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?; + + let signature = signing_method + .get_signature::>( + SignableMessage::SyncCommitteeSignature { + beacon_block_root, + slot, + }, + signing_context, + &self.spec, + &self.task_executor, + ) + .await + .map_err(Error::SpecificError)?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_SYNC_COMMITTEE_MESSAGES_TOTAL, + &[validator_metrics::SUCCESS], + ); + + Ok(SyncCommitteeMessage { + slot, + beacon_block_root, + validator_index, + signature, + }) + } + + async fn produce_signed_contribution_and_proof( + &self, + aggregator_index: u64, + aggregator_pubkey: PublicKeyBytes, + contribution: SyncCommitteeContribution, + selection_proof: SyncSelectionProof, + ) -> Result, Error> { + let signing_epoch = contribution.slot.epoch(E::slots_per_epoch()); + let signing_context = self.signing_context(Domain::ContributionAndProof, signing_epoch); + + // Bypass `with_validator_signing_method`: sync committee messages are not slashable. + let signing_method = self.doppelganger_bypassed_signing_method(aggregator_pubkey)?; + + let message = ContributionAndProof { + aggregator_index, + contribution, + selection_proof: selection_proof.into(), + }; + + let signature = signing_method + .get_signature::>( + SignableMessage::SignedContributionAndProof(&message), + signing_context, + &self.spec, + &self.task_executor, + ) + .await + .map_err(Error::SpecificError)?; + + validator_metrics::inc_counter_vec( + &validator_metrics::SIGNED_SYNC_COMMITTEE_CONTRIBUTIONS_TOTAL, + &[validator_metrics::SUCCESS], + ); + + Ok(SignedContributionAndProof { message, signature }) + } + + /// Prune the slashing protection database so that it remains performant. + /// + /// This function will only do actual pruning periodically, so it should usually be + /// cheap to call. The `first_run` flag can be used to print a more verbose message when pruning + /// runs. + fn prune_slashing_protection_db(&self, current_epoch: Epoch, first_run: bool) { + // Attempt to prune every SLASHING_PROTECTION_HISTORY_EPOCHs, with a tolerance for + // missing the epoch that aligns exactly. + let mut last_prune = self.slashing_protection_last_prune.lock(); + if current_epoch / SLASHING_PROTECTION_HISTORY_EPOCHS + <= *last_prune / SLASHING_PROTECTION_HISTORY_EPOCHS + { + return; + } + + if first_run { + info!( + self.log, + "Pruning slashing protection DB"; + "epoch" => current_epoch, + "msg" => "pruning may take several minutes the first time it runs" + ); + } else { + info!(self.log, "Pruning slashing protection DB"; "epoch" => current_epoch); + } + + let _timer = + validator_metrics::start_timer(&validator_metrics::SLASHING_PROTECTION_PRUNE_TIMES); + + let new_min_target_epoch = current_epoch.saturating_sub(SLASHING_PROTECTION_HISTORY_EPOCHS); + let new_min_slot = new_min_target_epoch.start_slot(E::slots_per_epoch()); + + let all_pubkeys: Vec<_> = self.voting_pubkeys(DoppelgangerStatus::ignored); + + if let Err(e) = self + .slashing_protection + .prune_all_signed_attestations(all_pubkeys.iter(), new_min_target_epoch) + { + error!( + self.log, + "Error during pruning of signed attestations"; + "error" => ?e, + ); + return; + } + + if let Err(e) = self + .slashing_protection + .prune_all_signed_blocks(all_pubkeys.iter(), new_min_slot) + { + error!( + self.log, + "Error during pruning of signed blocks"; + "error" => ?e, + ); + return; + } + + *last_prune = current_epoch; + + info!(self.log, "Completed pruning of slashing protection DB"); + } + + /// Returns `ProposalData` for the provided `pubkey` if it exists in `InitializedValidators`. + /// `ProposalData` fields include defaulting logic described in `get_fee_recipient_defaulting`, + /// `get_gas_limit_defaulting`, and `get_builder_proposals_defaulting`. + fn proposal_data(&self, pubkey: &PublicKeyBytes) -> Option { + self.validators + .read() + .validator(pubkey) + .map(|validator| ProposalData { + validator_index: validator.get_index(), + fee_recipient: self + .get_fee_recipient_defaulting(validator.get_suggested_fee_recipient()), + gas_limit: self.get_gas_limit_defaulting(validator.get_gas_limit()), + builder_proposals: self + .get_builder_proposals_defaulting(validator.get_builder_proposals()), + }) + } +} diff --git a/validator_client/signing_method/src/lib.rs b/validator_client/signing_method/src/lib.rs index f3b62c9500b..316c1d2205c 100644 --- a/validator_client/signing_method/src/lib.rs +++ b/validator_client/signing_method/src/lib.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use task_executor::TaskExecutor; use types::*; use url::Url; -use web3signer::{ForkInfo, SigningRequest, SigningResponse}; +use web3signer::{ForkInfo, MessageType, SigningRequest, SigningResponse}; pub use web3signer::Web3SignerObject; @@ -152,8 +152,13 @@ impl SigningMethod { genesis_validators_root, }); - self.get_signature_from_root(signable_message, signing_root, executor, fork_info) - .await + self.get_signature_from_root::( + signable_message, + signing_root, + executor, + fork_info, + ) + .await } pub async fn get_signature_from_root>( @@ -227,11 +232,7 @@ impl SigningMethod { // Determine the Web3Signer message type. let message_type = object.message_type(); - - if matches!( - object, - Web3SignerObject::Deposit { .. } | Web3SignerObject::ValidatorRegistration(_) - ) && fork_info.is_some() + if matches!(message_type, MessageType::ValidatorRegistration) && fork_info.is_some() { return Err(Error::GenesisForkVersionRequired); } diff --git a/validator_client/slashing_protection/src/lib.rs b/validator_client/slashing_protection/src/lib.rs index 51dd3e31642..825a34cabc7 100644 --- a/validator_client/slashing_protection/src/lib.rs +++ b/validator_client/slashing_protection/src/lib.rs @@ -27,7 +27,7 @@ pub const SLASHING_PROTECTION_FILENAME: &str = "slashing_protection.sqlite"; /// The attestation or block is not safe to sign. /// /// This could be because it's slashable, or because an error occurred. -#[derive(PartialEq, Debug)] +#[derive(PartialEq, Debug, Clone)] pub enum NotSafe { UnregisteredValidator(PublicKeyBytes), DisabledValidator(PublicKeyBytes), diff --git a/validator_client/slashing_protection/src/signed_attestation.rs b/validator_client/slashing_protection/src/signed_attestation.rs index 779b5f770aa..332f80c7045 100644 --- a/validator_client/slashing_protection/src/signed_attestation.rs +++ b/validator_client/slashing_protection/src/signed_attestation.rs @@ -10,7 +10,7 @@ pub struct SignedAttestation { } /// Reasons why an attestation may be slashable (or invalid). -#[derive(PartialEq, Debug)] +#[derive(PartialEq, Debug, Clone)] pub enum InvalidAttestation { /// The attestation has the same target epoch as an attestation from the DB (enclosed). DoubleVote(SignedAttestation), diff --git a/validator_client/slashing_protection/src/signed_block.rs b/validator_client/slashing_protection/src/signed_block.rs index 92ec2dcbe87..d46872529e9 100644 --- a/validator_client/slashing_protection/src/signed_block.rs +++ b/validator_client/slashing_protection/src/signed_block.rs @@ -9,7 +9,7 @@ pub struct SignedBlock { } /// Reasons why a block may be slashable. -#[derive(PartialEq, Debug)] +#[derive(PartialEq, Debug, Clone)] pub enum InvalidBlock { DoubleBlockProposal(SignedBlock), SlotViolatesLowerBound { block_slot: Slot, bound_slot: Slot }, diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index bb72ef81c80..5ea4c23bc4a 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -8,6 +8,7 @@ use directory::{ use eth2::types::Graffiti; use graffiti_file::GraffitiFile; use initialized_validators::Config as InitializedValidatorsConfig; +use lighthouse_validator_store::Config as ValidatorStoreConfig; use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; use slog::{info, warn, Logger}; @@ -19,7 +20,6 @@ use std::time::Duration; use types::{Address, GRAFFITI_BYTES_LEN}; use validator_http_api::{self, PK_FILENAME}; use validator_http_metrics; -use validator_store::Config as ValidatorStoreConfig; pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/"; diff --git a/validator_client/src/latency.rs b/validator_client/src/latency.rs index 22f02c7c0bc..e2a80876ec7 100644 --- a/validator_client/src/latency.rs +++ b/validator_client/src/latency.rs @@ -15,7 +15,7 @@ pub const SLOT_DELAY_DENOMINATOR: u32 = 12; pub fn start_latency_service( context: RuntimeContext, slot_clock: T, - beacon_nodes: Arc>, + beacon_nodes: Arc>, ) { let log = context.log().clone(); diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 8ebfe98b15e..5adfdf3349a 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -21,6 +21,7 @@ use doppelganger_service::DoppelgangerService; use environment::RuntimeContext; use eth2::{reqwest::ClientBuilder, BeaconNodeHttpClient, StatusCode, Timeouts}; use initialized_validators::Error::UnableToOpenVotingKeystore; +use lighthouse_validator_store::LighthouseValidatorStore; use notifier::spawn_notifier; use parking_lot::RwLock; use reqwest::Certificate; @@ -29,7 +30,6 @@ use slot_clock::SlotClock; use slot_clock::SystemTimeSlotClock; use std::fs::File; use std::io::Read; -use std::marker::PhantomData; use std::net::SocketAddr; use std::path::Path; use std::sync::Arc; @@ -43,12 +43,11 @@ use validator_http_api::ApiSecret; use validator_services::{ attestation_service::{AttestationService, AttestationServiceBuilder}, block_service::{BlockService, BlockServiceBuilder}, - duties_service::{self, DutiesService}, + duties_service::{self, DutiesService, DutiesServiceBuilder}, preparation_service::{PreparationService, PreparationServiceBuilder}, - sync::SyncDutiesMap, sync_committee_service::SyncCommitteeService, }; -use validator_store::ValidatorStore; +use validator_store::ValidatorStore as ValidatorStoreTrait; /// The interval between attempts to contact the beacon node during startup. const RETRY_DELAY: Duration = Duration::from_secs(2); @@ -73,20 +72,22 @@ const HTTP_GET_VALIDATOR_BLOCK_TIMEOUT_QUOTIENT: u32 = 4; const DOPPELGANGER_SERVICE_NAME: &str = "doppelganger"; +type ValidatorStore = LighthouseValidatorStore; + #[derive(Clone)] pub struct ProductionValidatorClient { context: RuntimeContext, - duties_service: Arc>, - block_service: BlockService, - attestation_service: AttestationService, - sync_committee_service: SyncCommitteeService, + duties_service: Arc, SystemTimeSlotClock>>, + block_service: BlockService, SystemTimeSlotClock>, + attestation_service: AttestationService, SystemTimeSlotClock>, + sync_committee_service: SyncCommitteeService, SystemTimeSlotClock>, doppelganger_service: Option>, - preparation_service: PreparationService, - validator_store: Arc>, + preparation_service: PreparationService, SystemTimeSlotClock>, + validator_store: Arc>, slot_clock: SystemTimeSlotClock, http_api_listen_addr: Option, config: Config, - beacon_nodes: Arc>, + beacon_nodes: Arc>, genesis_time: u64, } @@ -152,7 +153,7 @@ impl ProductionValidatorClient { let exit = context.executor.exit(); - let (_listen_addr, server) = validator_http_metrics::serve(ctx.clone(), exit) + let (_listen_addr, server) = validator_http_metrics::serve::(ctx.clone(), exit) .map_err(|e| format!("Unable to start metrics API server: {:?}", e))?; context @@ -384,20 +385,18 @@ impl ProductionValidatorClient { // Initialize the number of connected, avaliable beacon nodes to 0. set_gauge(&validator_metrics::AVAILABLE_BEACON_NODES_COUNT, 0); - let mut beacon_nodes: BeaconNodeFallback<_, E> = BeaconNodeFallback::new( + let mut beacon_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( candidates, config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), - log.clone(), ); - let mut proposer_nodes: BeaconNodeFallback<_, E> = BeaconNodeFallback::new( + let mut proposer_nodes: BeaconNodeFallback<_> = BeaconNodeFallback::new( proposer_candidates, config.beacon_node_fallback, config.broadcast_topics.clone(), context.eth2_config.spec.clone(), - log.clone(), ); // Perform some potentially long-running initialization tasks. @@ -421,10 +420,10 @@ impl ProductionValidatorClient { proposer_nodes.set_slot_clock(slot_clock.clone()); let beacon_nodes = Arc::new(beacon_nodes); - start_fallback_updater_service(context.clone(), beacon_nodes.clone())?; + start_fallback_updater_service::<_, E>(context.executor.clone(), beacon_nodes.clone())?; let proposer_nodes = Arc::new(proposer_nodes); - start_fallback_updater_service(context.clone(), proposer_nodes.clone())?; + start_fallback_updater_service::<_, E>(context.executor.clone(), proposer_nodes.clone())?; let doppelganger_service = if config.enable_doppelganger_protection { Some(Arc::new(DoppelgangerService::new( @@ -437,7 +436,7 @@ impl ProductionValidatorClient { None }; - let validator_store = Arc::new(ValidatorStore::new( + let validator_store = Arc::new(LighthouseValidatorStore::new( validators, slashing_protection, genesis_validators_root, @@ -465,20 +464,17 @@ impl ProductionValidatorClient { validator_store.prune_slashing_protection_db(slot.epoch(E::slots_per_epoch()), true); } - let duties_context = context.service_context("duties".into()); - let duties_service = Arc::new(DutiesService { - attesters: <_>::default(), - proposers: <_>::default(), - sync_duties: SyncDutiesMap::new(config.distributed), - slot_clock: slot_clock.clone(), - beacon_nodes: beacon_nodes.clone(), - validator_store: validator_store.clone(), - unknown_validator_next_poll_slots: <_>::default(), - spec: context.eth2_config.spec.clone(), - context: duties_context, - enable_high_validator_count_metrics: config.enable_high_validator_count_metrics, - distributed: config.distributed, - }); + let duties_service = Arc::new( + DutiesServiceBuilder::new() + .slot_clock(slot_clock.clone()) + .beacon_nodes(beacon_nodes.clone()) + .validator_store(validator_store.clone()) + .spec(context.eth2_config.spec.clone()) + .executor(context.executor.clone()) + .enable_high_validator_count_metrics(config.enable_high_validator_count_metrics) + .distributed(config.distributed) + .build()?, + ); // Update the metrics server. if let Some(ctx) = &validator_metrics_ctx { @@ -490,7 +486,7 @@ impl ProductionValidatorClient { .slot_clock(slot_clock.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) - .runtime_context(context.service_context("block".into())) + .executor(context.executor.clone()) .graffiti(config.graffiti) .graffiti_file(config.graffiti_file.clone()); @@ -506,14 +502,15 @@ impl ProductionValidatorClient { .slot_clock(slot_clock.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) - .runtime_context(context.service_context("attestation".into())) + .executor(context.executor.clone()) + .chain_spec(context.eth2_config.spec.clone()) .build()?; let preparation_service = PreparationServiceBuilder::new() .slot_clock(slot_clock.clone()) .validator_store(validator_store.clone()) .beacon_nodes(beacon_nodes.clone()) - .runtime_context(context.service_context("preparation".into())) + .executor(context.executor.clone()) .builder_registration_timestamp_override(config.builder_registration_timestamp_override) .validator_registration_batch_size(config.validator_registration_batch_size) .build()?; @@ -523,7 +520,7 @@ impl ProductionValidatorClient { validator_store.clone(), slot_clock.clone(), beacon_nodes.clone(), - context.service_context("sync_committee".into()), + context.executor.clone(), ); Ok(Self { @@ -568,12 +565,11 @@ impl ProductionValidatorClient { sse_logging_components: self.context.sse_logging_components.clone(), slot_clock: self.slot_clock.clone(), log: log.clone(), - _phantom: PhantomData, }); let exit = self.context.executor.exit(); - let (listen_addr, server) = validator_http_api::serve(ctx, exit) + let (listen_addr, server) = validator_http_api::serve::<_, E>(ctx, exit) .map_err(|e| format!("Unable to start HTTP API server: {:?}", e))?; self.context @@ -641,13 +637,13 @@ impl ProductionValidatorClient { } async fn init_from_beacon_node( - beacon_nodes: &BeaconNodeFallback, - proposer_nodes: &BeaconNodeFallback, + beacon_nodes: &BeaconNodeFallback, + proposer_nodes: &BeaconNodeFallback, context: &RuntimeContext, ) -> Result<(u64, Hash256), String> { loop { - beacon_nodes.update_all_candidates().await; - proposer_nodes.update_all_candidates().await; + beacon_nodes.update_all_candidates::().await; + proposer_nodes.update_all_candidates::().await; let num_available = beacon_nodes.num_available().await; let num_total = beacon_nodes.num_total().await; @@ -733,7 +729,7 @@ async fn init_from_beacon_node( } async fn wait_for_genesis( - beacon_nodes: &BeaconNodeFallback, + beacon_nodes: &BeaconNodeFallback, genesis_time: u64, context: &RuntimeContext, ) -> Result<(), String> { @@ -779,8 +775,8 @@ async fn wait_for_genesis( /// Request the version from the node, looping back and trying again on failure. Exit once the node /// has been contacted. -async fn poll_whilst_waiting_for_genesis( - beacon_nodes: &BeaconNodeFallback, +async fn poll_whilst_waiting_for_genesis( + beacon_nodes: &BeaconNodeFallback, genesis_time: Duration, log: &Logger, ) -> Result<(), String> { diff --git a/validator_client/src/notifier.rs b/validator_client/src/notifier.rs index ff66517795b..c0d73cc45c2 100644 --- a/validator_client/src/notifier.rs +++ b/validator_client/src/notifier.rs @@ -1,4 +1,5 @@ use crate::{DutiesService, ProductionValidatorClient}; +use lighthouse_validator_store::LighthouseValidatorStore; use metrics::set_gauge; use slog::{debug, error, info, Logger}; use slot_clock::SlotClock; @@ -19,7 +20,7 @@ pub fn spawn_notifier(client: &ProductionValidatorClient) -> Resu loop { if let Some(duration_to_next_slot) = duties_service.slot_clock.duration_to_next_slot() { sleep(duration_to_next_slot + slot_duration / 2).await; - notify(&duties_service, log).await; + notify::<_, E>(&duties_service, log).await; } else { error!(log, "Failed to read slot clock"); // If we can't read the slot clock, just wait another slot. @@ -35,7 +36,7 @@ pub fn spawn_notifier(client: &ProductionValidatorClient) -> Resu /// Performs a single notification routine. async fn notify( - duties_service: &DutiesService, + duties_service: &DutiesService, T>, log: &Logger, ) { let (candidate_info, num_available, num_synced) = diff --git a/validator_client/validator_services/Cargo.toml b/validator_client/validator_services/Cargo.toml index 21f0ae2d776..278370e79bd 100644 --- a/validator_client/validator_services/Cargo.toml +++ b/validator_client/validator_services/Cargo.toml @@ -6,17 +6,17 @@ authors = ["Sigma Prime "] [dependencies] beacon_node_fallback = { workspace = true } -bls = { workspace = true } -doppelganger_service = { workspace = true } -environment = { workspace = true } +bls = { workspace = true } eth2 = { workspace = true } futures = { workspace = true } graffiti_file = { workspace = true } +logging = { workspace = true } parking_lot = { workspace = true } safe_arith = { workspace = true } -slog = { workspace = true } slot_clock = { workspace = true } +task_executor = { workspace = true } tokio = { workspace = true } +tracing = { workspace = true } tree_hash = { workspace = true } types = { workspace = true } validator_metrics = { workspace = true } diff --git a/validator_client/validator_services/src/attestation_service.rs b/validator_client/validator_services/src/attestation_service.rs index e31ad4f661b..7f6e5657b16 100644 --- a/validator_client/validator_services/src/attestation_service.rs +++ b/validator_client/validator_services/src/attestation_service.rs @@ -1,44 +1,47 @@ use crate::duties_service::{DutiesService, DutyAndProof}; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use environment::RuntimeContext; use futures::future::join_all; -use slog::{crit, debug, error, info, trace, warn}; +use logging::crit; use slot_clock::SlotClock; use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; +use task_executor::TaskExecutor; use tokio::time::{sleep, sleep_until, Duration, Instant}; +use tracing::{debug, error, info, trace, warn}; use tree_hash::TreeHash; use types::{Attestation, AttestationData, ChainSpec, CommitteeIndex, EthSpec, Slot}; use validator_store::{Error as ValidatorStoreError, ValidatorStore}; /// Builds an `AttestationService`. #[derive(Default)] -pub struct AttestationServiceBuilder { - duties_service: Option>>, - validator_store: Option>>, +pub struct AttestationServiceBuilder { + duties_service: Option>>, + validator_store: Option>, slot_clock: Option, - beacon_nodes: Option>>, - context: Option>, + beacon_nodes: Option>>, + executor: Option, + chain_spec: Option>, } -impl AttestationServiceBuilder { +impl AttestationServiceBuilder { pub fn new() -> Self { Self { duties_service: None, validator_store: None, slot_clock: None, beacon_nodes: None, - context: None, + executor: None, + chain_spec: None, } } - pub fn duties_service(mut self, service: Arc>) -> Self { + pub fn duties_service(mut self, service: Arc>) -> Self { self.duties_service = Some(service); self } - pub fn validator_store(mut self, store: Arc>) -> Self { + pub fn validator_store(mut self, store: Arc) -> Self { self.validator_store = Some(store); self } @@ -48,17 +51,22 @@ impl AttestationServiceBuilder { self } - pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { + pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { self.beacon_nodes = Some(beacon_nodes); self } - pub fn runtime_context(mut self, context: RuntimeContext) -> Self { - self.context = Some(context); + pub fn executor(mut self, executor: TaskExecutor) -> Self { + self.executor = Some(executor); self } - pub fn build(self) -> Result, String> { + pub fn chain_spec(mut self, chain_spec: Arc) -> Self { + self.chain_spec = Some(chain_spec); + self + } + + pub fn build(self) -> Result, String> { Ok(AttestationService { inner: Arc::new(Inner { duties_service: self @@ -73,21 +81,25 @@ impl AttestationServiceBuilder { beacon_nodes: self .beacon_nodes .ok_or("Cannot build AttestationService without beacon_nodes")?, - context: self - .context - .ok_or("Cannot build AttestationService without runtime_context")?, + executor: self + .executor + .ok_or("Cannot build AttestationService without executor")?, + chain_spec: self + .chain_spec + .ok_or("Cannot build AttestationService without chain_spec")?, }), }) } } /// Helper to minimise `Arc` usage. -pub struct Inner { - duties_service: Arc>, - validator_store: Arc>, +pub struct Inner { + duties_service: Arc>, + validator_store: Arc, slot_clock: T, - beacon_nodes: Arc>, - context: RuntimeContext, + beacon_nodes: Arc>, + executor: TaskExecutor, + chain_spec: Arc, } /// Attempts to produce attestations for all known validators 1/3rd of the way through each slot. @@ -95,11 +107,11 @@ pub struct Inner { /// If any validators are on the same committee, a single attestation will be downloaded and /// returned to the beacon node. This attestation will have a signature from each of the /// validators. -pub struct AttestationService { - inner: Arc>, +pub struct AttestationService { + inner: Arc>, } -impl Clone for AttestationService { +impl Clone for AttestationService { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -107,19 +119,17 @@ impl Clone for AttestationService { } } -impl Deref for AttestationService { - type Target = Inner; +impl Deref for AttestationService { + type Target = Inner; fn deref(&self) -> &Self::Target { self.inner.deref() } } -impl AttestationService { +impl AttestationService { /// Starts the service which periodically produces attestations. pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> { - let log = self.context.log().clone(); - let slot_duration = Duration::from_secs(spec.seconds_per_slot); let duration_to_next_slot = self .slot_clock @@ -127,33 +137,24 @@ impl AttestationService { .ok_or("Unable to determine duration to next slot")?; info!( - log, - "Attestation production service started"; - "next_update_millis" => duration_to_next_slot.as_millis() + next_update_millis = duration_to_next_slot.as_millis(), + "Attestation production service started" ); - let executor = self.context.executor.clone(); + let executor = self.executor.clone(); let interval_fut = async move { loop { if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() { sleep(duration_to_next_slot + slot_duration / 3).await; - let log = self.context.log(); if let Err(e) = self.spawn_attestation_tasks(slot_duration) { - crit!( - log, - "Failed to spawn attestation tasks"; - "error" => e - ) + crit!(error = e, "Failed to spawn attestation tasks") } else { - trace!( - log, - "Spawned attestation tasks"; - ) + trace!("Spawned attestation tasks"); } } else { - error!(log, "Failed to read slot clock"); + error!("Failed to read slot clock"); // If we can't read the slot clock, just wait another slot. sleep(slot_duration).await; continue; @@ -200,7 +201,7 @@ impl AttestationService { .into_iter() .for_each(|(committee_index, validator_duties)| { // Spawn a separate task for each attestation. - self.inner.context.executor.spawn_ignoring_error( + self.inner.executor.spawn_ignoring_error( self.clone().publish_attestations_and_aggregates( slot, committee_index, @@ -235,7 +236,6 @@ impl AttestationService { validator_duties: Vec, aggregate_production_instant: Instant, ) -> Result<(), ()> { - let log = self.context.log(); let attestations_timer = validator_metrics::start_timer_vec( &validator_metrics::ATTESTATION_SERVICE_TIMES, &[validator_metrics::ATTESTATIONS], @@ -255,11 +255,10 @@ impl AttestationService { .await .map_err(move |e| { crit!( - log, - "Error during attestation routine"; - "error" => format!("{:?}", e), - "committee_index" => committee_index, - "slot" => slot.as_u64(), + error = format!("{:?}", e), + committee_index, + slot = slot.as_u64(), + "Error during attestation routine" ) })?; @@ -292,11 +291,10 @@ impl AttestationService { .await .map_err(move |e| { crit!( - log, - "Error during attestation routine"; - "error" => format!("{:?}", e), - "committee_index" => committee_index, - "slot" => slot.as_u64(), + error = format!("{:?}", e), + committee_index, + slot = slot.as_u64(), + "Error during attestation routine" ) })?; } @@ -322,8 +320,6 @@ impl AttestationService { committee_index: CommitteeIndex, validator_duties: &[DutyAndProof], ) -> Result, String> { - let log = self.context.log(); - if validator_duties.is_empty() { return Ok(None); } @@ -332,7 +328,7 @@ impl AttestationService { .slot_clock .now() .ok_or("Unable to determine current slot from clock")? - .epoch(E::slots_per_epoch()); + .epoch(S::E::slots_per_epoch()); let attestation_data = self .beacon_nodes @@ -357,36 +353,34 @@ impl AttestationService { let attestation_data = attestation_data_ref; // Ensure that the attestation matches the duties. - if !duty.match_attestation_data::(attestation_data, &self.context.eth2_config.spec) { + if !duty.match_attestation_data::(attestation_data, &self.chain_spec) { crit!( - log, - "Inconsistent validator duties during signing"; - "validator" => ?duty.pubkey, - "duty_slot" => duty.slot, - "attestation_slot" => attestation_data.slot, - "duty_index" => duty.committee_index, - "attestation_index" => attestation_data.index, + validator = ?duty.pubkey, + duty_slot = ?duty.slot, + attestation_slot = %attestation_data.slot, + duty_index = duty.committee_index, + attestation_index = attestation_data.index, + "Inconsistent validator duties during signing" ); return None; } - let mut attestation = match Attestation::::empty_for_signing( + let mut attestation = match Attestation::empty_for_signing( duty.committee_index, duty.committee_length as usize, attestation_data.slot, attestation_data.beacon_block_root, attestation_data.source, attestation_data.target, - &self.context.eth2_config.spec, + &self.chain_spec, ) { Ok(attestation) => attestation, Err(err) => { crit!( - log, - "Invalid validator duties during signing"; - "validator" => ?duty.pubkey, - "duty" => ?duty, - "err" => ?err, + validator = ?duty.pubkey, + ?duty, + ?err, + "Invalid validator duties during signing" ); return None; } @@ -407,24 +401,22 @@ impl AttestationService { // A pubkey can be missing when a validator was recently // removed via the API. warn!( - log, - "Missing pubkey for attestation"; - "info" => "a validator may have recently been removed from this VC", - "pubkey" => ?pubkey, - "validator" => ?duty.pubkey, - "committee_index" => committee_index, - "slot" => slot.as_u64(), + info = "a validator may have recently been removed from this VC", + pubkey = ?pubkey, + validator = ?duty.pubkey, + committee_index = committee_index, + slot = slot.as_u64(), + "Missing pubkey for attestation" ); None } Err(e) => { crit!( - log, - "Failed to sign attestation"; - "error" => ?e, - "validator" => ?duty.pubkey, - "committee_index" => committee_index, - "slot" => slot.as_u64(), + error = ?e, + validator = ?duty.pubkey, + committee_index, + slot = slot.as_u64(), + "Failed to sign attestation" ); None } @@ -439,14 +431,12 @@ impl AttestationService { .unzip(); if attestations.is_empty() { - warn!(log, "No attestations were published"); + warn!("No attestations were published"); return Ok(None); } let fork_name = self - .context - .eth2_config - .spec - .fork_name_at_slot::(attestation_data.slot); + .chain_spec + .fork_name_at_slot::(attestation_data.slot); // Post the attestations to the BN. match self @@ -469,22 +459,20 @@ impl AttestationService { .await { Ok(()) => info!( - log, - "Successfully published attestations"; - "count" => attestations.len(), - "validator_indices" => ?validator_indices, - "head_block" => ?attestation_data.beacon_block_root, - "committee_index" => attestation_data.index, - "slot" => attestation_data.slot.as_u64(), - "type" => "unaggregated", + count = attestations.len(), + validator_indices = ?validator_indices, + head_block = ?attestation_data.beacon_block_root, + committee_index = attestation_data.index, + slot = attestation_data.slot.as_u64(), + "type" = "unaggregated", + "Successfully published attestations" ), Err(e) => error!( - log, - "Unable to publish attestations"; - "error" => %e, - "committee_index" => attestation_data.index, - "slot" => slot.as_u64(), - "type" => "unaggregated", + error = %e, + committee_index = attestation_data.index, + slot = slot.as_u64(), + "type" = "unaggregated", + "Unable to publish attestations" ), } @@ -510,8 +498,6 @@ impl AttestationService { committee_index: CommitteeIndex, validator_duties: &[DutyAndProof], ) -> Result<(), String> { - let log = self.context.log(); - if !validator_duties .iter() .any(|duty_and_proof| duty_and_proof.selection_proof.is_some()) @@ -521,10 +507,8 @@ impl AttestationService { } let fork_name = self - .context - .eth2_config - .spec - .fork_name_at_slot::(attestation_data.slot); + .chain_spec + .fork_name_at_slot::(attestation_data.slot); let aggregated_attestation = &self .beacon_nodes @@ -568,8 +552,8 @@ impl AttestationService { let duty = &duty_and_proof.duty; let selection_proof = duty_and_proof.selection_proof.as_ref()?; - if !duty.match_attestation_data::(attestation_data, &self.context.eth2_config.spec) { - crit!(log, "Inconsistent validator duties during signing"); + if !duty.match_attestation_data::(attestation_data, &self.chain_spec) { + crit!("Inconsistent validator duties during signing"); return None; } @@ -587,19 +571,14 @@ impl AttestationService { Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { // A pubkey can be missing when a validator was recently // removed via the API. - debug!( - log, - "Missing pubkey for aggregate"; - "pubkey" => ?pubkey, - ); + debug!(?pubkey, "Missing pubkey for aggregate"); None } Err(e) => { crit!( - log, - "Failed to sign aggregate"; - "error" => ?e, - "pubkey" => ?duty.pubkey, + error = ?e, + pubkey = ?duty.pubkey, + "Failed to sign aggregate" ); None } @@ -643,14 +622,13 @@ impl AttestationService { for signed_aggregate_and_proof in signed_aggregate_and_proofs { let attestation = signed_aggregate_and_proof.message().aggregate(); info!( - log, - "Successfully published attestation"; - "aggregator" => signed_aggregate_and_proof.message().aggregator_index(), - "signatures" => attestation.num_set_aggregation_bits(), - "head_block" => format!("{:?}", attestation.data().beacon_block_root), - "committee_index" => attestation.committee_index(), - "slot" => attestation.data().slot.as_u64(), - "type" => "aggregated", + aggregator = signed_aggregate_and_proof.message().aggregator_index(), + signatures = attestation.num_set_aggregation_bits(), + head_block = format!("{:?}", attestation.data().beacon_block_root), + committee_index = attestation.committee_index(), + slot = attestation.data().slot.as_u64(), + "type" = "aggregated", + "Successfully published attestation" ); } } @@ -658,13 +636,12 @@ impl AttestationService { for signed_aggregate_and_proof in signed_aggregate_and_proofs { let attestation = &signed_aggregate_and_proof.message().aggregate(); crit!( - log, - "Failed to publish attestation"; - "error" => %e, - "aggregator" => signed_aggregate_and_proof.message().aggregator_index(), - "committee_index" => attestation.committee_index(), - "slot" => attestation.data().slot.as_u64(), - "type" => "aggregated", + error = %e, + aggregator = signed_aggregate_and_proof.message().aggregator_index(), + committee_index = attestation.committee_index(), + slot = attestation.data().slot.as_u64(), + "type" = "aggregated", + "Failed to publish attestation" ); } } @@ -679,11 +656,11 @@ impl AttestationService { /// Start the task at `pruning_instant` to avoid interference with other tasks. fn spawn_slashing_protection_pruning_task(&self, slot: Slot, pruning_instant: Instant) { let attestation_service = self.clone(); - let executor = self.inner.context.executor.clone(); - let current_epoch = slot.epoch(E::slots_per_epoch()); + let executor = self.inner.executor.clone(); + let current_epoch = slot.epoch(S::E::slots_per_epoch()); // Wait for `pruning_instant` in a regular task, and then switch to a blocking one. - self.inner.context.executor.spawn( + self.inner.executor.spawn( async move { sleep_until(pruning_instant).await; diff --git a/validator_client/validator_services/src/block_service.rs b/validator_client/validator_services/src/block_service.rs index 60eb0361ad3..7843c3ec3fa 100644 --- a/validator_client/validator_services/src/block_service.rs +++ b/validator_client/validator_services/src/block_service.rs @@ -1,20 +1,21 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback, Error as FallbackError, Errors}; use bls::SignatureBytes; -use environment::RuntimeContext; use eth2::types::{FullBlockContents, PublishBlockRequest}; use eth2::{BeaconNodeHttpClient, StatusCode}; use graffiti_file::{determine_graffiti, GraffitiFile}; -use slog::{crit, debug, error, info, trace, warn, Logger}; +use logging::crit; use slot_clock::SlotClock; use std::fmt::Debug; use std::future::Future; use std::ops::Deref; use std::sync::Arc; use std::time::Duration; +use task_executor::TaskExecutor; use tokio::sync::mpsc; +use tracing::{debug, error, info, trace, warn}; use types::{ - BlindedBeaconBlock, BlockType, EthSpec, Graffiti, PublicKeyBytes, SignedBlindedBeaconBlock, - Slot, + BlindedBeaconBlock, BlockType, ChainSpec, EthSpec, Graffiti, PublicKeyBytes, + SignedBlindedBeaconBlock, Slot, }; use validator_store::{Error as ValidatorStoreError, ValidatorStore}; @@ -44,30 +45,32 @@ impl From> for BlockError { /// Builds a `BlockService`. #[derive(Default)] -pub struct BlockServiceBuilder { - validator_store: Option>>, +pub struct BlockServiceBuilder { + validator_store: Option>, slot_clock: Option>, - beacon_nodes: Option>>, - proposer_nodes: Option>>, - context: Option>, + beacon_nodes: Option>>, + proposer_nodes: Option>>, + executor: Option, + chain_spec: Option>, graffiti: Option, graffiti_file: Option, } -impl BlockServiceBuilder { +impl BlockServiceBuilder { pub fn new() -> Self { Self { validator_store: None, slot_clock: None, beacon_nodes: None, proposer_nodes: None, - context: None, + executor: None, + chain_spec: None, graffiti: None, graffiti_file: None, } } - pub fn validator_store(mut self, store: Arc>) -> Self { + pub fn validator_store(mut self, store: Arc) -> Self { self.validator_store = Some(store); self } @@ -77,18 +80,23 @@ impl BlockServiceBuilder { self } - pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { + pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { self.beacon_nodes = Some(beacon_nodes); self } - pub fn proposer_nodes(mut self, proposer_nodes: Arc>) -> Self { + pub fn proposer_nodes(mut self, proposer_nodes: Arc>) -> Self { self.proposer_nodes = Some(proposer_nodes); self } - pub fn runtime_context(mut self, context: RuntimeContext) -> Self { - self.context = Some(context); + pub fn executor(mut self, executor: TaskExecutor) -> Self { + self.executor = Some(executor); + self + } + + pub fn chain_spec(mut self, chain_spec: Arc) -> Self { + self.chain_spec = Some(chain_spec); self } @@ -102,7 +110,7 @@ impl BlockServiceBuilder { self } - pub fn build(self) -> Result, String> { + pub fn build(self) -> Result, String> { Ok(BlockService { inner: Arc::new(Inner { validator_store: self @@ -114,9 +122,12 @@ impl BlockServiceBuilder { beacon_nodes: self .beacon_nodes .ok_or("Cannot build BlockService without beacon_node")?, - context: self - .context - .ok_or("Cannot build BlockService without runtime_context")?, + executor: self + .executor + .ok_or("Cannot build BlockService without executor")?, + chain_spec: self + .chain_spec + .ok_or("Cannot build BlockService without chain_spec")?, proposer_nodes: self.proposer_nodes, graffiti: self.graffiti, graffiti_file: self.graffiti_file, @@ -127,12 +138,12 @@ impl BlockServiceBuilder { // Combines a set of non-block-proposing `beacon_nodes` and only-block-proposing // `proposer_nodes`. -pub struct ProposerFallback { - beacon_nodes: Arc>, - proposer_nodes: Option>>, +pub struct ProposerFallback { + beacon_nodes: Arc>, + proposer_nodes: Option>>, } -impl ProposerFallback { +impl ProposerFallback { // Try `func` on `self.proposer_nodes` first. If that doesn't work, try `self.beacon_nodes`. pub async fn request_proposers_first(&self, func: F) -> Result<(), Errors> where @@ -177,22 +188,23 @@ impl ProposerFallback { } /// Helper to minimise `Arc` usage. -pub struct Inner { - validator_store: Arc>, +pub struct Inner { + validator_store: Arc, slot_clock: Arc, - pub beacon_nodes: Arc>, - pub proposer_nodes: Option>>, - context: RuntimeContext, + pub beacon_nodes: Arc>, + pub proposer_nodes: Option>>, + executor: TaskExecutor, + chain_spec: Arc, graffiti: Option, graffiti_file: Option, } /// Attempts to produce attestations for any block producer(s) at the start of the epoch. -pub struct BlockService { - inner: Arc>, +pub struct BlockService { + inner: Arc>, } -impl Clone for BlockService { +impl Clone for BlockService { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -200,8 +212,8 @@ impl Clone for BlockService { } } -impl Deref for BlockService { - type Target = Inner; +impl Deref for BlockService { + type Target = Inner; fn deref(&self) -> &Self::Target { self.inner.deref() @@ -214,23 +226,21 @@ pub struct BlockServiceNotification { pub block_proposers: Vec, } -impl BlockService { +impl BlockService { pub fn start_update_service( self, mut notification_rx: mpsc::Receiver, ) -> Result<(), String> { - let log = self.context.log().clone(); + info!("Block production service started"); - info!(log, "Block production service started"); - - let executor = self.inner.context.executor.clone(); + let executor = self.inner.executor.clone(); executor.spawn( async move { while let Some(notif) = notification_rx.recv().await { self.do_update(notif).await.ok(); } - debug!(log, "Block service shutting down"); + debug!("Block service shutting down"); }, "block_service", ); @@ -240,65 +250,55 @@ impl BlockService { /// Attempt to produce a block for any block producers in the `ValidatorStore`. async fn do_update(&self, notification: BlockServiceNotification) -> Result<(), ()> { - let log = self.context.log(); let _timer = validator_metrics::start_timer_vec( &validator_metrics::BLOCK_SERVICE_TIMES, &[validator_metrics::FULL_UPDATE], ); let slot = self.slot_clock.now().ok_or_else(move || { - crit!(log, "Duties manager failed to read slot clock"); + crit!("Duties manager failed to read slot clock"); })?; if notification.slot != slot { warn!( - log, - "Skipping block production for expired slot"; - "current_slot" => slot.as_u64(), - "notification_slot" => notification.slot.as_u64(), - "info" => "Your machine could be overloaded" + current_slot = slot.as_u64(), + notification_slot = notification.slot.as_u64(), + info = "Your machine could be overloaded", + "Skipping block production for expired slot" ); return Ok(()); } - if slot == self.context.eth2_config.spec.genesis_slot { + if slot == self.chain_spec.genesis_slot { debug!( - log, - "Not producing block at genesis slot"; - "proposers" => format!("{:?}", notification.block_proposers), + proposers = format!("{:?}", notification.block_proposers), + "Not producing block at genesis slot" ); return Ok(()); } - trace!( - log, - "Block service update started"; - "slot" => slot.as_u64() - ); + trace!(slot = slot.as_u64(), "Block service update started"); let proposers = notification.block_proposers; if proposers.is_empty() { trace!( - log, - "No local block proposers for this slot"; - "slot" => slot.as_u64() + slot = slot.as_u64(), + "No local block proposers for this slot" ) } else if proposers.len() > 1 { error!( - log, - "Multiple block proposers for this slot"; - "action" => "producing blocks for all proposers", - "num_proposers" => proposers.len(), - "slot" => slot.as_u64(), + action = "producing blocks for all proposers", + num_proposers = proposers.len(), + slot = slot.as_u64(), + "Multiple block proposers for this slot" ) } for validator_pubkey in proposers { let builder_boost_factor = self.get_builder_boost_factor(&validator_pubkey); let service = self.clone(); - let log = log.clone(); - self.inner.context.executor.spawn( + self.inner.executor.spawn( async move { let result = service .publish_block(slot, validator_pubkey, builder_boost_factor) @@ -308,11 +308,10 @@ impl BlockService { Ok(_) => {} Err(BlockError::Recoverable(e)) | Err(BlockError::Irrecoverable(e)) => { error!( - log, - "Error whilst producing block"; - "error" => ?e, - "block_slot" => ?slot, - "info" => "block v3 proposal failed, this error may or may not result in a missed block" + error = ?e, + block_slot = ?slot, + info = "block v3 proposal failed, this error may or may not result in a missed block", + "Error whilst producing block" ); } } @@ -326,42 +325,45 @@ impl BlockService { #[allow(clippy::too_many_arguments)] async fn sign_and_publish_block( &self, - proposer_fallback: ProposerFallback, + proposer_fallback: ProposerFallback, slot: Slot, graffiti: Option, validator_pubkey: &PublicKeyBytes, - unsigned_block: UnsignedBlock, + unsigned_block: UnsignedBlock, ) -> Result<(), BlockError> { - let log = self.context.log(); let signing_timer = validator_metrics::start_timer(&validator_metrics::BLOCK_SIGNING_TIMES); - let res = match unsigned_block { + let (block, maybe_blobs) = match unsigned_block { UnsignedBlock::Full(block_contents) => { let (block, maybe_blobs) = block_contents.deconstruct(); - self.validator_store - .sign_block(*validator_pubkey, block, slot) - .await - .map(|b| SignedBlock::Full(PublishBlockRequest::new(Arc::new(b), maybe_blobs))) + (block.into(), maybe_blobs) } - UnsignedBlock::Blinded(block) => self - .validator_store - .sign_block(*validator_pubkey, block, slot) - .await - .map(Arc::new) - .map(SignedBlock::Blinded), + UnsignedBlock::Blinded(block) => (block.into(), None), }; + let res = self + .validator_store + .sign_block(*validator_pubkey, block, slot) + .await + .map(|block| match block { + validator_store::SignedBlock::Full(block) => { + SignedBlock::Full(PublishBlockRequest::new(Arc::new(block), maybe_blobs)) + } + validator_store::SignedBlock::Blinded(block) => { + SignedBlock::Blinded(Arc::new(block)) + } + }); + let signed_block = match res { Ok(block) => block, Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { // A pubkey can be missing when a validator was recently removed // via the API. warn!( - log, - "Missing pubkey for block"; - "info" => "a validator may have recently been removed from this VC", - "pubkey" => ?pubkey, - "slot" => ?slot + info = "a validator may have recently been removed from this VC", + ?pubkey, + ?slot, + "Missing pubkey for block" ); return Ok(()); } @@ -377,10 +379,9 @@ impl BlockService { Duration::from_secs_f64(signing_timer.map_or(0.0, |t| t.stop_and_record())).as_millis(); info!( - log, - "Publishing signed block"; - "slot" => slot.as_u64(), - "signing_time_ms" => signing_time_ms, + slot = slot.as_u64(), + signing_time_ms = signing_time_ms, + "Publishing signed block" ); // Publish block with first available beacon node. @@ -396,13 +397,12 @@ impl BlockService { .await?; info!( - log, - "Successfully published block"; - "block_type" => ?signed_block.block_type(), - "deposits" => signed_block.num_deposits(), - "attestations" => signed_block.num_attestations(), - "graffiti" => ?graffiti.map(|g| g.as_utf8_lossy()), - "slot" => signed_block.slot().as_u64(), + block_type = ?signed_block.block_type(), + deposits = signed_block.num_deposits(), + attestations = signed_block.num_attestations(), + graffiti = ?graffiti.map(|g| g.as_utf8_lossy()), + slot = signed_block.slot().as_u64(), + "Successfully published block" ); Ok(()) } @@ -413,7 +413,6 @@ impl BlockService { validator_pubkey: PublicKeyBytes, builder_boost_factor: Option, ) -> Result<(), BlockError> { - let log = self.context.log(); let _timer = validator_metrics::start_timer_vec( &validator_metrics::BLOCK_SERVICE_TIMES, &[validator_metrics::BEACON_BLOCK], @@ -421,7 +420,7 @@ impl BlockService { let randao_reveal = match self .validator_store - .randao_reveal(validator_pubkey, slot.epoch(E::slots_per_epoch())) + .randao_reveal(validator_pubkey, slot.epoch(S::E::slots_per_epoch())) .await { Ok(signature) => signature.into(), @@ -429,11 +428,10 @@ impl BlockService { // A pubkey can be missing when a validator was recently removed // via the API. warn!( - log, - "Missing pubkey for block randao"; - "info" => "a validator may have recently been removed from this VC", - "pubkey" => ?pubkey, - "slot" => ?slot + info = "a validator may have recently been removed from this VC", + ?pubkey, + ?slot, + "Missing pubkey for block randao" ); return Ok(()); } @@ -447,7 +445,6 @@ impl BlockService { let graffiti = determine_graffiti( &validator_pubkey, - log, self.graffiti_file.clone(), self.validator_store.graffiti(&validator_pubkey), self.graffiti, @@ -461,11 +458,7 @@ impl BlockService { proposer_nodes: self.proposer_nodes.clone(), }; - info!( - log, - "Requesting unsigned block"; - "slot" => slot.as_u64(), - ); + info!(slot = slot.as_u64(), "Requesting unsigned block"); // Request block from first responsive beacon node. // @@ -484,7 +477,6 @@ impl BlockService { graffiti, proposer_index, builder_boost_factor, - log, ) .await .map_err(|e| { @@ -511,10 +503,9 @@ impl BlockService { async fn publish_signed_block_contents( &self, - signed_block: &SignedBlock, + signed_block: &SignedBlock, beacon_node: BeaconNodeHttpClient, ) -> Result<(), BlockError> { - let log = self.context.log(); let slot = signed_block.slot(); match signed_block { SignedBlock::Full(signed_block) => { @@ -525,7 +516,7 @@ impl BlockService { beacon_node .post_beacon_blocks_v2_ssz(signed_block, None) .await - .or_else(|e| handle_block_post_error(e, slot, log))? + .or_else(|e| handle_block_post_error(e, slot))? } SignedBlock::Blinded(signed_block) => { let _post_timer = validator_metrics::start_timer_vec( @@ -535,7 +526,7 @@ impl BlockService { beacon_node .post_beacon_blinded_blocks_v2_ssz(signed_block, None) .await - .or_else(|e| handle_block_post_error(e, slot, log))? + .or_else(|e| handle_block_post_error(e, slot))? } } Ok::<_, BlockError>(()) @@ -548,10 +539,9 @@ impl BlockService { graffiti: Option, proposer_index: Option, builder_boost_factor: Option, - log: &Logger, - ) -> Result, BlockError> { + ) -> Result, BlockError> { let (block_response, _) = beacon_node - .get_validator_blocks_v3::( + .get_validator_blocks_v3::( slot, randao_reveal_ref, graffiti.as_ref(), @@ -570,11 +560,7 @@ impl BlockService { eth2::types::ProduceBlockV3Response::Blinded(block) => UnsignedBlock::Blinded(block), }; - info!( - log, - "Received unsigned block"; - "slot" => slot.as_u64(), - ); + info!(slot = slot.as_u64(), "Received unsigned block"); if proposer_index != Some(unsigned_block.proposer_index()) { return Err(BlockError::Recoverable( "Proposer index does not match block proposer. Beacon chain re-orged".to_string(), @@ -593,15 +579,9 @@ impl BlockService { // Apply per validator configuration first. let validator_builder_boost_factor = self .validator_store - .determine_validator_builder_boost_factor(validator_pubkey); - - // Fallback to process-wide configuration if needed. - let maybe_builder_boost_factor = validator_builder_boost_factor.or_else(|| { - self.validator_store - .determine_default_builder_boost_factor() - }); + .determine_builder_boost_factor(validator_pubkey); - if let Some(builder_boost_factor) = maybe_builder_boost_factor { + if let Some(builder_boost_factor) = validator_builder_boost_factor { // if builder boost factor is set to 100 it should be treated // as None to prevent unnecessary calculations that could // lead to loss of information. @@ -662,23 +642,21 @@ impl SignedBlock { } } -fn handle_block_post_error(err: eth2::Error, slot: Slot, log: &Logger) -> Result<(), BlockError> { +fn handle_block_post_error(err: eth2::Error, slot: Slot) -> Result<(), BlockError> { // Handle non-200 success codes. if let Some(status) = err.status() { if status == StatusCode::ACCEPTED { info!( - log, - "Block is already known to BN or might be invalid"; - "slot" => slot, - "status_code" => status.as_u16(), + %slot, + status_code = status.as_u16(), + "Block is already known to BN or might be invalid" ); return Ok(()); } else if status.is_success() { debug!( - log, - "Block published with non-standard success code"; - "slot" => slot, - "status_code" => status.as_u16(), + %slot, + status_code = status.as_u16(), + "Block published with non-standard success code" ); return Ok(()); } diff --git a/validator_client/validator_services/src/duties_service.rs b/validator_client/validator_services/src/duties_service.rs index 187eb4feb50..cdbbd95a10f 100644 --- a/validator_client/validator_services/src/duties_service.rs +++ b/validator_client/validator_services/src/duties_service.rs @@ -10,25 +10,24 @@ use crate::block_service::BlockServiceNotification; use crate::sync::poll_sync_committee_duties; use crate::sync::SyncDutiesMap; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use doppelganger_service::DoppelgangerStatus; -use environment::RuntimeContext; use eth2::types::{ AttesterData, BeaconCommitteeSubscription, DutiesResponse, ProposerData, StateId, ValidatorId, }; use futures::{stream, StreamExt}; use parking_lot::RwLock; use safe_arith::{ArithError, SafeArith}; -use slog::{debug, error, info, warn, Logger}; use slot_clock::SlotClock; use std::cmp::min; use std::collections::{hash_map, BTreeMap, HashMap, HashSet}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; +use task_executor::TaskExecutor; use tokio::{sync::mpsc::Sender, time::sleep}; +use tracing::{debug, error, info, warn}; use types::{ChainSpec, Epoch, EthSpec, Hash256, PublicKeyBytes, SelectionProof, Slot}; use validator_metrics::{get_int_gauge, set_int_gauge, ATTESTATION_DUTY}; -use validator_store::{Error as ValidatorStoreError, ValidatorStore}; +use validator_store::{DoppelgangerStatus, Error as ValidatorStoreError, ValidatorStore}; /// Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch. const HISTORICAL_DUTIES_EPOCHS: u64 = 2; @@ -87,16 +86,16 @@ const _: () = assert!(ATTESTATION_SUBSCRIPTION_OFFSETS[0] > MIN_ATTESTATION_SUBS // The info in the enum variants is displayed in logging, clippy thinks it's dead code. #[derive(Debug)] -pub enum Error { +pub enum Error { UnableToReadSlotClock, FailedToDownloadAttesters(#[allow(dead_code)] String), - FailedToProduceSelectionProof(#[allow(dead_code)] ValidatorStoreError), + FailedToProduceSelectionProof(#[allow(dead_code)] ValidatorStoreError), InvalidModulo(#[allow(dead_code)] ArithError), Arith(#[allow(dead_code)] ArithError), SyncDutiesNotFound(#[allow(dead_code)] u64), } -impl From for Error { +impl From for Error { fn from(e: ArithError) -> Self { Self::Arith(e) } @@ -125,11 +124,11 @@ pub struct SubscriptionSlots { /// Create a selection proof for `duty`. /// /// Return `Ok(None)` if the attesting validator is not an aggregator. -async fn make_selection_proof( +async fn make_selection_proof( duty: &AttesterData, - validator_store: &ValidatorStore, + validator_store: &S, spec: &ChainSpec, -) -> Result, Error> { +) -> Result, Error> { let selection_proof = validator_store .produce_selection_proof(duty.pubkey, duty.slot) .await @@ -205,25 +204,124 @@ type DependentRoot = Hash256; type AttesterMap = HashMap>; type ProposerMap = HashMap)>; +pub struct DutiesServiceBuilder { + /// Provides the canonical list of locally-managed validators. + validator_store: Option>, + /// Tracks the current slot. + slot_clock: Option, + /// Provides HTTP access to remote beacon nodes. + beacon_nodes: Option>>, + /// The runtime for spawning tasks. + executor: Option, + /// The current chain spec. + spec: Option>, + //// Whether we permit large validator counts in the metrics. + enable_high_validator_count_metrics: bool, + /// If this validator is running in distributed mode. + distributed: bool, +} + +impl Default for DutiesServiceBuilder { + fn default() -> Self { + Self::new() + } +} + +impl DutiesServiceBuilder { + pub fn new() -> Self { + Self { + validator_store: None, + slot_clock: None, + beacon_nodes: None, + executor: None, + spec: None, + enable_high_validator_count_metrics: false, + distributed: false, + } + } + + pub fn validator_store(mut self, validator_store: Arc) -> Self { + self.validator_store = Some(validator_store); + self + } + + pub fn slot_clock(mut self, slot_clock: T) -> Self { + self.slot_clock = Some(slot_clock); + self + } + + pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { + self.beacon_nodes = Some(beacon_nodes); + self + } + + pub fn executor(mut self, executor: TaskExecutor) -> Self { + self.executor = Some(executor); + self + } + + pub fn spec(mut self, spec: Arc) -> Self { + self.spec = Some(spec); + self + } + + pub fn enable_high_validator_count_metrics( + mut self, + enable_high_validator_count_metrics: bool, + ) -> Self { + self.enable_high_validator_count_metrics = enable_high_validator_count_metrics; + self + } + + pub fn distributed(mut self, distributed: bool) -> Self { + self.distributed = distributed; + self + } + + pub fn build(self) -> Result, String> { + Ok(DutiesService { + attesters: Default::default(), + proposers: Default::default(), + sync_duties: SyncDutiesMap::new(self.distributed), + validator_store: self + .validator_store + .ok_or("Cannot build DutiesService without validator_store")?, + unknown_validator_next_poll_slots: Default::default(), + slot_clock: self + .slot_clock + .ok_or("Cannot build DutiesService without slot_clock")?, + beacon_nodes: self + .beacon_nodes + .ok_or("Cannot build DutiesService without beacon_nodes")?, + executor: self + .executor + .ok_or("Cannot build DutiesService without executor")?, + spec: self.spec.ok_or("Cannot build DutiesService without spec")?, + enable_high_validator_count_metrics: self.enable_high_validator_count_metrics, + distributed: self.distributed, + }) + } +} + /// See the module-level documentation. -pub struct DutiesService { +pub struct DutiesService { /// Maps a validator public key to their duties for each epoch. pub attesters: RwLock, /// Maps an epoch to all *local* proposers in this epoch. Notably, this does not contain /// proposals for any validators which are not registered locally. pub proposers: RwLock, /// Map from validator index to sync committee duties. - pub sync_duties: SyncDutiesMap, + pub sync_duties: SyncDutiesMap, /// Provides the canonical list of locally-managed validators. - pub validator_store: Arc>, + pub validator_store: Arc, /// Maps unknown validator pubkeys to the next slot time when a poll should be conducted again. pub unknown_validator_next_poll_slots: RwLock>, /// Tracks the current slot. pub slot_clock: T, /// Provides HTTP access to remote beacon nodes. - pub beacon_nodes: Arc>, + pub beacon_nodes: Arc>, /// The runtime for spawning tasks. - pub context: RuntimeContext, + pub executor: TaskExecutor, /// The current chain spec. pub spec: Arc, //// Whether we permit large validator counts in the metrics. @@ -232,7 +330,7 @@ pub struct DutiesService { pub distributed: bool, } -impl DutiesService { +impl DutiesService { /// Returns the total number of validators known to the duties service. pub fn total_validator_count(&self) -> usize { self.validator_store.num_voting_validators() @@ -282,7 +380,7 @@ impl DutiesService { /// /// It is possible that multiple validators have an identical proposal slot, however that is /// likely the result of heavy forking (lol) or inconsistent beacon node connections. - pub fn block_proposers(&self, slot: Slot) -> HashSet { + pub fn block_proposers(&self, slot: Slot) -> HashSet { let epoch = slot.epoch(E::slots_per_epoch()); // Only collect validators that are considered safe in terms of doppelganger protection. @@ -308,7 +406,7 @@ impl DutiesService { /// Returns all `ValidatorDuty` for the given `slot`. pub fn attesters(&self, slot: Slot) -> Vec { - let epoch = slot.epoch(E::slots_per_epoch()); + let epoch = slot.epoch(S::E::slots_per_epoch()); // Only collect validators that are considered safe in terms of doppelganger protection. let signing_pubkeys: HashSet<_> = self @@ -346,15 +444,15 @@ impl DutiesService { /// process every slot, which has the chance of creating a theoretically unlimited backlog of tasks. /// It was a conscious decision to choose to drop tasks on an overloaded/latent system rather than /// overload it even more. -pub fn start_update_service( - core_duties_service: Arc>, +pub fn start_update_service( + core_duties_service: Arc>, mut block_service_tx: Sender, ) { /* * Spawn the task which updates the map of pubkey to validator index. */ let duties_service = core_duties_service.clone(); - core_duties_service.context.executor.spawn( + core_duties_service.executor.spawn( async move { loop { // Run this poll before the wait, this should hopefully download all the indices @@ -377,8 +475,7 @@ pub fn start_update_service( * Spawn the task which keeps track of local block proposal duties. */ let duties_service = core_duties_service.clone(); - let log = core_duties_service.context.log().clone(); - core_duties_service.context.executor.spawn( + core_duties_service.executor.spawn( async move { loop { if let Some(duration) = duties_service.slot_clock.duration_to_next_slot() { @@ -393,9 +490,8 @@ pub fn start_update_service( if let Err(e) = poll_beacon_proposers(&duties_service, &mut block_service_tx).await { error!( - log, - "Failed to poll beacon proposers"; - "error" => ?e + error = ?e, + "Failed to poll beacon proposers" ) } } @@ -407,8 +503,7 @@ pub fn start_update_service( * Spawn the task which keeps track of local attestation duties. */ let duties_service = core_duties_service.clone(); - let log = core_duties_service.context.log().clone(); - core_duties_service.context.executor.spawn( + core_duties_service.executor.spawn( async move { loop { if let Some(duration) = duties_service.slot_clock.duration_to_next_slot() { @@ -422,9 +517,8 @@ pub fn start_update_service( if let Err(e) = poll_beacon_attesters(&duties_service).await { error!( - log, - "Failed to poll beacon attesters"; - "error" => ?e + error = ?e, + "Failed to poll beacon attesters" ); } } @@ -434,15 +528,13 @@ pub fn start_update_service( // Spawn the task which keeps track of local sync committee duties. let duties_service = core_duties_service.clone(); - let log = core_duties_service.context.log().clone(); - core_duties_service.context.executor.spawn( + core_duties_service.executor.spawn( async move { loop { if let Err(e) = poll_sync_committee_duties(&duties_service).await { error!( - log, - "Failed to poll sync committee duties"; - "error" => ?e + error = ?e, + "Failed to poll sync committee duties" ); } @@ -466,16 +558,14 @@ pub fn start_update_service( /// Iterate through all the voting pubkeys in the `ValidatorStore` and attempt to learn any unknown /// validator indices. -async fn poll_validator_indices( - duties_service: &DutiesService, +async fn poll_validator_indices( + duties_service: &DutiesService, ) { let _timer = validator_metrics::start_timer_vec( &validator_metrics::DUTIES_SERVICE_TIMES, &[validator_metrics::UPDATE_INDICES], ); - let log = duties_service.context.log(); - // Collect *all* pubkeys for resolving indices, even those undergoing doppelganger protection. // // Since doppelganger protection queries rely on validator indices it is important to ensure we @@ -488,16 +578,14 @@ async fn poll_validator_indices( // This is on its own line to avoid some weirdness with locks and if statements. let is_known = duties_service .validator_store - .initialized_validators() - .read() - .get_index(&pubkey) + .validator_index(&pubkey) .is_some(); if !is_known { let current_slot_opt = duties_service.slot_clock.now(); if let Some(current_slot) = current_slot_opt { - let is_first_slot_of_epoch = current_slot % E::slots_per_epoch() == 0; + let is_first_slot_of_epoch = current_slot % S::E::slots_per_epoch() == 0; // Query an unknown validator later if it was queried within the last epoch, or if // the current slot is the first slot of an epoch. @@ -541,17 +629,14 @@ async fn poll_validator_indices( match download_result { Ok(Some(response)) => { info!( - log, - "Validator exists in beacon chain"; - "pubkey" => ?pubkey, - "validator_index" => response.data.index, - "fee_recipient" => fee_recipient + ?pubkey, + validator_index = response.data.index, + fee_recipient, + "Validator exists in beacon chain" ); duties_service .validator_store - .initialized_validators() - .write() - .set_index(&pubkey, response.data.index); + .set_validator_index(&pubkey, response.data.index); duties_service .unknown_validator_next_poll_slots @@ -562,28 +647,22 @@ async fn poll_validator_indices( // the beacon chain. Ok(None) => { if let Some(current_slot) = current_slot_opt { - let next_poll_slot = current_slot.saturating_add(E::slots_per_epoch()); + let next_poll_slot = current_slot.saturating_add(S::E::slots_per_epoch()); duties_service .unknown_validator_next_poll_slots .write() .insert(pubkey, next_poll_slot); } - debug!( - log, - "Validator without index"; - "pubkey" => ?pubkey, - "fee_recipient" => fee_recipient - ) + debug!(?pubkey, fee_recipient, "Validator without index") } // Don't exit early on an error, keep attempting to resolve other indices. Err(e) => { error!( - log, - "Failed to resolve pubkey to index"; - "error" => %e, - "pubkey" => ?pubkey, - "fee_recipient" => fee_recipient + error = %e, + ?pubkey, + fee_recipient, + "Failed to resolve pubkey to index" ) } } @@ -599,21 +678,19 @@ async fn poll_validator_indices( /// 2. As above, but for the next-epoch. /// 3. Push out any attestation subnet subscriptions to the BN. /// 4. Prune old entries from `duties_service.attesters`. -async fn poll_beacon_attesters( - duties_service: &Arc>, -) -> Result<(), Error> { +async fn poll_beacon_attesters( + duties_service: &Arc>, +) -> Result<(), Error> { let current_epoch_timer = validator_metrics::start_timer_vec( &validator_metrics::DUTIES_SERVICE_TIMES, &[validator_metrics::UPDATE_ATTESTERS_CURRENT_EPOCH], ); - let log = duties_service.context.log(); - let current_slot = duties_service .slot_clock .now() .ok_or(Error::UnableToReadSlotClock)?; - let current_epoch = current_slot.epoch(E::slots_per_epoch()); + let current_epoch = current_slot.epoch(S::E::slots_per_epoch()); let next_epoch = current_epoch + 1; // Collect *all* pubkeys, even those undergoing doppelganger protection. @@ -627,10 +704,8 @@ async fn poll_beacon_attesters( let local_indices = { let mut local_indices = Vec::with_capacity(local_pubkeys.len()); - let vals_ref = duties_service.validator_store.initialized_validators(); - let vals = vals_ref.read(); for &pubkey in &local_pubkeys { - if let Some(validator_index) = vals.get_index(&pubkey) { + if let Some(validator_index) = duties_service.validator_store.validator_index(&pubkey) { local_indices.push(validator_index) } } @@ -647,15 +722,14 @@ async fn poll_beacon_attesters( .await { error!( - log, - "Failed to download attester duties"; - "current_epoch" => current_epoch, - "request_epoch" => current_epoch, - "err" => ?e, + %current_epoch, + request_epoch = %current_epoch, + err = ?e, + "Failed to download attester duties" ) } - update_per_validator_duty_metrics::(duties_service, current_epoch, current_slot); + update_per_validator_duty_metrics(duties_service, current_epoch, current_slot); drop(current_epoch_timer); let next_epoch_timer = validator_metrics::start_timer_vec( @@ -669,15 +743,14 @@ async fn poll_beacon_attesters( .await { error!( - log, - "Failed to download attester duties"; - "current_epoch" => current_epoch, - "request_epoch" => next_epoch, - "err" => ?e, + %current_epoch, + request_epoch = %next_epoch, + err = ?e, + "Failed to download attester duties" ) } - update_per_validator_duty_metrics::(duties_service, next_epoch, current_slot); + update_per_validator_duty_metrics(duties_service, next_epoch, current_slot); drop(next_epoch_timer); let subscriptions_timer = validator_metrics::start_timer_vec( @@ -698,7 +771,7 @@ async fn poll_beacon_attesters( * std::cmp::max( 1, local_pubkeys.len() * ATTESTATION_SUBSCRIPTION_OFFSETS.len() - / E::slots_per_epoch() as usize, + / S::E::slots_per_epoch() as usize, ) / overallocation_denominator; let mut subscriptions = Vec::with_capacity(num_expected_subscriptions); @@ -752,9 +825,8 @@ async fn poll_beacon_attesters( .await; if subscription_result.as_ref().is_ok() { debug!( - log, - "Broadcast attestation subscriptions"; - "count" => subscriptions.len(), + count = subscriptions.len(), + "Broadcast attestation subscriptions" ); for subscription_slots in subscription_slots_to_confirm { subscription_slots.record_successful_subscription_at(current_slot); @@ -762,9 +834,8 @@ async fn poll_beacon_attesters( } else if let Err(e) = subscription_result { if e.num_errors() < duties_service.beacon_nodes.num_total().await { warn!( - log, - "Some subscriptions failed"; - "error" => %e, + error = %e, + "Some subscriptions failed" ); // If subscriptions were sent to at least one node, regard that as a success. // There is some redundancy built into the subscription schedule to handle failures. @@ -773,9 +844,8 @@ async fn poll_beacon_attesters( } } else { error!( - log, - "All subscriptions failed"; - "error" => %e + error = %e, + "All subscriptions failed" ); } } @@ -797,20 +867,17 @@ async fn poll_beacon_attesters( /// For the given `local_indices` and `local_pubkeys`, download the duties for the given `epoch` and /// store them in `duties_service.attesters`. -async fn poll_beacon_attesters_for_epoch( - duties_service: &Arc>, +async fn poll_beacon_attesters_for_epoch( + duties_service: &Arc>, epoch: Epoch, local_indices: &[u64], local_pubkeys: &HashSet, -) -> Result<(), Error> { - let log = duties_service.context.log(); - +) -> Result<(), Error> { // No need to bother the BN if we don't have any validators. if local_indices.is_empty() { debug!( - duties_service.context.log(), - "No validators, not downloading duties"; - "epoch" => epoch, + %epoch, + "No validators, not downloading duties" ); return Ok(()); } @@ -889,10 +956,9 @@ async fn poll_beacon_attesters_for_epoch( ); debug!( - log, - "Downloaded attester duties"; - "dependent_root" => %dependent_root, - "num_new_duties" => new_duties.len(), + %dependent_root, + num_new_duties = new_duties.len(), + "Downloaded attester duties" ); // Update the duties service with the new `DutyAndProof` messages. @@ -923,10 +989,9 @@ async fn poll_beacon_attesters_for_epoch( && prior_duty_and_proof.duty == duty_and_proof.duty { warn!( - log, - "Redundant attester duty update"; - "dependent_root" => %dependent_root, - "validator_index" => duty.validator_index, + %dependent_root, + validator_index = duty.validator_index, + "Redundant attester duty update" ); continue; } @@ -934,11 +999,10 @@ async fn poll_beacon_attesters_for_epoch( // Using `already_warned` avoids excessive logs. if dependent_root != *prior_dependent_root && already_warned.take().is_some() { warn!( - log, - "Attester duties re-org"; - "prior_dependent_root" => %prior_dependent_root, - "dependent_root" => %dependent_root, - "note" => "this may happen from time to time" + %prior_dependent_root, + %dependent_root, + note = "this may happen from time to time", + "Attester duties re-org" ) } *mut_value = (dependent_root, duty_and_proof); @@ -952,7 +1016,7 @@ async fn poll_beacon_attesters_for_epoch( // Spawn the background task to compute selection proofs. let subservice = duties_service.clone(); - duties_service.context.executor.spawn( + duties_service.executor.spawn( async move { fill_in_selection_proofs(subservice, new_duties, dependent_root).await; }, @@ -963,8 +1027,8 @@ async fn poll_beacon_attesters_for_epoch( } /// Get a filtered list of local validators for which we don't already know their duties for that epoch -fn get_uninitialized_validators( - duties_service: &Arc>, +fn get_uninitialized_validators( + duties_service: &Arc>, epoch: &Epoch, local_pubkeys: &HashSet, ) -> Vec { @@ -980,8 +1044,8 @@ fn get_uninitialized_validators( .collect::>() } -fn update_per_validator_duty_metrics( - duties_service: &Arc>, +fn update_per_validator_duty_metrics( + duties_service: &Arc>, epoch: Epoch, current_slot: Slot, ) { @@ -996,14 +1060,14 @@ fn update_per_validator_duty_metrics( get_int_gauge(&ATTESTATION_DUTY, &[&validator_index.to_string()]) { let existing_slot = Slot::new(existing_slot_gauge.get() as u64); - let existing_epoch = existing_slot.epoch(E::slots_per_epoch()); + let existing_epoch = existing_slot.epoch(S::E::slots_per_epoch()); // First condition ensures that we switch to the next epoch duty slot // once the current epoch duty slot passes. // Second condition is to ensure that next epoch duties don't override // current epoch duties. if existing_slot < current_slot - || (duty_slot.epoch(E::slots_per_epoch()) <= existing_epoch + || (duty_slot.epoch(S::E::slots_per_epoch()) <= existing_epoch && duty_slot > current_slot && duty_slot != existing_slot) { @@ -1021,11 +1085,11 @@ fn update_per_validator_duty_metrics( } } -async fn post_validator_duties_attester( - duties_service: &Arc>, +async fn post_validator_duties_attester( + duties_service: &Arc>, epoch: Epoch, validator_indices: &[u64], -) -> Result>, Error> { +) -> Result>, Error> { duties_service .beacon_nodes .first_success(|beacon_node| async move { @@ -1045,13 +1109,11 @@ async fn post_validator_duties_attester( /// /// Duties are computed in batches each slot. If a re-org is detected then the process will /// terminate early as it is assumed the selection proofs from `duties` are no longer relevant. -async fn fill_in_selection_proofs( - duties_service: Arc>, +async fn fill_in_selection_proofs( + duties_service: Arc>, duties: Vec, dependent_root: Hash256, ) { - let log = duties_service.context.log(); - // Sort duties by slot in a BTreeMap. let mut duties_by_slot: BTreeMap> = BTreeMap::new(); @@ -1099,7 +1161,7 @@ async fn fill_in_selection_proofs( .then(|duty| async { let opt_selection_proof = make_selection_proof( &duty, - &duties_service.validator_store, + duties_service.validator_store.as_ref(), &duties_service.spec, ) .await?; @@ -1119,20 +1181,18 @@ async fn fill_in_selection_proofs( // A pubkey can be missing when a validator was recently // removed via the API. warn!( - log, - "Missing pubkey for duty and proof"; - "info" => "a validator may have recently been removed from this VC", - "pubkey" => ?pubkey, + info = "a validator may have recently been removed from this VC", + ?pubkey, + "Missing pubkey for duty and proof" ); // Do not abort the entire batch for a single failure. continue; } Err(e) => { error!( - log, - "Failed to produce duty and proof"; - "error" => ?e, - "msg" => "may impair attestation duties" + error = ?e, + msg = "may impair attestation duties", + "Failed to produce duty and proof" ); // Do not abort the entire batch for a single failure. continue; @@ -1140,7 +1200,7 @@ async fn fill_in_selection_proofs( }; let attester_map = attesters.entry(duty.pubkey).or_default(); - let epoch = duty.slot.epoch(E::slots_per_epoch()); + let epoch = duty.slot.epoch(S::E::slots_per_epoch()); match attester_map.entry(epoch) { hash_map::Entry::Occupied(mut entry) => { // No need to update duties for which no proof was computed. @@ -1157,9 +1217,8 @@ async fn fill_in_selection_proofs( // Our selection proofs are no longer relevant due to a reorg, abandon // this entire background process. debug!( - log, - "Stopping selection proof background task"; - "reason" => "re-org" + reason = "re-org", + "Stopping selection proof background task" ); return; } @@ -1182,11 +1241,10 @@ async fn fill_in_selection_proofs( let time_taken_ms = Duration::from_secs_f64(timer.map_or(0.0, |t| t.stop_and_record())).as_millis(); debug!( - log, - "Computed attestation selection proofs"; - "batch_size" => batch_size, - "lookahead_slot" => lookahead_slot, - "time_taken_ms" => time_taken_ms + batch_size, + %lookahead_slot, + time_taken_ms, + "Computed attestation selection proofs" ); } else { // Just sleep for one slot if we are unable to read the system clock, this gives @@ -1219,33 +1277,30 @@ async fn fill_in_selection_proofs( /// through the slow path every time. I.e., the proposal will only happen after we've been able to /// download and process the duties from the BN. This means it is very important to ensure this /// function is as fast as possible. -async fn poll_beacon_proposers( - duties_service: &DutiesService, +async fn poll_beacon_proposers( + duties_service: &DutiesService, block_service_tx: &mut Sender, -) -> Result<(), Error> { +) -> Result<(), Error> { let _timer = validator_metrics::start_timer_vec( &validator_metrics::DUTIES_SERVICE_TIMES, &[validator_metrics::UPDATE_PROPOSERS], ); - let log = duties_service.context.log(); - let current_slot = duties_service .slot_clock .now() .ok_or(Error::UnableToReadSlotClock)?; - let current_epoch = current_slot.epoch(E::slots_per_epoch()); + let current_epoch = current_slot.epoch(S::E::slots_per_epoch()); // Notify the block proposal service for any proposals that we have in our cache. // // See the function-level documentation for more information. - let initial_block_proposers = duties_service.block_proposers(current_slot); - notify_block_production_service( + let initial_block_proposers = duties_service.block_proposers::(current_slot); + notify_block_production_service::( current_slot, &initial_block_proposers, block_service_tx, - &duties_service.validator_store, - log, + duties_service.validator_store.as_ref(), ) .await; @@ -1284,10 +1339,9 @@ async fn poll_beacon_proposers( .collect::>(); debug!( - log, - "Downloaded proposer duties"; - "dependent_root" => %dependent_root, - "num_relevant_duties" => relevant_duties.len(), + %dependent_root, + num_relevant_duties = relevant_duties.len(), + "Downloaded proposer duties" ); if let Some((prior_dependent_root, _)) = duties_service @@ -1297,20 +1351,18 @@ async fn poll_beacon_proposers( { if dependent_root != prior_dependent_root { warn!( - log, - "Proposer duties re-org"; - "prior_dependent_root" => %prior_dependent_root, - "dependent_root" => %dependent_root, - "msg" => "this may happen from time to time" + %prior_dependent_root, + %dependent_root, + msg = "this may happen from time to time", + "Proposer duties re-org" ) } } } // Don't return early here, we still want to try and produce blocks using the cached values. Err(e) => error!( - log, - "Failed to download proposer duties"; - "err" => %e, + err = %e, + "Failed to download proposer duties" ), } @@ -1320,7 +1372,7 @@ async fn poll_beacon_proposers( // Then, compute the difference between these two sets to obtain a set of block proposers // which were not included in the initial notification to the `BlockService`. let additional_block_producers = duties_service - .block_proposers(current_slot) + .block_proposers::(current_slot) .difference(&initial_block_proposers) .copied() .collect::>(); @@ -1330,18 +1382,16 @@ async fn poll_beacon_proposers( // // See the function-level documentation for more reasoning about this behaviour. if !additional_block_producers.is_empty() { - notify_block_production_service( + notify_block_production_service::( current_slot, &additional_block_producers, block_service_tx, - &duties_service.validator_store, - log, + duties_service.validator_store.as_ref(), ) .await; debug!( - log, - "Detected new block proposer"; - "current_slot" => current_slot, + %current_slot, + "Detected new block proposer" ); validator_metrics::inc_counter(&validator_metrics::PROPOSAL_CHANGED); } @@ -1357,12 +1407,11 @@ async fn poll_beacon_proposers( } /// Notify the block service if it should produce a block. -async fn notify_block_production_service( +async fn notify_block_production_service( current_slot: Slot, block_proposers: &HashSet, block_service_tx: &mut Sender, - validator_store: &ValidatorStore, - log: &Logger, + validator_store: &S, ) { let non_doppelganger_proposers = block_proposers .iter() @@ -1379,10 +1428,9 @@ async fn notify_block_production_service( .await { error!( - log, - "Failed to notify block service"; - "current_slot" => current_slot, - "error" => %e + %current_slot, + error = %e, + "Failed to notify block service" ); }; } diff --git a/validator_client/validator_services/src/preparation_service.rs b/validator_client/validator_services/src/preparation_service.rs index 480f4af2b3c..8a0334bf78e 100644 --- a/validator_client/validator_services/src/preparation_service.rs +++ b/validator_client/validator_services/src/preparation_service.rs @@ -1,21 +1,22 @@ use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; use bls::PublicKeyBytes; -use doppelganger_service::DoppelgangerStatus; -use environment::RuntimeContext; use parking_lot::RwLock; -use slog::{debug, error, info, warn}; use slot_clock::SlotClock; use std::collections::HashMap; use std::hash::Hash; use std::ops::Deref; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; +use task_executor::TaskExecutor; use tokio::time::{sleep, Duration}; +use tracing::{debug, error, info, warn}; use types::{ Address, ChainSpec, EthSpec, ProposerPreparationData, SignedValidatorRegistrationData, ValidatorRegistrationData, }; -use validator_store::{Error as ValidatorStoreError, ProposalData, ValidatorStore}; +use validator_store::{ + DoppelgangerStatus, Error as ValidatorStoreError, ProposalData, ValidatorStore, +}; /// Number of epochs before the Bellatrix hard fork to begin posting proposer preparations. const PROPOSER_PREPARATION_LOOKAHEAD_EPOCHS: u64 = 2; @@ -25,28 +26,28 @@ const EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION: u64 = 1; /// Builds an `PreparationService`. #[derive(Default)] -pub struct PreparationServiceBuilder { - validator_store: Option>>, +pub struct PreparationServiceBuilder { + validator_store: Option>, slot_clock: Option, - beacon_nodes: Option>>, - context: Option>, + beacon_nodes: Option>>, + executor: Option, builder_registration_timestamp_override: Option, validator_registration_batch_size: Option, } -impl PreparationServiceBuilder { +impl PreparationServiceBuilder { pub fn new() -> Self { Self { validator_store: None, slot_clock: None, beacon_nodes: None, - context: None, + executor: None, builder_registration_timestamp_override: None, validator_registration_batch_size: None, } } - pub fn validator_store(mut self, store: Arc>) -> Self { + pub fn validator_store(mut self, store: Arc) -> Self { self.validator_store = Some(store); self } @@ -56,13 +57,13 @@ impl PreparationServiceBuilder { self } - pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { + pub fn beacon_nodes(mut self, beacon_nodes: Arc>) -> Self { self.beacon_nodes = Some(beacon_nodes); self } - pub fn runtime_context(mut self, context: RuntimeContext) -> Self { - self.context = Some(context); + pub fn executor(mut self, executor: TaskExecutor) -> Self { + self.executor = Some(executor); self } @@ -82,7 +83,7 @@ impl PreparationServiceBuilder { self } - pub fn build(self) -> Result, String> { + pub fn build(self) -> Result, String> { Ok(PreparationService { inner: Arc::new(Inner { validator_store: self @@ -94,9 +95,9 @@ impl PreparationServiceBuilder { beacon_nodes: self .beacon_nodes .ok_or("Cannot build PreparationService without beacon_nodes")?, - context: self - .context - .ok_or("Cannot build PreparationService without runtime_context")?, + executor: self + .executor + .ok_or("Cannot build PreparationService without executor")?, builder_registration_timestamp_override: self .builder_registration_timestamp_override, validator_registration_batch_size: self.validator_registration_batch_size.ok_or( @@ -109,11 +110,11 @@ impl PreparationServiceBuilder { } /// Helper to minimise `Arc` usage. -pub struct Inner { - validator_store: Arc>, +pub struct Inner { + validator_store: Arc, slot_clock: T, - beacon_nodes: Arc>, - context: RuntimeContext, + beacon_nodes: Arc>, + executor: TaskExecutor, builder_registration_timestamp_override: Option, // Used to track unpublished validator registration changes. validator_registration_cache: @@ -145,11 +146,11 @@ impl From for ValidatorRegistrationKey { } /// Attempts to produce proposer preparations for all known validators at the beginning of each epoch. -pub struct PreparationService { - inner: Arc>, +pub struct PreparationService { + inner: Arc>, } -impl Clone for PreparationService { +impl Clone for PreparationService { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -157,15 +158,15 @@ impl Clone for PreparationService { } } -impl Deref for PreparationService { - type Target = Inner; +impl Deref for PreparationService { + type Target = Inner; fn deref(&self) -> &Self::Target { self.inner.deref() } } -impl PreparationService { +impl PreparationService { pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> { self.clone().start_validator_registration_service(spec)?; self.start_proposer_prepare_service(spec) @@ -173,15 +174,10 @@ impl PreparationService { /// Starts the service which periodically produces proposer preparations. pub fn start_proposer_prepare_service(self, spec: &ChainSpec) -> Result<(), String> { - let log = self.context.log().clone(); - let slot_duration = Duration::from_secs(spec.seconds_per_slot); - info!( - log, - "Proposer preparation service started"; - ); + info!("Proposer preparation service started"); - let executor = self.context.executor.clone(); + let executor = self.executor.clone(); let spec = spec.clone(); let interval_fut = async move { @@ -192,9 +188,8 @@ impl PreparationService { .await .map_err(|e| { error!( - log, - "Error during proposer preparation"; - "error" => ?e, + error = ?e, + "Error during proposer preparation" ) }) .unwrap_or(()); @@ -203,7 +198,7 @@ impl PreparationService { if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() { sleep(duration_to_next_slot).await; } else { - error!(log, "Failed to read slot clock"); + error!("Failed to read slot clock"); // If we can't read the slot clock, just wait another slot. sleep(slot_duration).await; } @@ -216,30 +211,25 @@ impl PreparationService { /// Starts the service which periodically sends connected beacon nodes validator registration information. pub fn start_validator_registration_service(self, spec: &ChainSpec) -> Result<(), String> { - let log = self.context.log().clone(); - - info!( - log, - "Validator registration service started"; - ); + info!("Validator registration service started"); let spec = spec.clone(); let slot_duration = Duration::from_secs(spec.seconds_per_slot); - let executor = self.context.executor.clone(); + let executor = self.executor.clone(); let validator_registration_fut = async move { loop { // Poll the endpoint immediately to ensure fee recipients are received. if let Err(e) = self.register_validators().await { - error!(log,"Error during validator registration";"error" => ?e); + error!(error = ?e,"Error during validator registration"); } // Wait one slot if the register validator request fails or if we should not publish at the current slot. if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() { sleep(duration_to_next_slot).await; } else { - error!(log, "Failed to read slot clock"); + error!("Failed to read slot clock"); // If we can't read the slot clock, just wait another slot. sleep(slot_duration).await; } @@ -254,10 +244,9 @@ impl PreparationService { /// This avoids spamming the BN with preparations before the Bellatrix fork epoch, which may /// cause errors if it doesn't support the preparation API. fn should_publish_at_current_slot(&self, spec: &ChainSpec) -> bool { - let current_epoch = self - .slot_clock - .now() - .map_or(E::genesis_epoch(), |slot| slot.epoch(E::slots_per_epoch())); + let current_epoch = self.slot_clock.now().map_or(S::E::genesis_epoch(), |slot| { + slot.epoch(S::E::slots_per_epoch()) + }); spec.bellatrix_fork_epoch.map_or(false, |fork_epoch| { current_epoch + PROPOSER_PREPARATION_LOOKAHEAD_EPOCHS >= fork_epoch }) @@ -274,7 +263,6 @@ impl PreparationService { } fn collect_preparation_data(&self, spec: &ChainSpec) -> Vec { - let log = self.context.log(); self.collect_proposal_data(|pubkey, proposal_data| { if let Some(fee_recipient) = proposal_data.fee_recipient { Some(ProposerPreparationData { @@ -285,10 +273,9 @@ impl PreparationService { } else { if spec.bellatrix_fork_epoch.is_some() { error!( - log, - "Validator is missing fee recipient"; - "msg" => "update validator_definitions.yml", - "pubkey" => ?pubkey + msg = "update validator_definitions.yml", + ?pubkey, + "Validator is missing fee recipient" ); } None @@ -336,8 +323,6 @@ impl PreparationService { &self, preparation_data: Vec, ) -> Result<(), String> { - let log = self.context.log(); - // Post the proposer preparations to the BN. let preparation_data_len = preparation_data.len(); let preparation_entries = preparation_data.as_slice(); @@ -351,14 +336,12 @@ impl PreparationService { .await { Ok(()) => debug!( - log, - "Published proposer preparation"; - "count" => preparation_data_len, + count = preparation_data_len, + "Published proposer preparation" ), Err(e) => error!( - log, - "Unable to publish proposer preparation to all beacon nodes"; - "error" => %e, + error = %e, + "Unable to publish proposer preparation to all beacon nodes" ), } Ok(()) @@ -384,7 +367,8 @@ impl PreparationService { // Check if any have changed or it's been `EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION`. if let Some(slot) = self.slot_clock.now() { - if slot % (E::slots_per_epoch() * EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION) == 0 { + if slot % (S::E::slots_per_epoch() * EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION) == 0 + { self.publish_validator_registration_data(registration_keys) .await?; } else if !changed_keys.is_empty() { @@ -400,8 +384,6 @@ impl PreparationService { &self, registration_keys: Vec, ) -> Result<(), String> { - let log = self.context.log(); - let registration_data_len = registration_keys.len(); let mut signed = Vec::with_capacity(registration_data_len); @@ -442,19 +424,14 @@ impl PreparationService { Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { // A pubkey can be missing when a validator was recently // removed via the API. - debug!( - log, - "Missing pubkey for registration data"; - "pubkey" => ?pubkey, - ); + debug!(?pubkey, "Missing pubkey for registration data"); continue; } Err(e) => { error!( - log, - "Unable to sign validator registration data"; - "error" => ?e, - "pubkey" => ?pubkey + error = ?e, + ?pubkey, + "Unable to sign validator registration data" ); continue; } @@ -479,14 +456,12 @@ impl PreparationService { .await { Ok(()) => info!( - log, - "Published validator registrations to the builder network"; - "count" => batch.len(), + count = batch.len(), + "Published validator registrations to the builder network" ), Err(e) => warn!( - log, - "Unable to publish validator registrations to the builder network"; - "error" => %e, + error = %e, + "Unable to publish validator registrations to the builder network" ), } } diff --git a/validator_client/validator_services/src/sync.rs b/validator_client/validator_services/src/sync.rs index af501326f42..8b20f166ab0 100644 --- a/validator_client/validator_services/src/sync.rs +++ b/validator_client/validator_services/src/sync.rs @@ -1,14 +1,13 @@ use crate::duties_service::{DutiesService, Error}; -use doppelganger_service::DoppelgangerStatus; use futures::future::join_all; +use logging::crit; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use slog::{crit, debug, info, warn}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; -use std::marker::PhantomData; use std::sync::Arc; +use tracing::{debug, info, warn}; use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId}; -use validator_store::Error as ValidatorStoreError; +use validator_store::{DoppelgangerStatus, Error as ValidatorStoreError, ValidatorStore}; /// Number of epochs in advance to compute selection proofs when not in `distributed` mode. pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; @@ -27,12 +26,11 @@ pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; /// 2. One-at-a-time locking. For the innermost locks on the aggregator duties, all of the functions /// in this file take care to only lock one validator at a time. We never hold a lock while /// trying to obtain another one (hence no lock ordering issues). -pub struct SyncDutiesMap { +pub struct SyncDutiesMap { /// Map from sync committee period to duties for members of that sync committee. committees: RwLock>, /// Whether we are in `distributed` mode and using reduced lookahead for aggregate pre-compute. distributed: bool, - _phantom: PhantomData, } /// Duties for a single sync committee period. @@ -80,12 +78,11 @@ pub struct SlotDuties { pub aggregators: HashMap>, } -impl SyncDutiesMap { +impl SyncDutiesMap { pub fn new(distributed: bool) -> Self { Self { committees: RwLock::new(HashMap::new()), distributed, - _phantom: PhantomData, } } @@ -103,7 +100,7 @@ impl SyncDutiesMap { } /// Number of slots in advance to compute selection proofs - fn aggregation_pre_compute_slots(&self) -> u64 { + fn aggregation_pre_compute_slots(&self) -> u64 { if self.distributed { AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED } else { @@ -116,7 +113,7 @@ impl SyncDutiesMap { /// Return the slot up to which proofs should be pre-computed, as well as a vec of /// `(previous_pre_compute_slot, sync_duty)` pairs for all validators which need to have proofs /// computed. See `fill_in_aggregation_proofs` for the actual calculation. - fn prepare_for_aggregator_pre_compute( + fn prepare_for_aggregator_pre_compute( &self, committee_period: u64, current_slot: Slot, @@ -126,7 +123,7 @@ impl SyncDutiesMap { current_slot, first_slot_of_period::(committee_period, spec), ); - let pre_compute_lookahead_slots = self.aggregation_pre_compute_slots(); + let pre_compute_lookahead_slots = self.aggregation_pre_compute_slots::(); let pre_compute_slot = std::cmp::min( current_slot + pre_compute_lookahead_slots, last_slot_of_period::(committee_period, spec), @@ -186,7 +183,7 @@ impl SyncDutiesMap { /// Get duties for all validators for the given `wall_clock_slot`. /// /// This is the entry-point for the sync committee service. - pub fn get_duties_for_slot( + pub fn get_duties_for_slot( &self, wall_clock_slot: Slot, spec: &ChainSpec, @@ -283,16 +280,16 @@ fn last_slot_of_period(sync_committee_period: u64, spec: &ChainSpec) first_slot_of_period::(sync_committee_period + 1, spec) - 1 } -pub async fn poll_sync_committee_duties( - duties_service: &Arc>, -) -> Result<(), Error> { +pub async fn poll_sync_committee_duties( + duties_service: &Arc>, +) -> Result<(), Error> { let sync_duties = &duties_service.sync_duties; let spec = &duties_service.spec; let current_slot = duties_service .slot_clock .now() .ok_or(Error::UnableToReadSlotClock)?; - let current_epoch = current_slot.epoch(E::slots_per_epoch()); + let current_epoch = current_slot.epoch(S::E::slots_per_epoch()); // If the Altair fork is yet to be activated, do not attempt to poll for duties. if spec @@ -316,10 +313,8 @@ pub async fn poll_sync_committee_duties( let local_indices = { let mut local_indices = Vec::with_capacity(local_pubkeys.len()); - let vals_ref = duties_service.validator_store.initialized_validators(); - let vals = vals_ref.read(); for &pubkey in &local_pubkeys { - if let Some(validator_index) = vals.get_index(&pubkey) { + if let Some(validator_index) = duties_service.validator_store.validator_index(&pubkey) { local_indices.push(validator_index) } } @@ -341,11 +336,15 @@ pub async fn poll_sync_committee_duties( // Pre-compute aggregator selection proofs for the current period. let (current_pre_compute_slot, new_pre_compute_duties) = sync_duties - .prepare_for_aggregator_pre_compute(current_sync_committee_period, current_slot, spec); + .prepare_for_aggregator_pre_compute::( + current_sync_committee_period, + current_slot, + spec, + ); if !new_pre_compute_duties.is_empty() { let sub_duties_service = duties_service.clone(); - duties_service.context.executor.spawn( + duties_service.executor.spawn( async move { fill_in_aggregation_proofs( sub_duties_service, @@ -378,18 +377,22 @@ pub async fn poll_sync_committee_duties( } // Pre-compute aggregator selection proofs for the next period. - let aggregate_pre_compute_lookahead_slots = sync_duties.aggregation_pre_compute_slots(); + let aggregate_pre_compute_lookahead_slots = sync_duties.aggregation_pre_compute_slots::(); if (current_slot + aggregate_pre_compute_lookahead_slots) - .epoch(E::slots_per_epoch()) + .epoch(S::E::slots_per_epoch()) .sync_committee_period(spec)? == next_sync_committee_period { let (pre_compute_slot, new_pre_compute_duties) = sync_duties - .prepare_for_aggregator_pre_compute(next_sync_committee_period, current_slot, spec); + .prepare_for_aggregator_pre_compute::( + next_sync_committee_period, + current_slot, + spec, + ); if !new_pre_compute_duties.is_empty() { let sub_duties_service = duties_service.clone(); - duties_service.context.executor.spawn( + duties_service.executor.spawn( async move { fill_in_aggregation_proofs( sub_duties_service, @@ -408,29 +411,26 @@ pub async fn poll_sync_committee_duties( Ok(()) } -pub async fn poll_sync_committee_duties_for_period( - duties_service: &Arc>, +pub async fn poll_sync_committee_duties_for_period( + duties_service: &Arc>, local_indices: &[u64], sync_committee_period: u64, -) -> Result<(), Error> { +) -> Result<(), Error> { let spec = &duties_service.spec; - let log = duties_service.context.log(); // no local validators don't need to poll for sync committee if local_indices.is_empty() { debug!( - duties_service.context.log(), - "No validators, not polling for sync committee duties"; - "sync_committee_period" => sync_committee_period, + sync_committee_period, + "No validators, not polling for sync committee duties" ); return Ok(()); } debug!( - log, - "Fetching sync committee duties"; - "sync_committee_period" => sync_committee_period, - "num_validators" => local_indices.len(), + sync_committee_period, + num_validators = local_indices.len(), + "Fetching sync committee duties" ); let period_start_epoch = spec.epochs_per_sync_committee_period * sync_committee_period; @@ -452,16 +452,15 @@ pub async fn poll_sync_committee_duties_for_period res.data, Err(e) => { warn!( - log, - "Failed to download sync committee duties"; - "sync_committee_period" => sync_committee_period, - "error" => %e, + sync_committee_period, + error = %e, + "Failed to download sync committee duties" ); return Ok(()); } }; - debug!(log, "Fetched sync duties from BN"; "count" => duties.len()); + debug!(count = duties.len(), "Fetched sync duties from BN"); // Add duties to map. let committee_duties = duties_service @@ -479,9 +478,8 @@ pub async fn poll_sync_committee_duties_for_period "this could be due to a really long re-org, or a bug" + message = "this could be due to a really long re-org, or a bug", + "Sync committee duties changed" ); } updated_due_to_reorg @@ -489,10 +487,8 @@ pub async fn poll_sync_committee_duties_for_period duty.validator_index, - "sync_committee_period" => sync_committee_period, + validator_index = duty.validator_index, + sync_committee_period, "Validator in sync committee" ); *validator_duties = Some(ValidatorDuties::new(duty)); @@ -502,21 +498,18 @@ pub async fn poll_sync_committee_duties_for_period( - duties_service: Arc>, +pub async fn fill_in_aggregation_proofs( + duties_service: Arc>, pre_compute_duties: &[(Slot, SyncDuty)], sync_committee_period: u64, current_slot: Slot, pre_compute_slot: Slot, ) { - let log = duties_service.context.log(); - debug!( - log, - "Calculating sync selection proofs"; - "period" => sync_committee_period, - "current_slot" => current_slot, - "pre_compute_slot" => pre_compute_slot + period = sync_committee_period, + %current_slot, + %pre_compute_slot, + "Calculating sync selection proofs" ); // Generate selection proofs for each validator at each slot, one slot at a time. @@ -528,13 +521,12 @@ pub async fn fill_in_aggregation_proofs( continue; } - let subnet_ids = match duty.subnet_ids::() { + let subnet_ids = match duty.subnet_ids::() { Ok(subnet_ids) => subnet_ids, Err(e) => { crit!( - log, - "Arithmetic error computing subnet IDs"; - "error" => ?e, + error = ?e, + "Arithmetic error computing subnet IDs" ); continue; } @@ -556,45 +548,41 @@ pub async fn fill_in_aggregation_proofs( // A pubkey can be missing when a validator was recently // removed via the API. debug!( - log, - "Missing pubkey for sync selection proof"; - "pubkey" => ?pubkey, - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, + ?pubkey, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Missing pubkey for sync selection proof" ); return None; } Err(e) => { warn!( - log, - "Unable to sign selection proof"; - "error" => ?e, - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, + error = ?e, + pubkey = ?duty.pubkey, + slot = %proof_slot, + "Unable to sign selection proof" ); return None; } }; - match proof.is_aggregator::() { + match proof.is_aggregator::() { Ok(true) => { debug!( - log, - "Validator is sync aggregator"; - "validator_index" => duty.validator_index, - "slot" => proof_slot, - "subnet_id" => %subnet_id, + validator_index = duty.validator_index, + slot = %proof_slot, + %subnet_id, + "Validator is sync aggregator" ); Some(((proof_slot, *subnet_id), proof)) } Ok(false) => None, Err(e) => { warn!( - log, - "Error determining is_aggregator"; - "pubkey" => ?duty.pubkey, - "slot" => proof_slot, - "error" => ?e, + pubkey = ?duty.pubkey, + slot = %proof_slot, + error = ?e, + "Error determining is_aggregator" ); None } @@ -614,11 +602,7 @@ pub async fn fill_in_aggregation_proofs( // Add to global storage (we add regularly so the proofs can be used ASAP). let sync_map = duties_service.sync_duties.committees.read(); let Some(committee_duties) = sync_map.get(&sync_committee_period) else { - debug!( - log, - "Missing sync duties"; - "period" => sync_committee_period, - ); + debug!(period = sync_committee_period, "Missing sync duties"); continue; }; let validators = committee_duties.validators.read(); @@ -629,20 +613,18 @@ pub async fn fill_in_aggregation_proofs( duty.aggregation_duties.proofs.write().extend(proofs); } else { debug!( - log, - "Missing sync duty to update"; - "validator_index" => validator_index, - "period" => sync_committee_period, + validator_index, + period = sync_committee_period, + "Missing sync duty to update" ); } } if num_validators_updated > 0 { debug!( - log, - "Finished computing sync selection proofs"; - "slot" => slot, - "updated_validators" => num_validators_updated, + %slot, + updated_validators = num_validators_updated, + "Finished computing sync selection proofs" ); } } diff --git a/validator_client/validator_services/src/sync_committee_service.rs b/validator_client/validator_services/src/sync_committee_service.rs index 3ab5b33b6cc..081447b9b7b 100644 --- a/validator_client/validator_services/src/sync_committee_service.rs +++ b/validator_client/validator_services/src/sync_committee_service.rs @@ -1,16 +1,17 @@ use crate::duties_service::DutiesService; use beacon_node_fallback::{ApiTopic, BeaconNodeFallback}; -use environment::RuntimeContext; use eth2::types::BlockId; use futures::future::join_all; use futures::future::FutureExt; -use slog::{crit, debug, error, info, trace, warn}; +use logging::crit; use slot_clock::SlotClock; use std::collections::HashMap; use std::ops::Deref; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use task_executor::TaskExecutor; use tokio::time::{sleep, sleep_until, Duration, Instant}; +use tracing::{debug, error, info, trace, warn}; use types::{ ChainSpec, EthSpec, Hash256, PublicKeyBytes, Slot, SyncCommitteeSubscription, SyncContributionData, SyncDuty, SyncSelectionProof, SyncSubnetId, @@ -19,11 +20,11 @@ use validator_store::{Error as ValidatorStoreError, ValidatorStore}; pub const SUBSCRIPTION_LOOKAHEAD_EPOCHS: u64 = 4; -pub struct SyncCommitteeService { - inner: Arc>, +pub struct SyncCommitteeService { + inner: Arc>, } -impl Clone for SyncCommitteeService { +impl Clone for SyncCommitteeService { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -31,33 +32,33 @@ impl Clone for SyncCommitteeService { } } -impl Deref for SyncCommitteeService { - type Target = Inner; +impl Deref for SyncCommitteeService { + type Target = Inner; fn deref(&self) -> &Self::Target { self.inner.deref() } } -pub struct Inner { - duties_service: Arc>, - validator_store: Arc>, +pub struct Inner { + duties_service: Arc>, + validator_store: Arc, slot_clock: T, - beacon_nodes: Arc>, - context: RuntimeContext, + beacon_nodes: Arc>, + executor: TaskExecutor, /// Boolean to track whether the service has posted subscriptions to the BN at least once. /// /// This acts as a latch that fires once upon start-up, and then never again. first_subscription_done: AtomicBool, } -impl SyncCommitteeService { +impl SyncCommitteeService { pub fn new( - duties_service: Arc>, - validator_store: Arc>, + duties_service: Arc>, + validator_store: Arc, slot_clock: T, - beacon_nodes: Arc>, - context: RuntimeContext, + beacon_nodes: Arc>, + executor: TaskExecutor, ) -> Self { Self { inner: Arc::new(Inner { @@ -65,7 +66,7 @@ impl SyncCommitteeService { validator_store, slot_clock, beacon_nodes, - context, + executor, first_subscription_done: AtomicBool::new(false), }), } @@ -79,14 +80,13 @@ impl SyncCommitteeService { .spec .altair_fork_epoch .and_then(|fork_epoch| { - let current_epoch = self.slot_clock.now()?.epoch(E::slots_per_epoch()); + let current_epoch = self.slot_clock.now()?.epoch(S::E::slots_per_epoch()); Some(current_epoch >= fork_epoch) }) .unwrap_or(false) } pub fn start_update_service(self, spec: &ChainSpec) -> Result<(), String> { - let log = self.context.log().clone(); let slot_duration = Duration::from_secs(spec.seconds_per_slot); let duration_to_next_slot = self .slot_clock @@ -94,18 +94,16 @@ impl SyncCommitteeService { .ok_or("Unable to determine duration to next slot")?; info!( - log, - "Sync committee service started"; - "next_update_millis" => duration_to_next_slot.as_millis() + next_update_millis = duration_to_next_slot.as_millis(), + "Sync committee service started" ); - let executor = self.context.executor.clone(); + let executor = self.executor.clone(); let interval_fut = async move { loop { if let Some(duration_to_next_slot) = self.slot_clock.duration_to_next_slot() { // Wait for contribution broadcast interval 1/3 of the way through the slot. - let log = self.context.log(); sleep(duration_to_next_slot + slot_duration / 3).await; // Do nothing if the Altair fork has not yet occurred. @@ -115,21 +113,17 @@ impl SyncCommitteeService { if let Err(e) = self.spawn_contribution_tasks(slot_duration).await { crit!( - log, - "Failed to spawn sync contribution tasks"; - "error" => e + error = ?e, + "Failed to spawn sync contribution tasks" ) } else { - trace!( - log, - "Spawned sync contribution tasks"; - ) + trace!("Spawned sync contribution tasks") } // Do subscriptions for future slots/epochs. self.spawn_subscription_tasks(); } else { - error!(log, "Failed to read slot clock"); + error!("Failed to read slot clock"); // If we can't read the slot clock, just wait another slot. sleep(slot_duration).await; } @@ -141,7 +135,6 @@ impl SyncCommitteeService { } async fn spawn_contribution_tasks(&self, slot_duration: Duration) -> Result<(), String> { - let log = self.context.log().clone(); let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; let duration_to_next_slot = self .slot_clock @@ -158,18 +151,14 @@ impl SyncCommitteeService { let Some(slot_duties) = self .duties_service .sync_duties - .get_duties_for_slot(slot, &self.duties_service.spec) + .get_duties_for_slot::(slot, &self.duties_service.spec) else { - debug!(log, "No duties known for slot {}", slot); + debug!("No duties known for slot {}", slot); return Ok(()); }; if slot_duties.duties.is_empty() { - debug!( - log, - "No local validators in current sync committee"; - "slot" => slot, - ); + debug!(%slot, "No local validators in current sync committee"); return Ok(()); } @@ -196,11 +185,10 @@ impl SyncCommitteeService { Ok(block) => block.data.root, Err(errs) => { warn!( - log, + errors = errs.to_string(), + %slot, "Refusing to sign sync committee messages for an optimistic head block or \ - a block head with unknown optimistic status"; - "errors" => errs.to_string(), - "slot" => slot, + a block head with unknown optimistic status" ); return Ok(()); } @@ -209,7 +197,7 @@ impl SyncCommitteeService { // Spawn one task to publish all of the sync committee signatures. let validator_duties = slot_duties.duties; let service = self.clone(); - self.inner.context.executor.spawn( + self.inner.executor.spawn( async move { service .publish_sync_committee_signatures(slot, block_root, validator_duties) @@ -221,7 +209,7 @@ impl SyncCommitteeService { let aggregators = slot_duties.aggregators; let service = self.clone(); - self.inner.context.executor.spawn( + self.inner.executor.spawn( async move { service .publish_sync_committee_aggregates( @@ -246,8 +234,6 @@ impl SyncCommitteeService { beacon_block_root: Hash256, validator_duties: Vec, ) -> Result<(), ()> { - let log = self.context.log(); - // Create futures to produce sync committee signatures. let signature_futures = validator_duties.iter().map(|duty| async move { match self @@ -265,21 +251,19 @@ impl SyncCommitteeService { // A pubkey can be missing when a validator was recently // removed via the API. debug!( - log, - "Missing pubkey for sync committee signature"; - "pubkey" => ?pubkey, - "validator_index" => duty.validator_index, - "slot" => slot, + ?pubkey, + validator_index = duty.validator_index, + %slot, + "Missing pubkey for sync committee signature" ); None } Err(e) => { crit!( - log, - "Failed to sign sync committee signature"; - "validator_index" => duty.validator_index, - "slot" => slot, - "error" => ?e, + validator_index = duty.validator_index, + %slot, + error = ?e, + "Failed to sign sync committee signature" ); None } @@ -302,19 +286,17 @@ impl SyncCommitteeService { .await .map_err(|e| { error!( - log, - "Unable to publish sync committee messages"; - "slot" => slot, - "error" => %e, + %slot, + error = %e, + "Unable to publish sync committee messages" ); })?; info!( - log, - "Successfully published sync committee messages"; - "count" => committee_signatures.len(), - "head_block" => ?beacon_block_root, - "slot" => slot, + count = committee_signatures.len(), + head_block = ?beacon_block_root, + %slot, + "Successfully published sync committee messages" ); Ok(()) @@ -329,7 +311,7 @@ impl SyncCommitteeService { ) { for (subnet_id, subnet_aggregators) in aggregators { let service = self.clone(); - self.inner.context.executor.spawn( + self.inner.executor.spawn( async move { service .publish_sync_committee_aggregate_for_subnet( @@ -357,8 +339,6 @@ impl SyncCommitteeService { ) -> Result<(), ()> { sleep_until(aggregate_instant).await; - let log = self.context.log(); - let contribution = &self .beacon_nodes .first_success(|beacon_node| async move { @@ -369,26 +349,20 @@ impl SyncCommitteeService { }; beacon_node - .get_validator_sync_committee_contribution::(&sync_contribution_data) + .get_validator_sync_committee_contribution(&sync_contribution_data) .await }) .await .map_err(|e| { crit!( - log, - "Failed to produce sync contribution"; - "slot" => slot, - "beacon_block_root" => ?beacon_block_root, - "error" => %e, + %slot, + ?beacon_block_root, + error = %e, + "Failed to produce sync contribution" ) })? .ok_or_else(|| { - crit!( - log, - "No aggregate contribution found"; - "slot" => slot, - "beacon_block_root" => ?beacon_block_root, - ); + crit!(%slot, ?beacon_block_root, "No aggregate contribution found"); })? .data; @@ -409,20 +383,14 @@ impl SyncCommitteeService { Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { // A pubkey can be missing when a validator was recently // removed via the API. - debug!( - log, - "Missing pubkey for sync contribution"; - "pubkey" => ?pubkey, - "slot" => slot, - ); + debug!(?pubkey, %slot, "Missing pubkey for sync contribution"); None } Err(e) => { crit!( - log, - "Unable to sign sync committee contribution"; - "slot" => slot, - "error" => ?e, + %slot, + error = ?e, + "Unable to sign sync committee contribution" ); None } @@ -447,20 +415,18 @@ impl SyncCommitteeService { .await .map_err(|e| { error!( - log, - "Unable to publish signed contributions and proofs"; - "slot" => slot, - "error" => %e, + %slot, + error = %e, + "Unable to publish signed contributions and proofs" ); })?; info!( - log, - "Successfully published sync contributions"; - "subnet" => %subnet_id, - "beacon_block_root" => %beacon_block_root, - "num_signers" => contribution.aggregation_bits.num_set_bits(), - "slot" => slot, + subnet = %subnet_id, + beacon_block_root = %beacon_block_root, + num_signers = contribution.aggregation_bits.num_set_bits(), + %slot, + "Successfully published sync contributions" ); Ok(()) @@ -468,14 +434,13 @@ impl SyncCommitteeService { fn spawn_subscription_tasks(&self) { let service = self.clone(); - let log = self.context.log().clone(); - self.inner.context.executor.spawn( + + self.inner.executor.spawn( async move { service.publish_subscriptions().await.unwrap_or_else(|e| { error!( - log, - "Error publishing subscriptions"; - "error" => ?e, + error = ?e, + "Error publishing subscriptions" ) }); }, @@ -484,7 +449,6 @@ impl SyncCommitteeService { } async fn publish_subscriptions(self) -> Result<(), String> { - let log = self.context.log().clone(); let spec = &self.duties_service.spec; let slot = self.slot_clock.now().ok_or("Failed to read slot clock")?; @@ -494,10 +458,10 @@ impl SyncCommitteeService { // At the start of every epoch during the current period, re-post the subscriptions // to the beacon node. This covers the case where the BN has forgotten the subscriptions // due to a restart, or where the VC has switched to a fallback BN. - let current_period = sync_period_of_slot::(slot, spec)?; + let current_period = sync_period_of_slot::(slot, spec)?; if !self.first_subscription_done.load(Ordering::Relaxed) - || slot.as_u64() % E::slots_per_epoch() == 0 + || slot.as_u64() % S::E::slots_per_epoch() == 0 { duty_slots.push((slot, current_period)); } @@ -505,9 +469,9 @@ impl SyncCommitteeService { // Near the end of the current period, push subscriptions for the next period to the // beacon node. We aggressively push every slot in the lead-up, as this is the main way // that we want to ensure that the BN is subscribed (well in advance). - let lookahead_slot = slot + SUBSCRIPTION_LOOKAHEAD_EPOCHS * E::slots_per_epoch(); + let lookahead_slot = slot + SUBSCRIPTION_LOOKAHEAD_EPOCHS * S::E::slots_per_epoch(); - let lookahead_period = sync_period_of_slot::(lookahead_slot, spec)?; + let lookahead_period = sync_period_of_slot::(lookahead_slot, spec)?; if lookahead_period > current_period { duty_slots.push((lookahead_slot, lookahead_period)); @@ -521,16 +485,11 @@ impl SyncCommitteeService { let mut subscriptions = vec![]; for (duty_slot, sync_committee_period) in duty_slots { - debug!( - log, - "Fetching subscription duties"; - "duty_slot" => duty_slot, - "current_slot" => slot, - ); + debug!(%duty_slot, %slot, "Fetching subscription duties"); match self .duties_service .sync_duties - .get_duties_for_slot(duty_slot, spec) + .get_duties_for_slot::(duty_slot, spec) { Some(duties) => subscriptions.extend(subscriptions_from_sync_duties( duties.duties, @@ -539,9 +498,8 @@ impl SyncCommitteeService { )), None => { debug!( - log, - "No duties for subscription"; - "slot" => duty_slot, + slot = %duty_slot, + "No duties for subscription" ); all_succeeded = false; } @@ -549,29 +507,23 @@ impl SyncCommitteeService { } if subscriptions.is_empty() { - debug!( - log, - "No sync subscriptions to send"; - "slot" => slot, - ); + debug!(%slot, "No sync subscriptions to send"); return Ok(()); } // Post subscriptions to BN. debug!( - log, - "Posting sync subscriptions to BN"; - "count" => subscriptions.len(), + count = subscriptions.len(), + "Posting sync subscriptions to BN" ); let subscriptions_slice = &subscriptions; for subscription in subscriptions_slice { debug!( - log, - "Subscription"; - "validator_index" => subscription.validator_index, - "validator_sync_committee_indices" => ?subscription.sync_committee_indices, - "until_epoch" => subscription.until_epoch, + validator_index = subscription.validator_index, + validator_sync_committee_indices = ?subscription.sync_committee_indices, + until_epoch = %subscription.until_epoch, + "Subscription" ); } @@ -585,10 +537,9 @@ impl SyncCommitteeService { .await { error!( - log, - "Unable to post sync committee subscriptions"; - "slot" => slot, - "error" => %e, + %slot, + error = %e, + "Unable to post sync committee subscriptions" ); all_succeeded = false; } diff --git a/validator_client/validator_store/Cargo.toml b/validator_client/validator_store/Cargo.toml index 99c3025a30d..91df9dc3abd 100644 --- a/validator_client/validator_store/Cargo.toml +++ b/validator_client/validator_store/Cargo.toml @@ -4,20 +4,6 @@ version = "0.1.0" edition = { workspace = true } authors = ["Sigma Prime "] -[lib] -name = "validator_store" -path = "src/lib.rs" - [dependencies] -account_utils = { workspace = true } -doppelganger_service = { workspace = true } -initialized_validators = { workspace = true } -parking_lot = { workspace = true } -serde = { workspace = true } -signing_method = { workspace = true } slashing_protection = { workspace = true } -slog = { workspace = true } -slot_clock = { workspace = true } -task_executor = { workspace = true } types = { workspace = true } -validator_metrics = { workspace = true } diff --git a/validator_client/validator_store/src/lib.rs b/validator_client/validator_store/src/lib.rs index 837af5b51d7..503c693191a 100644 --- a/validator_client/validator_store/src/lib.rs +++ b/validator_client/validator_store/src/lib.rs @@ -1,30 +1,17 @@ -use account_utils::validator_definitions::{PasswordStorage, ValidatorDefinition}; -use doppelganger_service::{DoppelgangerService, DoppelgangerStatus, DoppelgangerValidatorStore}; -use initialized_validators::InitializedValidators; -use parking_lot::{Mutex, RwLock}; -use serde::{Deserialize, Serialize}; -use signing_method::{Error as SigningError, SignableMessage, SigningContext, SigningMethod}; -use slashing_protection::{ - interchange::Interchange, InterchangeError, NotSafe, Safe, SlashingDatabase, -}; -use slog::{crit, error, info, warn, Logger}; -use slot_clock::SlotClock; -use std::marker::PhantomData; -use std::path::Path; -use std::sync::Arc; -use task_executor::TaskExecutor; +use slashing_protection::NotSafe; +use std::fmt::Debug; +use std::future::Future; use types::{ - attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, - AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, - Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, PublicKeyBytes, SelectionProof, Signature, - SignedAggregateAndProof, SignedBeaconBlock, SignedContributionAndProof, SignedRoot, - SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, - SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, - ValidatorRegistrationData, VoluntaryExit, + Address, Attestation, AttestationError, BeaconBlock, BlindedBeaconBlock, Epoch, EthSpec, + Graffiti, Hash256, PublicKeyBytes, SelectionProof, Signature, SignedAggregateAndProof, + SignedBeaconBlock, SignedBlindedBeaconBlock, SignedContributionAndProof, + SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeContribution, + SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, + VoluntaryExit, }; -#[derive(Debug, PartialEq)] -pub enum Error { +#[derive(Debug, PartialEq, Clone)] +pub enum Error { DoppelgangerProtected(PublicKeyBytes), UnknownToDoppelgangerService(PublicKeyBytes), UnknownPubkey(PublicKeyBytes), @@ -33,31 +20,15 @@ pub enum Error { GreaterThanCurrentSlot { slot: Slot, current_slot: Slot }, GreaterThanCurrentEpoch { epoch: Epoch, current_epoch: Epoch }, UnableToSignAttestation(AttestationError), - UnableToSign(SigningError), + SpecificError(T), } -impl From for Error { - fn from(e: SigningError) -> Self { - Error::UnableToSign(e) +impl From for Error { + fn from(e: T) -> Self { + Error::SpecificError(e) } } -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct Config { - /// Fallback fee recipient address. - pub fee_recipient: Option
, - /// Fallback gas limit. - pub gas_limit: Option, - /// Enable use of the blinded block endpoints during proposals. - pub builder_proposals: bool, - /// Enable slashing protection even while using web3signer keys. - pub enable_web3signer_slashing_protection: bool, - /// If true, Lighthouse will prefer builder proposals, if available. - pub prefer_builder_proposals: bool, - /// Specifies the boost factor, a percentage multiplier to apply to the builder's payload value. - pub builder_boost_factor: Option, -} - /// A helper struct, used for passing data from the validator store to services. pub struct ProposalData { pub validator_index: Option, @@ -66,188 +37,9 @@ pub struct ProposalData { pub builder_proposals: bool, } -/// Number of epochs of slashing protection history to keep. -/// -/// This acts as a maximum safe-guard against clock drift. -const SLASHING_PROTECTION_HISTORY_EPOCHS: u64 = 512; - -/// Currently used as the default gas limit in execution clients. -/// -/// https://github.com/ethereum/builder-specs/issues/17 -pub const DEFAULT_GAS_LIMIT: u64 = 30_000_000; - -pub struct ValidatorStore { - validators: Arc>, - slashing_protection: SlashingDatabase, - slashing_protection_last_prune: Arc>, - genesis_validators_root: Hash256, - spec: Arc, - log: Logger, - doppelganger_service: Option>, - slot_clock: T, - fee_recipient_process: Option
, - gas_limit: Option, - builder_proposals: bool, - enable_web3signer_slashing_protection: bool, - prefer_builder_proposals: bool, - builder_boost_factor: Option, - task_executor: TaskExecutor, - _phantom: PhantomData, -} - -impl DoppelgangerValidatorStore for ValidatorStore { - fn get_validator_index(&self, pubkey: &PublicKeyBytes) -> Option { - self.validator_index(pubkey) - } -} - -impl ValidatorStore { - // All arguments are different types. Making the fields `pub` is undesired. A builder seems - // unnecessary. - #[allow(clippy::too_many_arguments)] - pub fn new( - validators: InitializedValidators, - slashing_protection: SlashingDatabase, - genesis_validators_root: Hash256, - spec: Arc, - doppelganger_service: Option>, - slot_clock: T, - config: &Config, - task_executor: TaskExecutor, - log: Logger, - ) -> Self { - Self { - validators: Arc::new(RwLock::new(validators)), - slashing_protection, - slashing_protection_last_prune: Arc::new(Mutex::new(Epoch::new(0))), - genesis_validators_root, - spec, - log, - doppelganger_service, - slot_clock, - fee_recipient_process: config.fee_recipient, - gas_limit: config.gas_limit, - builder_proposals: config.builder_proposals, - enable_web3signer_slashing_protection: config.enable_web3signer_slashing_protection, - prefer_builder_proposals: config.prefer_builder_proposals, - builder_boost_factor: config.builder_boost_factor, - task_executor, - _phantom: PhantomData, - } - } - - /// Register all local validators in doppelganger protection to try and prevent instances of - /// duplicate validators operating on the network at the same time. - /// - /// This function has no effect if doppelganger protection is disabled. - pub fn register_all_in_doppelganger_protection_if_enabled(&self) -> Result<(), String> { - if let Some(doppelganger_service) = &self.doppelganger_service { - for pubkey in self.validators.read().iter_voting_pubkeys() { - doppelganger_service.register_new_validator::(*pubkey, &self.slot_clock)? - } - } - - Ok(()) - } - - /// Returns `true` if doppelganger protection is enabled, or else `false`. - pub fn doppelganger_protection_enabled(&self) -> bool { - self.doppelganger_service.is_some() - } - - pub fn initialized_validators(&self) -> Arc> { - self.validators.clone() - } - - /// Indicates if the `voting_public_key` exists in self and is enabled. - pub fn has_validator(&self, voting_public_key: &PublicKeyBytes) -> bool { - self.validators - .read() - .validator(voting_public_key) - .is_some() - } - - /// Insert a new validator to `self`, where the validator is represented by an EIP-2335 - /// keystore on the filesystem. - #[allow(clippy::too_many_arguments)] - pub async fn add_validator_keystore>( - &self, - voting_keystore_path: P, - password_storage: PasswordStorage, - enable: bool, - graffiti: Option, - suggested_fee_recipient: Option
, - gas_limit: Option, - builder_proposals: Option, - builder_boost_factor: Option, - prefer_builder_proposals: Option, - ) -> Result { - let mut validator_def = ValidatorDefinition::new_keystore_with_password( - voting_keystore_path, - password_storage, - graffiti.map(Into::into), - suggested_fee_recipient, - gas_limit, - builder_proposals, - builder_boost_factor, - prefer_builder_proposals, - ) - .map_err(|e| format!("failed to create validator definitions: {:?}", e))?; - - validator_def.enabled = enable; - - self.add_validator(validator_def).await - } - - /// Insert a new validator to `self`. - /// - /// This function includes: - /// - /// - Adding the validator definition to the YAML file, saving it to the filesystem. - /// - Enabling the validator with the slashing protection database. - /// - If `enable == true`, starting to perform duties for the validator. - // FIXME: ignore this clippy lint until the validator store is refactored to use async locks - #[allow(clippy::await_holding_lock)] - pub async fn add_validator( - &self, - validator_def: ValidatorDefinition, - ) -> Result { - let validator_pubkey = validator_def.voting_public_key.compress(); - - self.slashing_protection - .register_validator(validator_pubkey) - .map_err(|e| format!("failed to register validator: {:?}", e))?; - - if let Some(doppelganger_service) = &self.doppelganger_service { - doppelganger_service - .register_new_validator::(validator_pubkey, &self.slot_clock)?; - } - - self.validators - .write() - .add_definition_replace_disabled(validator_def.clone()) - .await - .map_err(|e| format!("Unable to add definition: {:?}", e))?; - - Ok(validator_def) - } - - /// Returns `ProposalData` for the provided `pubkey` if it exists in `InitializedValidators`. - /// `ProposalData` fields include defaulting logic described in `get_fee_recipient_defaulting`, - /// `get_gas_limit_defaulting`, and `get_builder_proposals_defaulting`. - pub fn proposal_data(&self, pubkey: &PublicKeyBytes) -> Option { - self.validators - .read() - .validator(pubkey) - .map(|validator| ProposalData { - validator_index: validator.get_index(), - fee_recipient: self - .get_fee_recipient_defaulting(validator.get_suggested_fee_recipient()), - gas_limit: self.get_gas_limit_defaulting(validator.get_gas_limit()), - builder_proposals: self - .get_builder_proposals_defaulting(validator.get_builder_proposals()), - }) - } +pub trait ValidatorStore: Send + Sync { + type Error: Debug + Send + Sync; + type E: EthSpec; /// Attempts to resolve the pubkey to a validator index. /// @@ -255,9 +47,7 @@ impl ValidatorStore { /// /// - Unknown. /// - Known, but with an unknown index. - pub fn validator_index(&self, pubkey: &PublicKeyBytes) -> Option { - self.validators.read().get_index(pubkey) - } + fn validator_index(&self, pubkey: &PublicKeyBytes) -> Option; /// Returns all voting pubkeys for all enabled validators. /// @@ -268,255 +58,25 @@ impl ValidatorStore { /// protection and are safe-enough to sign messages. /// - `DoppelgangerStatus::ignored`: returns all the pubkeys from `only_safe` *plus* those still /// undergoing protection. This is useful for collecting duties or other non-signing tasks. - #[allow(clippy::needless_collect)] // Collect is required to avoid holding a lock. - pub fn voting_pubkeys(&self, filter_func: F) -> I + fn voting_pubkeys(&self, filter_func: F) -> I where I: FromIterator, - F: Fn(DoppelgangerStatus) -> Option, - { - // Collect all the pubkeys first to avoid interleaving locks on `self.validators` and - // `self.doppelganger_service()`. - let pubkeys = self - .validators - .read() - .iter_voting_pubkeys() - .cloned() - .collect::>(); - - pubkeys - .into_iter() - .map(|pubkey| { - self.doppelganger_service - .as_ref() - .map(|doppelganger_service| doppelganger_service.validator_status(pubkey)) - // Allow signing on all pubkeys if doppelganger protection is disabled. - .unwrap_or_else(|| DoppelgangerStatus::SigningEnabled(pubkey)) - }) - .filter_map(filter_func) - .collect() - } - - /// Returns doppelganger statuses for all enabled validators. - #[allow(clippy::needless_collect)] // Collect is required to avoid holding a lock. - pub fn doppelganger_statuses(&self) -> Vec { - // Collect all the pubkeys first to avoid interleaving locks on `self.validators` and - // `self.doppelganger_service`. - let pubkeys = self - .validators - .read() - .iter_voting_pubkeys() - .cloned() - .collect::>(); - - pubkeys - .into_iter() - .map(|pubkey| { - self.doppelganger_service - .as_ref() - .map(|doppelganger_service| doppelganger_service.validator_status(pubkey)) - // Allow signing on all pubkeys if doppelganger protection is disabled. - .unwrap_or_else(|| DoppelgangerStatus::SigningEnabled(pubkey)) - }) - .collect() - } + F: Fn(DoppelgangerStatus) -> Option; /// Check if the `validator_pubkey` is permitted by the doppleganger protection to sign /// messages. - pub fn doppelganger_protection_allows_signing(&self, validator_pubkey: PublicKeyBytes) -> bool { - self.doppelganger_service - .as_ref() - // If there's no doppelganger service then we assume it is purposefully disabled and - // declare that all keys are safe with regard to it. - .map_or(true, |doppelganger_service| { - doppelganger_service - .validator_status(validator_pubkey) - .only_safe() - .is_some() - }) - } - - pub fn num_voting_validators(&self) -> usize { - self.validators.read().num_enabled() - } - - fn fork(&self, epoch: Epoch) -> Fork { - self.spec.fork_at_epoch(epoch) - } - - /// Returns a `SigningMethod` for `validator_pubkey` *only if* that validator is considered safe - /// by doppelganger protection. - fn doppelganger_checked_signing_method( - &self, - validator_pubkey: PublicKeyBytes, - ) -> Result, Error> { - if self.doppelganger_protection_allows_signing(validator_pubkey) { - self.validators - .read() - .signing_method(&validator_pubkey) - .ok_or(Error::UnknownPubkey(validator_pubkey)) - } else { - Err(Error::DoppelgangerProtected(validator_pubkey)) - } - } - - /// Returns a `SigningMethod` for `validator_pubkey` regardless of that validators doppelganger - /// protection status. - /// - /// ## Warning - /// - /// This method should only be used for signing non-slashable messages. - fn doppelganger_bypassed_signing_method( - &self, - validator_pubkey: PublicKeyBytes, - ) -> Result, Error> { - self.validators - .read() - .signing_method(&validator_pubkey) - .ok_or(Error::UnknownPubkey(validator_pubkey)) - } - - fn signing_context(&self, domain: Domain, signing_epoch: Epoch) -> SigningContext { - if domain == Domain::VoluntaryExit { - if self.spec.fork_name_at_epoch(signing_epoch).deneb_enabled() { - // EIP-7044 - SigningContext { - domain, - epoch: signing_epoch, - fork: Fork { - previous_version: self.spec.capella_fork_version, - current_version: self.spec.capella_fork_version, - epoch: signing_epoch, - }, - genesis_validators_root: self.genesis_validators_root, - } - } else { - SigningContext { - domain, - epoch: signing_epoch, - fork: self.fork(signing_epoch), - genesis_validators_root: self.genesis_validators_root, - } - } - } else { - SigningContext { - domain, - epoch: signing_epoch, - fork: self.fork(signing_epoch), - genesis_validators_root: self.genesis_validators_root, - } - } - } - - pub async fn randao_reveal( - &self, - validator_pubkey: PublicKeyBytes, - signing_epoch: Epoch, - ) -> Result { - let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; - let signing_context = self.signing_context(Domain::Randao, signing_epoch); - - let signature = signing_method - .get_signature::>( - SignableMessage::RandaoReveal(signing_epoch), - signing_context, - &self.spec, - &self.task_executor, - ) - .await?; - - Ok(signature) - } + fn doppelganger_protection_allows_signing(&self, validator_pubkey: PublicKeyBytes) -> bool; - pub fn graffiti(&self, validator_pubkey: &PublicKeyBytes) -> Option { - self.validators.read().graffiti(validator_pubkey) - } + fn num_voting_validators(&self) -> usize; + fn graffiti(&self, validator_pubkey: &PublicKeyBytes) -> Option; /// Returns the fee recipient for the given public key. The priority order for fetching /// the fee recipient is: /// 1. validator_definitions.yml /// 2. process level fee recipient - pub fn get_fee_recipient(&self, validator_pubkey: &PublicKeyBytes) -> Option
{ - // If there is a `suggested_fee_recipient` in the validator definitions yaml - // file, use that value. - self.get_fee_recipient_defaulting(self.suggested_fee_recipient(validator_pubkey)) - } - - pub fn get_fee_recipient_defaulting(&self, fee_recipient: Option
) -> Option
{ - // If there's nothing in the file, try the process-level default value. - fee_recipient.or(self.fee_recipient_process) - } - - /// Returns the suggested_fee_recipient from `validator_definitions.yml` if any. - /// This has been pulled into a private function so the read lock is dropped easily - fn suggested_fee_recipient(&self, validator_pubkey: &PublicKeyBytes) -> Option
{ - self.validators - .read() - .suggested_fee_recipient(validator_pubkey) - } - - /// Returns the gas limit for the given public key. The priority order for fetching - /// the gas limit is: - /// - /// 1. validator_definitions.yml - /// 2. process level gas limit - /// 3. `DEFAULT_GAS_LIMIT` - pub fn get_gas_limit(&self, validator_pubkey: &PublicKeyBytes) -> u64 { - self.get_gas_limit_defaulting(self.validators.read().gas_limit(validator_pubkey)) - } - - fn get_gas_limit_defaulting(&self, gas_limit: Option) -> u64 { - // If there is a `gas_limit` in the validator definitions yaml - // file, use that value. - gas_limit - // If there's nothing in the file, try the process-level default value. - .or(self.gas_limit) - // If there's no process-level default, use the `DEFAULT_GAS_LIMIT`. - .unwrap_or(DEFAULT_GAS_LIMIT) - } - - /// Returns a `bool` for the given public key that denotes whether this validator should use the - /// builder API. The priority order for fetching this value is: - /// - /// 1. validator_definitions.yml - /// 2. process level flag - pub fn get_builder_proposals(&self, validator_pubkey: &PublicKeyBytes) -> bool { - // If there is a `suggested_fee_recipient` in the validator definitions yaml - // file, use that value. - self.get_builder_proposals_defaulting( - self.validators.read().builder_proposals(validator_pubkey), - ) - } - - /// Returns a `u64` for the given public key that denotes the builder boost factor. The priority order for fetching this value is: - /// - /// 1. validator_definitions.yml - /// 2. process level flag - pub fn get_builder_boost_factor(&self, validator_pubkey: &PublicKeyBytes) -> Option { - self.validators - .read() - .builder_boost_factor(validator_pubkey) - .or(self.builder_boost_factor) - } - - /// Returns a `bool` for the given public key that denotes whether this validator should prefer a - /// builder payload. The priority order for fetching this value is: - /// - /// 1. validator_definitions.yml - /// 2. process level flag - pub fn get_prefer_builder_proposals(&self, validator_pubkey: &PublicKeyBytes) -> bool { - self.validators - .read() - .prefer_builder_proposals(validator_pubkey) - .unwrap_or(self.prefer_builder_proposals) - } - - fn get_builder_proposals_defaulting(&self, builder_proposals: Option) -> bool { - builder_proposals - // If there's nothing in the file, try the process-level default value. - .unwrap_or(self.builder_proposals) - } + fn get_fee_recipient(&self, validator_pubkey: &PublicKeyBytes) -> Option
; - /// Translate the per validator `builder_proposals`, `builder_boost_factor` and + /// Translate the `builder_proposals`, `builder_boost_factor` and /// `prefer_builder_proposals` to a boost factor, if available. /// - If `prefer_builder_proposals` is true, set boost factor to `u64::MAX` to indicate a /// preference for builder payloads. @@ -524,593 +84,191 @@ impl ValidatorStore { /// - If `builder_proposals` is set to false, set boost factor to 0 to indicate a preference for /// local payloads. /// - Else return `None` to indicate no preference between builder and local payloads. - pub fn determine_validator_builder_boost_factor( - &self, - validator_pubkey: &PublicKeyBytes, - ) -> Option { - let validator_prefer_builder_proposals = self - .validators - .read() - .prefer_builder_proposals(validator_pubkey); - - if matches!(validator_prefer_builder_proposals, Some(true)) { - return Some(u64::MAX); - } + fn determine_builder_boost_factor(&self, validator_pubkey: &PublicKeyBytes) -> Option; - self.validators - .read() - .builder_boost_factor(validator_pubkey) - .or_else(|| { - if matches!( - self.validators.read().builder_proposals(validator_pubkey), - Some(false) - ) { - return Some(0); - } - None - }) - } + fn randao_reveal( + &self, + validator_pubkey: PublicKeyBytes, + signing_epoch: Epoch, + ) -> impl Future>> + Send; - /// Translate the process-wide `builder_proposals`, `builder_boost_factor` and - /// `prefer_builder_proposals` configurations to a boost factor. - /// - If `prefer_builder_proposals` is true, set boost factor to `u64::MAX` to indicate a - /// preference for builder payloads. - /// - If `builder_boost_factor` is a value other than None, return its value as the boost factor. - /// - If `builder_proposals` is set to false, set boost factor to 0 to indicate a preference for - /// local payloads. - /// - Else return `None` to indicate no preference between builder and local payloads. - pub fn determine_default_builder_boost_factor(&self) -> Option { - if self.prefer_builder_proposals { - return Some(u64::MAX); - } - self.builder_boost_factor.or({ - if !self.builder_proposals { - Some(0) - } else { - None - } - }) - } + fn set_validator_index(&self, validator_pubkey: &PublicKeyBytes, index: u64); - pub async fn sign_block>( + fn sign_block( &self, validator_pubkey: PublicKeyBytes, - block: BeaconBlock, + block: UnsignedBlock, current_slot: Slot, - ) -> Result, Error> { - // Make sure the block slot is not higher than the current slot to avoid potential attacks. - if block.slot() > current_slot { - warn!( - self.log, - "Not signing block with slot greater than current slot"; - "block_slot" => block.slot().as_u64(), - "current_slot" => current_slot.as_u64() - ); - return Err(Error::GreaterThanCurrentSlot { - slot: block.slot(), - current_slot, - }); - } - - let signing_epoch = block.epoch(); - let signing_context = self.signing_context(Domain::BeaconProposer, signing_epoch); - let domain_hash = signing_context.domain_hash(&self.spec); - - let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; - - // Check for slashing conditions. - let slashing_status = if signing_method - .requires_local_slashing_protection(self.enable_web3signer_slashing_protection) - { - self.slashing_protection.check_and_insert_block_proposal( - &validator_pubkey, - &block.block_header(), - domain_hash, - ) - } else { - Ok(Safe::Valid) - }; + ) -> impl Future, Error>> + Send; - match slashing_status { - // We can safely sign this block without slashing. - Ok(Safe::Valid) => { - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_BLOCKS_TOTAL, - &[validator_metrics::SUCCESS], - ); - - let signature = signing_method - .get_signature::( - SignableMessage::BeaconBlock(&block), - signing_context, - &self.spec, - &self.task_executor, - ) - .await?; - Ok(SignedBeaconBlock::from_block(block, signature)) - } - Ok(Safe::SameData) => { - warn!( - self.log, - "Skipping signing of previously signed block"; - ); - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_BLOCKS_TOTAL, - &[validator_metrics::SAME_DATA], - ); - Err(Error::SameData) - } - Err(NotSafe::UnregisteredValidator(pk)) => { - warn!( - self.log, - "Not signing block for unregistered validator"; - "msg" => "Carefully consider running with --init-slashing-protection (see --help)", - "public_key" => format!("{:?}", pk) - ); - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_BLOCKS_TOTAL, - &[validator_metrics::UNREGISTERED], - ); - Err(Error::Slashable(NotSafe::UnregisteredValidator(pk))) - } - Err(e) => { - crit!( - self.log, - "Not signing slashable block"; - "error" => format!("{:?}", e) - ); - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_BLOCKS_TOTAL, - &[validator_metrics::SLASHABLE], - ); - Err(Error::Slashable(e)) - } - } - } - - pub async fn sign_attestation( + fn sign_attestation( &self, validator_pubkey: PublicKeyBytes, validator_committee_position: usize, - attestation: &mut Attestation, + attestation: &mut Attestation, current_epoch: Epoch, - ) -> Result<(), Error> { - // Make sure the target epoch is not higher than the current epoch to avoid potential attacks. - if attestation.data().target.epoch > current_epoch { - return Err(Error::GreaterThanCurrentEpoch { - epoch: attestation.data().target.epoch, - current_epoch, - }); - } - - // Get the signing method and check doppelganger protection. - let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; + ) -> impl Future>> + Send; - // Checking for slashing conditions. - let signing_epoch = attestation.data().target.epoch; - let signing_context = self.signing_context(Domain::BeaconAttester, signing_epoch); - let domain_hash = signing_context.domain_hash(&self.spec); - let slashing_status = if signing_method - .requires_local_slashing_protection(self.enable_web3signer_slashing_protection) - { - self.slashing_protection.check_and_insert_attestation( - &validator_pubkey, - attestation.data(), - domain_hash, - ) - } else { - Ok(Safe::Valid) - }; - - match slashing_status { - // We can safely sign this attestation. - Ok(Safe::Valid) => { - let signature = signing_method - .get_signature::>( - SignableMessage::AttestationData(attestation.data()), - signing_context, - &self.spec, - &self.task_executor, - ) - .await?; - attestation - .add_signature(&signature, validator_committee_position) - .map_err(Error::UnableToSignAttestation)?; - - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, - &[validator_metrics::SUCCESS], - ); - - Ok(()) - } - Ok(Safe::SameData) => { - warn!( - self.log, - "Skipping signing of previously signed attestation" - ); - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, - &[validator_metrics::SAME_DATA], - ); - Err(Error::SameData) - } - Err(NotSafe::UnregisteredValidator(pk)) => { - warn!( - self.log, - "Not signing attestation for unregistered validator"; - "msg" => "Carefully consider running with --init-slashing-protection (see --help)", - "public_key" => format!("{:?}", pk) - ); - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, - &[validator_metrics::UNREGISTERED], - ); - Err(Error::Slashable(NotSafe::UnregisteredValidator(pk))) - } - Err(e) => { - crit!( - self.log, - "Not signing slashable attestation"; - "attestation" => format!("{:?}", attestation.data()), - "error" => format!("{:?}", e) - ); - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_ATTESTATIONS_TOTAL, - &[validator_metrics::SLASHABLE], - ); - Err(Error::Slashable(e)) - } - } - } - - pub async fn sign_voluntary_exit( + fn sign_voluntary_exit( &self, validator_pubkey: PublicKeyBytes, voluntary_exit: VoluntaryExit, - ) -> Result { - let signing_epoch = voluntary_exit.epoch; - let signing_context = self.signing_context(Domain::VoluntaryExit, signing_epoch); - let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; - - let signature = signing_method - .get_signature::>( - SignableMessage::VoluntaryExit(&voluntary_exit), - signing_context, - &self.spec, - &self.task_executor, - ) - .await?; + ) -> impl Future>> + Send; - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_VOLUNTARY_EXITS_TOTAL, - &[validator_metrics::SUCCESS], - ); - - Ok(SignedVoluntaryExit { - message: voluntary_exit, - signature, - }) - } - - pub async fn sign_validator_registration_data( + fn sign_validator_registration_data( &self, validator_registration_data: ValidatorRegistrationData, - ) -> Result { - let domain_hash = self.spec.get_builder_domain(); - let signing_root = validator_registration_data.signing_root(domain_hash); - - let signing_method = - self.doppelganger_bypassed_signing_method(validator_registration_data.pubkey)?; - let signature = signing_method - .get_signature_from_root::>( - SignableMessage::ValidatorRegistration(&validator_registration_data), - signing_root, - &self.task_executor, - None, - ) - .await?; - - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_VALIDATOR_REGISTRATIONS_TOTAL, - &[validator_metrics::SUCCESS], - ); - - Ok(SignedValidatorRegistrationData { - message: validator_registration_data, - signature, - }) - } + ) -> impl Future>> + Send; /// Signs an `AggregateAndProof` for a given validator. /// /// The resulting `SignedAggregateAndProof` is sent on the aggregation channel and cannot be /// modified by actors other than the signing validator. - pub async fn produce_signed_aggregate_and_proof( + fn produce_signed_aggregate_and_proof( &self, validator_pubkey: PublicKeyBytes, aggregator_index: u64, - aggregate: Attestation, + aggregate: Attestation, selection_proof: SelectionProof, - ) -> Result, Error> { - let signing_epoch = aggregate.data().target.epoch; - let signing_context = self.signing_context(Domain::AggregateAndProof, signing_epoch); - - let message = - AggregateAndProof::from_attestation(aggregator_index, aggregate, selection_proof); - - let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; - let signature = signing_method - .get_signature::>( - SignableMessage::SignedAggregateAndProof(message.to_ref()), - signing_context, - &self.spec, - &self.task_executor, - ) - .await?; - - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_AGGREGATES_TOTAL, - &[validator_metrics::SUCCESS], - ); - - Ok(SignedAggregateAndProof::from_aggregate_and_proof( - message, signature, - )) - } + ) -> impl Future, Error>> + Send; /// Produces a `SelectionProof` for the `slot`, signed by with corresponding secret key to /// `validator_pubkey`. - pub async fn produce_selection_proof( + fn produce_selection_proof( &self, validator_pubkey: PublicKeyBytes, slot: Slot, - ) -> Result { - let signing_epoch = slot.epoch(E::slots_per_epoch()); - let signing_context = self.signing_context(Domain::SelectionProof, signing_epoch); - - // Bypass the `with_validator_signing_method` function. - // - // This is because we don't care about doppelganger protection when it comes to selection - // proofs. They are not slashable and we need them to subscribe to subnets on the BN. - // - // As long as we disallow `SignedAggregateAndProof` then these selection proofs will never - // be published on the network. - let signing_method = self.doppelganger_bypassed_signing_method(validator_pubkey)?; - - let signature = signing_method - .get_signature::>( - SignableMessage::SelectionProof(slot), - signing_context, - &self.spec, - &self.task_executor, - ) - .await - .map_err(Error::UnableToSign)?; - - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_SELECTION_PROOFS_TOTAL, - &[validator_metrics::SUCCESS], - ); - - Ok(signature.into()) - } + ) -> impl Future>> + Send; /// Produce a `SyncSelectionProof` for `slot` signed by the secret key of `validator_pubkey`. - pub async fn produce_sync_selection_proof( + fn produce_sync_selection_proof( &self, validator_pubkey: &PublicKeyBytes, slot: Slot, subnet_id: SyncSubnetId, - ) -> Result { - let signing_epoch = slot.epoch(E::slots_per_epoch()); - let signing_context = - self.signing_context(Domain::SyncCommitteeSelectionProof, signing_epoch); - - // Bypass `with_validator_signing_method`: sync committee messages are not slashable. - let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?; - - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_SYNC_SELECTION_PROOFS_TOTAL, - &[validator_metrics::SUCCESS], - ); + ) -> impl Future>> + Send; - let message = SyncAggregatorSelectionData { - slot, - subcommittee_index: subnet_id.into(), - }; - - let signature = signing_method - .get_signature::>( - SignableMessage::SyncSelectionProof(&message), - signing_context, - &self.spec, - &self.task_executor, - ) - .await - .map_err(Error::UnableToSign)?; - - Ok(signature.into()) - } - - pub async fn produce_sync_committee_signature( + fn produce_sync_committee_signature( &self, slot: Slot, beacon_block_root: Hash256, validator_index: u64, validator_pubkey: &PublicKeyBytes, - ) -> Result { - let signing_epoch = slot.epoch(E::slots_per_epoch()); - let signing_context = self.signing_context(Domain::SyncCommittee, signing_epoch); - - // Bypass `with_validator_signing_method`: sync committee messages are not slashable. - let signing_method = self.doppelganger_bypassed_signing_method(*validator_pubkey)?; - - let signature = signing_method - .get_signature::>( - SignableMessage::SyncCommitteeSignature { - beacon_block_root, - slot, - }, - signing_context, - &self.spec, - &self.task_executor, - ) - .await - .map_err(Error::UnableToSign)?; - - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_SYNC_COMMITTEE_MESSAGES_TOTAL, - &[validator_metrics::SUCCESS], - ); + ) -> impl Future>> + Send; - Ok(SyncCommitteeMessage { - slot, - beacon_block_root, - validator_index, - signature, - }) - } - - pub async fn produce_signed_contribution_and_proof( + fn produce_signed_contribution_and_proof( &self, aggregator_index: u64, aggregator_pubkey: PublicKeyBytes, - contribution: SyncCommitteeContribution, + contribution: SyncCommitteeContribution, selection_proof: SyncSelectionProof, - ) -> Result, Error> { - let signing_epoch = contribution.slot.epoch(E::slots_per_epoch()); - let signing_context = self.signing_context(Domain::ContributionAndProof, signing_epoch); - - // Bypass `with_validator_signing_method`: sync committee messages are not slashable. - let signing_method = self.doppelganger_bypassed_signing_method(aggregator_pubkey)?; + ) -> impl Future, Error>> + Send; - let message = ContributionAndProof { - aggregator_index, - contribution, - selection_proof: selection_proof.into(), - }; + /// Prune the slashing protection database so that it remains performant. + /// + /// This function will only do actual pruning periodically, so it should usually be + /// cheap to call. The `first_run` flag can be used to print a more verbose message when pruning + /// runs. + fn prune_slashing_protection_db(&self, current_epoch: Epoch, first_run: bool); - let signature = signing_method - .get_signature::>( - SignableMessage::SignedContributionAndProof(&message), - signing_context, - &self.spec, - &self.task_executor, - ) - .await - .map_err(Error::UnableToSign)?; + /// Returns `ProposalData` for the provided `pubkey` if it exists in `InitializedValidators`. + /// `ProposalData` fields include defaulting logic described in `get_fee_recipient_defaulting`, + /// `get_gas_limit_defaulting`, and `get_builder_proposals_defaulting`. + fn proposal_data(&self, pubkey: &PublicKeyBytes) -> Option; +} - validator_metrics::inc_counter_vec( - &validator_metrics::SIGNED_SYNC_COMMITTEE_CONTRIBUTIONS_TOTAL, - &[validator_metrics::SUCCESS], - ); +pub enum UnsignedBlock { + Full(BeaconBlock), + Blinded(BlindedBeaconBlock), +} - Ok(SignedContributionAndProof { message, signature }) +impl From> for UnsignedBlock { + fn from(block: BeaconBlock) -> Self { + UnsignedBlock::Full(block) } +} - pub fn import_slashing_protection( - &self, - interchange: Interchange, - ) -> Result<(), InterchangeError> { - self.slashing_protection - .import_interchange_info(interchange, self.genesis_validators_root)?; - Ok(()) +impl From> for UnsignedBlock { + fn from(block: BlindedBeaconBlock) -> Self { + UnsignedBlock::Blinded(block) } +} - /// Export slashing protection data while also disabling the given keys in the database. - /// - /// If any key is unknown to the slashing protection database it will be silently omitted - /// from the result. It is the caller's responsibility to check whether all keys provided - /// had data returned for them. - pub fn export_slashing_protection_for_keys( - &self, - pubkeys: &[PublicKeyBytes], - ) -> Result { - self.slashing_protection.with_transaction(|txn| { - let known_pubkeys = pubkeys - .iter() - .filter_map(|pubkey| { - let validator_id = self - .slashing_protection - .get_validator_id_ignoring_status(txn, pubkey) - .ok()?; +pub enum SignedBlock { + Full(SignedBeaconBlock), + Blinded(SignedBlindedBeaconBlock), +} - Some( - self.slashing_protection - .update_validator_status(txn, validator_id, false) - .map(|()| *pubkey), - ) - }) - .collect::, _>>()?; - self.slashing_protection.export_interchange_info_in_txn( - self.genesis_validators_root, - Some(&known_pubkeys), - txn, - ) - }) +impl From> for SignedBlock { + fn from(block: SignedBeaconBlock) -> Self { + SignedBlock::Full(block) } +} - /// Prune the slashing protection database so that it remains performant. +impl From> for SignedBlock { + fn from(block: SignedBlindedBeaconBlock) -> Self { + SignedBlock::Blinded(block) + } +} + +/// A wrapper around `PublicKeyBytes` which encodes information about the status of a validator +/// pubkey with regards to doppelganger protection. +#[derive(Debug, PartialEq)] +pub enum DoppelgangerStatus { + /// Doppelganger protection has approved this for signing. /// - /// This function will only do actual pruning periodically, so it should usually be - /// cheap to call. The `first_run` flag can be used to print a more verbose message when pruning - /// runs. - pub fn prune_slashing_protection_db(&self, current_epoch: Epoch, first_run: bool) { - // Attempt to prune every SLASHING_PROTECTION_HISTORY_EPOCHs, with a tolerance for - // missing the epoch that aligns exactly. - let mut last_prune = self.slashing_protection_last_prune.lock(); - if current_epoch / SLASHING_PROTECTION_HISTORY_EPOCHS - <= *last_prune / SLASHING_PROTECTION_HISTORY_EPOCHS - { - return; - } + /// This is because the service has waited some period of time to + /// detect other instances of this key on the network. + SigningEnabled(PublicKeyBytes), + /// Doppelganger protection is still waiting to detect other instances. + /// + /// Do not use this pubkey for signing slashable messages!! + /// + /// However, it can safely be used for other non-slashable operations (e.g., collecting duties + /// or subscribing to subnets). + SigningDisabled(PublicKeyBytes), + /// This pubkey is unknown to the doppelganger service. + /// + /// This represents a serious internal error in the program. This validator will be permanently + /// disabled! + UnknownToDoppelganger(PublicKeyBytes), +} - if first_run { - info!( - self.log, - "Pruning slashing protection DB"; - "epoch" => current_epoch, - "msg" => "pruning may take several minutes the first time it runs" - ); - } else { - info!(self.log, "Pruning slashing protection DB"; "epoch" => current_epoch); +impl DoppelgangerStatus { + /// Only return a pubkey if it is explicitly safe for doppelganger protection. + /// + /// If `Some(pubkey)` is returned, doppelganger has declared it safe for signing. + /// + /// ## Note + /// + /// "Safe" is only best-effort by doppelganger. There is no guarantee that a doppelganger + /// doesn't exist. + pub fn only_safe(self) -> Option { + match self { + DoppelgangerStatus::SigningEnabled(pubkey) => Some(pubkey), + DoppelgangerStatus::SigningDisabled(_) => None, + DoppelgangerStatus::UnknownToDoppelganger(_) => None, } + } - let _timer = - validator_metrics::start_timer(&validator_metrics::SLASHING_PROTECTION_PRUNE_TIMES); - - let new_min_target_epoch = current_epoch.saturating_sub(SLASHING_PROTECTION_HISTORY_EPOCHS); - let new_min_slot = new_min_target_epoch.start_slot(E::slots_per_epoch()); - - let all_pubkeys: Vec<_> = self.voting_pubkeys(DoppelgangerStatus::ignored); - - if let Err(e) = self - .slashing_protection - .prune_all_signed_attestations(all_pubkeys.iter(), new_min_target_epoch) - { - error!( - self.log, - "Error during pruning of signed attestations"; - "error" => ?e, - ); - return; + /// Returns a key regardless of whether or not doppelganger has approved it. Such a key might be + /// used for signing non-slashable messages, duties collection or other activities. + /// + /// If the validator is unknown to doppelganger then `None` will be returned. + pub fn ignored(self) -> Option { + match self { + DoppelgangerStatus::SigningEnabled(pubkey) => Some(pubkey), + DoppelgangerStatus::SigningDisabled(pubkey) => Some(pubkey), + DoppelgangerStatus::UnknownToDoppelganger(_) => None, } + } - if let Err(e) = self - .slashing_protection - .prune_all_signed_blocks(all_pubkeys.iter(), new_min_slot) - { - error!( - self.log, - "Error during pruning of signed blocks"; - "error" => ?e, - ); - return; + /// Only return a pubkey if it will not be used for signing due to doppelganger detection. + pub fn only_unsafe(self) -> Option { + match self { + DoppelgangerStatus::SigningEnabled(_) => None, + DoppelgangerStatus::SigningDisabled(pubkey) => Some(pubkey), + DoppelgangerStatus::UnknownToDoppelganger(pubkey) => Some(pubkey), } - - *last_prune = current_epoch; - - info!(self.log, "Completed pruning of slashing protection DB"); } }