Fix electra light client types (#6361)

* persist light client updates

* update beacon chain to serve light client updates

* resolve todos

* cache best update

* extend cache parts

* is better light client update

* resolve merge conflict

* initial api changes

* add lc update db column

* fmt

* added tests

* add sim

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* fix some weird issues with the simulator

* tests

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* test changes

* merge conflict

* testing

* started work on ef tests and some code clean up

* update tests

* linting

* noop pre altair, were still failing on electra though

* allow for zeroed light client header

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* merge unstable

* remove unwraps

* remove unwraps

* fetch bootstrap without always querying for state

* storing bootstrap parts in db

* mroe code cleanup

* test

* prune sync committee branches from dropped chains

* Update light_client_update.rs

* merge unstable

* move functionality to helper methods

* refactor is best update fn

* refactor is best update fn

* improve organization of light client server cache logic

* fork diget calc, and only spawn as many blcoks as we need for the lc update test

* resovle merge conflict

* add electra bootstrap logic, add logic to cache current sync committee

* add latest sync committe branch cache

* fetch lc update from the cache if it exists

* fmt

* Fix beacon_chain tests

* Add debug code to update ranking_order ef test

* Fix compare code

* merge conflicts

* merge conflict

* add better error messaging

* resolve merge conflicts

* remove lc update from basicsim

* rename sync comittte variable and fix persist condition

* refactor get_light_client_update logic

* add better comments, return helpful error messages over http and rpc

* pruning canonical non checkpoint slots

* fix test

* rerun test

* update pruning logic, add tests

* fix tests

* fix imports

* fmt

* refactor db code

* Refactor db method

* Refactor db method

* lc electra changes

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* add additional comments

* testing lc merkle changes

* lc electra

* update struct defs

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* fix merge

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-bootstrap

* fix merge

* linting

* merge conflict

* prevent overflow

* enable lc server for http api tests

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* get tests working:

* remove related TODOs

* fix test lint

* Merge branch 'persist-light-client-bootstrap' of https://github.com/eserilev/lighthouse into light-client-electra

* fix tests

* fix conflicts

* remove prints

* Merge branch 'persist-light-client-bootstrap' of https://github.com/eserilev/lighthouse into light-client-electra

* remove warning

* resolve conflicts

* merge conflicts

* linting

* remove comments

* cleanup

* linting

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* pre/post electra light client cached data

* add proof type alias

* move is_empty_branch method out of impl

* add ssz tests for all forks

* refactor beacon state proof codepaths

* rename method

* fmt

* clean up proof logic

* refactor merkle proof api

* fmt

* Merge branch 'unstable' into light-client-electra

* Use superstruct mapping macros

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into light-client-electra

* rename proof to merkleproof

* fmt

* Resolve merge conflicts

* merge conflicts

beacon_node/beacon_chain/src/light_client_server_cache.rs

@@ -1,25 +1,19 @@
 use crate::errors::BeaconChainError;
 use crate::{metrics, BeaconChainTypes, BeaconStore};
-use eth2::types::light_client_update::CurrentSyncCommitteeProofLen;
 use parking_lot::{Mutex, RwLock};
 use safe_arith::SafeArith;
 use slog::{debug, Logger};
 use ssz::Decode;
-use ssz_types::FixedVector;
 use std::num::NonZeroUsize;
 use std::sync::Arc;
 use store::DBColumn;
 use store::KeyValueStore;
 use tree_hash::TreeHash;
-use types::light_client_update::{
-    FinalizedRootProofLen, NextSyncCommitteeProofLen, CURRENT_SYNC_COMMITTEE_INDEX,
-    FINALIZED_ROOT_INDEX, NEXT_SYNC_COMMITTEE_INDEX,
-};
 use types::non_zero_usize::new_non_zero_usize;
 use types::{
     BeaconBlockRef, BeaconState, ChainSpec, Checkpoint, EthSpec, ForkName, Hash256,
     LightClientBootstrap, LightClientFinalityUpdate, LightClientOptimisticUpdate,
-    LightClientUpdate, Slot, SyncAggregate, SyncCommittee,
+    LightClientUpdate, MerkleProof, Slot, SyncAggregate, SyncCommittee,
 };
 
 /// A prev block cache miss requires to re-generate the state of the post-parent block. Items in the
@@ -69,17 +63,14 @@ impl<T: BeaconChainTypes> LightClientServerCache<T> {
         block_post_state: &mut BeaconState<T::EthSpec>,
     ) -> Result<(), BeaconChainError> {
         let _timer = metrics::start_timer(&metrics::LIGHT_CLIENT_SERVER_CACHE_STATE_DATA_TIMES);
-
+        let fork_name = spec.fork_name_at_slot::<T::EthSpec>(block.slot());
         // Only post-altair
-        if spec.fork_name_at_slot::<T::EthSpec>(block.slot()) == ForkName::Base {
-            return Ok(());
+        if fork_name.altair_enabled() {
+            // Persist in memory cache for a descendent block
+            let cached_data = LightClientCachedData::from_state(block_post_state)?;
+            self.prev_block_cache.lock().put(block_root, cached_data);
         }
 
-        // Persist in memory cache for a descendent block
-
-        let cached_data = LightClientCachedData::from_state(block_post_state)?;
-        self.prev_block_cache.lock().put(block_root, cached_data);
-
         Ok(())
     }
 
@@ -413,34 +404,31 @@ impl<T: BeaconChainTypes> Default for LightClientServerCache<T> {
     }
 }
 
-type FinalityBranch = FixedVector<Hash256, FinalizedRootProofLen>;
-type NextSyncCommitteeBranch = FixedVector<Hash256, NextSyncCommitteeProofLen>;
-type CurrentSyncCommitteeBranch = FixedVector<Hash256, CurrentSyncCommitteeProofLen>;
-
 #[derive(Clone)]
 struct LightClientCachedData<E: EthSpec> {
     finalized_checkpoint: Checkpoint,
-    finality_branch: FinalityBranch,
-    next_sync_committee_branch: NextSyncCommitteeBranch,
-    current_sync_committee_branch: CurrentSyncCommitteeBranch,
+    finality_branch: MerkleProof,
+    next_sync_committee_branch: MerkleProof,
+    current_sync_committee_branch: MerkleProof,
     next_sync_committee: Arc<SyncCommittee<E>>,
     current_sync_committee: Arc<SyncCommittee<E>>,
     finalized_block_root: Hash256,
 }
 
 impl<E: EthSpec> LightClientCachedData<E> {
     fn from_state(state: &mut BeaconState<E>) -> Result<Self, BeaconChainError> {
+        let (finality_branch, next_sync_committee_branch, current_sync_committee_branch) = (
+            state.compute_finalized_root_proof()?,
+            state.compute_current_sync_committee_proof()?,
+            state.compute_next_sync_committee_proof()?,
+        );
         Ok(Self {
             finalized_checkpoint: state.finalized_checkpoint(),
-            finality_branch: state.compute_merkle_proof(FINALIZED_ROOT_INDEX)?.into(),
+            finality_branch,
             next_sync_committee: state.next_sync_committee()?.clone(),
             current_sync_committee: state.current_sync_committee()?.clone(),
-            next_sync_committee_branch: state
-                .compute_merkle_proof(NEXT_SYNC_COMMITTEE_INDEX)?
-                .into(),
-            current_sync_committee_branch: state
-                .compute_merkle_proof(CURRENT_SYNC_COMMITTEE_INDEX)?
-                .into(),
+            next_sync_committee_branch,
+            current_sync_committee_branch,
             finalized_block_root: state.finalized_checkpoint().root,
         })
     }

beacon_node/beacon_chain/tests/store_tests.rs

@@ -206,13 +206,6 @@ async fn light_client_bootstrap_test() {
         .build()
         .expect("should build");
 
-    let current_state = harness.get_current_state();
-
-    if ForkName::Electra == current_state.fork_name_unchecked() {
-        // TODO(electra) fix beacon state `compute_merkle_proof`
-        return;
-    }
-
     let finalized_checkpoint = beacon_chain
         .canonical_head
         .cached_head()
@@ -353,11 +346,6 @@ async fn light_client_updates_test() {
 
     let current_state = harness.get_current_state();
 
-    if ForkName::Electra == current_state.fork_name_unchecked() {
-        // TODO(electra) fix beacon state `compute_merkle_proof`
-        return;
-    }
-
     // calculate the sync period from the previous slot
     let sync_period = (current_state.slot() - Slot::new(1))
         .epoch(E::slots_per_epoch())

beacon_node/store/src/hot_cold_store.rs

@@ -44,7 +44,6 @@ use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 use types::data_column_sidecar::{ColumnIndex, DataColumnSidecar, DataColumnSidecarList};
-use types::light_client_update::CurrentSyncCommitteeProofLen;
 use types::*;
 
 /// On-disk database that stores finalized states efficiently.
@@ -641,15 +640,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     pub fn get_sync_committee_branch(
         &self,
         block_root: &Hash256,
-    ) -> Result<Option<FixedVector<Hash256, CurrentSyncCommitteeProofLen>>, Error> {
+    ) -> Result<Option<MerkleProof>, Error> {
         let column = DBColumn::SyncCommitteeBranch;
 
         if let Some(bytes) = self
             .hot_db
             .get_bytes(column.into(), &block_root.as_ssz_bytes())?
         {
-            let sync_committee_branch: FixedVector<Hash256, CurrentSyncCommitteeProofLen> =
-                FixedVector::from_ssz_bytes(&bytes)?;
+            let sync_committee_branch = Vec::<Hash256>::from_ssz_bytes(&bytes)?;
             return Ok(Some(sync_committee_branch));
         }
 
@@ -677,7 +675,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     pub fn store_sync_committee_branch(
         &self,
         block_root: Hash256,
-        sync_committee_branch: &FixedVector<Hash256, CurrentSyncCommitteeProofLen>,
+        sync_committee_branch: &MerkleProof,
     ) -> Result<(), Error> {
         let column = DBColumn::SyncCommitteeBranch;
         self.hot_db.put_bytes(

consensus/types/src/beacon_state.rs

@@ -2506,33 +2506,64 @@ impl<E: EthSpec> BeaconState<E> {
         Ok(())
     }
 
-    pub fn compute_merkle_proof(&self, generalized_index: usize) -> Result<Vec<Hash256>, Error> {
-        // 1. Convert generalized index to field index.
-        let field_index = match generalized_index {
+    pub fn compute_current_sync_committee_proof(&self) -> Result<Vec<Hash256>, Error> {
+        // Sync committees are top-level fields, subtract off the generalized indices
+        // for the internal nodes. Result should be 22 or 23, the field offset of the committee
+        // in the `BeaconState`:
+        // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
+        let field_index = if self.fork_name_unchecked().electra_enabled() {
+            light_client_update::CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA
+        } else {
             light_client_update::CURRENT_SYNC_COMMITTEE_INDEX
-            | light_client_update::NEXT_SYNC_COMMITTEE_INDEX => {
-                // Sync committees are top-level fields, subtract off the generalized indices
-                // for the internal nodes. Result should be 22 or 23, the field offset of the committee
-                // in the `BeaconState`:
-                // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
-                generalized_index
-                    .checked_sub(self.num_fields_pow2())
-                    .ok_or(Error::IndexNotSupported(generalized_index))?
-            }
-            light_client_update::FINALIZED_ROOT_INDEX => {
-                // Finalized root is the right child of `finalized_checkpoint`, divide by two to get
-                // the generalized index of `state.finalized_checkpoint`.
-                let finalized_checkpoint_generalized_index = generalized_index / 2;
-                // Subtract off the internal nodes. Result should be 105/2 - 32 = 20 which matches
-                // position of `finalized_checkpoint` in `BeaconState`.
-                finalized_checkpoint_generalized_index
-                    .checked_sub(self.num_fields_pow2())
-                    .ok_or(Error::IndexNotSupported(generalized_index))?
-            }
-            _ => return Err(Error::IndexNotSupported(generalized_index)),
         };
+        let leaves = self.get_beacon_state_leaves();
+        self.generate_proof(field_index, &leaves)
+    }
 
-        // 2. Get all `BeaconState` leaves.
+    pub fn compute_next_sync_committee_proof(&self) -> Result<Vec<Hash256>, Error> {
+        // Sync committees are top-level fields, subtract off the generalized indices
+        // for the internal nodes. Result should be 22 or 23, the field offset of the committee
+        // in the `BeaconState`:
+        // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
+        let field_index = if self.fork_name_unchecked().electra_enabled() {
+            light_client_update::NEXT_SYNC_COMMITTEE_INDEX_ELECTRA
+        } else {
+            light_client_update::NEXT_SYNC_COMMITTEE_INDEX
+        };
+        let leaves = self.get_beacon_state_leaves();
+        self.generate_proof(field_index, &leaves)
+    }
+
+    pub fn compute_finalized_root_proof(&self) -> Result<Vec<Hash256>, Error> {
+        // Finalized root is the right child of `finalized_checkpoint`, divide by two to get
+        // the generalized index of `state.finalized_checkpoint`.
+        let field_index = if self.fork_name_unchecked().electra_enabled() {
+            // Index should be 169/2 - 64 = 20 which matches the position
+            // of `finalized_checkpoint` in `BeaconState`
+            light_client_update::FINALIZED_ROOT_INDEX_ELECTRA
+        } else {
+            // Index should be 105/2 - 32 = 20 which matches the position
+            // of `finalized_checkpoint` in `BeaconState`
+            light_client_update::FINALIZED_ROOT_INDEX
+        };
+        let leaves = self.get_beacon_state_leaves();
+        let mut proof = self.generate_proof(field_index, &leaves)?;
+        proof.insert(0, self.finalized_checkpoint().epoch.tree_hash_root());
+        Ok(proof)
+    }
+
+    fn generate_proof(
+        &self,
+        field_index: usize,
+        leaves: &[Hash256],
+    ) -> Result<Vec<Hash256>, Error> {
+        let depth = self.num_fields_pow2().ilog2() as usize;
+        let tree = merkle_proof::MerkleTree::create(leaves, depth);
+        let (_, proof) = tree.generate_proof(field_index, depth)?;
+        Ok(proof)
+    }
+
+    fn get_beacon_state_leaves(&self) -> Vec<Hash256> {
         let mut leaves = vec![];
         #[allow(clippy::arithmetic_side_effects)]
         match self {
@@ -2568,18 +2599,7 @@ impl<E: EthSpec> BeaconState<E> {
             }
         };
 
-        // 3. Make deposit tree.
-        // Use the depth of the `BeaconState` fields (i.e. `log2(32) = 5`).
-        let depth = light_client_update::CURRENT_SYNC_COMMITTEE_PROOF_LEN;
-        let tree = merkle_proof::MerkleTree::create(&leaves, depth);
-        let (_, mut proof) = tree.generate_proof(field_index, depth)?;
-
-        // 4. If we're proving the finalized root, patch in the finalized epoch to complete the proof.
-        if generalized_index == light_client_update::FINALIZED_ROOT_INDEX {
-            proof.insert(0, self.finalized_checkpoint().epoch.tree_hash_root());
-        }
-
-        Ok(proof)
+        leaves
     }
 }
 

consensus/types/src/lib.rs

@@ -200,7 +200,7 @@ pub use crate::light_client_optimistic_update::{
 };
 pub use crate::light_client_update::{
     Error as LightClientUpdateError, LightClientUpdate, LightClientUpdateAltair,
-    LightClientUpdateCapella, LightClientUpdateDeneb, LightClientUpdateElectra,
+    LightClientUpdateCapella, LightClientUpdateDeneb, LightClientUpdateElectra, MerkleProof,
 };
 pub use crate::participation_flags::ParticipationFlags;
 pub use crate::payload::{

consensus/types/src/light_client_bootstrap.rs

@@ -57,7 +57,16 @@ pub struct LightClientBootstrap<E: EthSpec> {
     /// The `SyncCommittee` used in the requested period.
     pub current_sync_committee: Arc<SyncCommittee<E>>,
     /// Merkle proof for sync committee
+    #[superstruct(
+        only(Altair, Capella, Deneb),
+        partial_getter(rename = "current_sync_committee_branch_altair")
+    )]
     pub current_sync_committee_branch: FixedVector<Hash256, CurrentSyncCommitteeProofLen>,
+    #[superstruct(
+        only(Electra),
+        partial_getter(rename = "current_sync_committee_branch_electra")
+    )]
+    pub current_sync_committee_branch: FixedVector<Hash256, CurrentSyncCommitteeProofLenElectra>,
 }
 
 impl<E: EthSpec> LightClientBootstrap<E> {
@@ -115,7 +124,7 @@ impl<E: EthSpec> LightClientBootstrap<E> {
     pub fn new(
         block: &SignedBlindedBeaconBlock<E>,
         current_sync_committee: Arc<SyncCommittee<E>>,
-        current_sync_committee_branch: FixedVector<Hash256, CurrentSyncCommitteeProofLen>,
+        current_sync_committee_branch: Vec<Hash256>,
         chain_spec: &ChainSpec,
     ) -> Result<Self, Error> {
         let light_client_bootstrap = match block
@@ -126,22 +135,22 @@ impl<E: EthSpec> LightClientBootstrap<E> {
             ForkName::Altair | ForkName::Bellatrix => Self::Altair(LightClientBootstrapAltair {
                 header: LightClientHeaderAltair::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
             ForkName::Capella => Self::Capella(LightClientBootstrapCapella {
                 header: LightClientHeaderCapella::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
             ForkName::Deneb => Self::Deneb(LightClientBootstrapDeneb {
                 header: LightClientHeaderDeneb::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
             ForkName::Electra => Self::Electra(LightClientBootstrapElectra {
                 header: LightClientHeaderElectra::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
         };
 
@@ -155,9 +164,7 @@ impl<E: EthSpec> LightClientBootstrap<E> {
     ) -> Result<Self, Error> {
         let mut header = beacon_state.latest_block_header().clone();
         header.state_root = beacon_state.update_tree_hash_cache()?;
-        let current_sync_committee_branch =
-            FixedVector::new(beacon_state.compute_merkle_proof(CURRENT_SYNC_COMMITTEE_INDEX)?)?;
-
+        let current_sync_committee_branch = beacon_state.compute_current_sync_committee_proof()?;
         let current_sync_committee = beacon_state.current_sync_committee()?.clone();
 
         let light_client_bootstrap = match block
@@ -168,22 +175,22 @@ impl<E: EthSpec> LightClientBootstrap<E> {
             ForkName::Altair | ForkName::Bellatrix => Self::Altair(LightClientBootstrapAltair {
                 header: LightClientHeaderAltair::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
             ForkName::Capella => Self::Capella(LightClientBootstrapCapella {
                 header: LightClientHeaderCapella::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
             ForkName::Deneb => Self::Deneb(LightClientBootstrapDeneb {
                 header: LightClientHeaderDeneb::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
             ForkName::Electra => Self::Electra(LightClientBootstrapElectra {
                 header: LightClientHeaderElectra::block_to_light_client_header(block)?,
                 current_sync_committee,
-                current_sync_committee_branch,
+                current_sync_committee_branch: current_sync_committee_branch.into(),
             }),
         };
 
@@ -210,8 +217,28 @@ impl<E: EthSpec> ForkVersionDeserialize for LightClientBootstrap<E> {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-    use crate::MainnetEthSpec;
+    // `ssz_tests!` can only be defined once per namespace
+    #[cfg(test)]
+    mod altair {
+        use crate::{LightClientBootstrapAltair, MainnetEthSpec};
+        ssz_tests!(LightClientBootstrapAltair<MainnetEthSpec>);
+    }
+
+    #[cfg(test)]
+    mod capella {
+        use crate::{LightClientBootstrapCapella, MainnetEthSpec};
+        ssz_tests!(LightClientBootstrapCapella<MainnetEthSpec>);
+    }
 
-    ssz_tests!(LightClientBootstrapDeneb<MainnetEthSpec>);
+    #[cfg(test)]
+    mod deneb {
+        use crate::{LightClientBootstrapDeneb, MainnetEthSpec};
+        ssz_tests!(LightClientBootstrapDeneb<MainnetEthSpec>);
+    }
+
+    #[cfg(test)]
+    mod electra {
+        use crate::{LightClientBootstrapElectra, MainnetEthSpec};
+        ssz_tests!(LightClientBootstrapElectra<MainnetEthSpec>);
+    }
 }