STR-949 chainstate manager v2 and minor chainstate restructures

crates/chaintsn/src/transition.rs

@@ -33,9 +33,10 @@ use crate::{
 /// plays out all the updates a block makes to the chain, but it will abort if
 /// there are any semantic issues that don't make sense.
 ///
-/// This operates on a state cache that's expected to be empty, panics
-/// otherwise.  Does not check the `state_root` in the header for correctness,
-/// so that can be unset so it can be use during block assembly.
+/// This operates on a state cache that's expected to be empty, may panic if
+/// changes have been made, although this is not guaranteed.  Does not check the
+/// `state_root` in the header for correctness, so that can be unset so it can
+/// be use during block assembly.
 pub fn process_block(
     state: &mut StateCache,
     header: &impl L2Header,

crates/db/src/traits.rs

@@ -194,20 +194,27 @@ pub enum BlockStatus {
     Invalid,
 }
 
-/// Db trait for the (consensus layer) chain state database.  For now we only
-/// have a modestly sized "toplevel" chain state and no "large" state like the
-/// EL does.  This trait is designed to permit a change to storing larger state
-/// like that in the future without *too* much extra effort.  We decide new
-/// states by providing the database with a generic "write batch" and offloading
-/// the effort of deciding how to compute that write batch to the database impl.
+/// Low-level Strata chainstate database.  This provides the basic interface for
+/// storing and fetching write batches and toplevel states on disk.
+///
+/// Currently we do not have a "bulk" state that we would want to avoid storing
+/// in memory all at once.  In the future, we expect that this interface would
+/// be extended to expose a "finalized" state that's fully materialized, along
+/// with functions to walk the finalized state forwards and backwards.  We can
+/// use the unmerged write batches to construct a view of more recent states
+/// than the fully materialized state in-memory.
+///
+/// For now, the full state is just the "toplevel" state that can always be
+/// expected to be of moderate size in memory.
+// TODO maybe rewrite this around storing write batches according to blkid?
 pub trait ChainstateDatabase {
     /// Writes the genesis chainstate at index 0.
-    fn write_genesis_state(&self, toplevel: &Chainstate) -> DbResult<()>;
+    fn write_genesis_state(&self, toplevel: Chainstate) -> DbResult<()>;
 
     /// Stores a write batch in the database, possibly computing that state
     /// under the hood from the writes.  Will not overwrite existing data,
     /// previous writes must be purged first in order to be replaced.
-    fn write_state_update(&self, idx: u64, batch: &WriteBatch) -> DbResult<()>;
+    fn write_state_update(&self, idx: u64, batch: WriteBatch) -> DbResult<()>;
 
     /// Tells the database to purge state before a certain block index (height).
     fn purge_historical_state_before(&self, before_idx: u64) -> DbResult<()>;
@@ -225,7 +232,8 @@ pub trait ChainstateDatabase {
     /// Gets the write batch stored to compute a height.
     fn get_writes_at(&self, idx: u64) -> DbResult<Option<WriteBatch>>;
 
-    /// Gets the toplevel chain state at a particular block index (height).
+    /// Gets the toplevel chain state at a particular block slot, if it can be
+    /// retrieved.
     fn get_toplevel_state(&self, idx: u64) -> DbResult<Option<Chainstate>>;
 }
 

crates/primitives/src/epoch.rs

@@ -0,0 +1,109 @@
+//! Types relating to epoch bookkeeping.
+//!
+//! An epoch of a range of sequential blocks defined by the terminal block of
+//! the epoch going back to (but not including) the terminal block of a previous
+//! epoch.  This uniquely identifies the epoch's final state indirectly,
+//! although it's possible for conflicting epochs with different terminal blocks
+//! to exist in theory, depending on the consensus algorithm.
+//!
+//! Epochs are *usually* always the same number of slots, but we're not
+//! guaranteeing this yet, so we always include both the epoch number and slot
+//! number of the terminal block.
+//!
+//! We also have a sentinel "null" epoch used to refer to the "finalized epoch"
+//! as of the genesis block.
+
+use arbitrary::Arbitrary;
+use borsh::{BorshDeserialize, BorshSerialize};
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    buf::Buf32,
+    l2::{L2BlockCommitment, L2BlockId},
+};
+
+/// Commits to a particular epoch by the last block and slot.
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    Arbitrary,
+    BorshDeserialize,
+    BorshSerialize,
+    Deserialize,
+    Serialize,
+)]
+pub struct EpochCommitment {
+    epoch: u64,
+    last_slot: u64,
+    last_blkid: L2BlockId,
+}
+
+impl EpochCommitment {
+    pub fn new(epoch: u64, last_slot: u64, last_blkid: L2BlockId) -> Self {
+        Self {
+            epoch,
+            last_slot,
+            last_blkid,
+        }
+    }
+
+    /// Creates a "null" epoch with
+    pub fn null() -> Self {
+        Self::new(0, 0, L2BlockId::from(Buf32::zero()))
+    }
+
+    pub fn epoch(&self) -> u64 {
+        self.epoch
+    }
+
+    pub fn last_slot(&self) -> u64 {
+        self.last_slot
+    }
+
+    pub fn last_blkid(&self) -> &L2BlockId {
+        &self.last_blkid
+    }
+
+    /// Returns a [`L2BlockCommitment`] for the final block of the epoch.
+    pub fn to_block_commitment(&self) -> L2BlockCommitment {
+        L2BlockCommitment::new(self.last_slot, self.last_blkid)
+    }
+
+    /// Returns if the terminal blkid is zero.
+    pub fn is_null(&self) -> bool {
+        Buf32::from(self.last_blkid).is_zero()
+    }
+
+    /// Checks if the epoch is sane.
+    ///
+    /// Ie. if the terminal blkid is zero then the last slot and epoch number
+    /// are also zero.
+    fn sanity_check(&self) -> bool {
+        if self.is_null() {
+            self.last_slot == 0 && self.epoch == 0
+        } else {
+            self.last_slot != 0
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::EpochCommitment;
+
+    #[test]
+    fn test_epoch_sanity() {
+        // TODO write test
+    }
+
+    #[test]
+    fn test_epoch_insanity() {
+        // TODO write test
+    }
+}

crates/primitives/src/l1/block.rs

@@ -45,6 +45,39 @@ impl From<L1BlockId> for BlockHash {
     }
 }
 
+#[derive(
+    Copy,
+    Clone,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    Arbitrary,
+    BorshDeserialize,
+    BorshSerialize,
+    Deserialize,
+    Serialize,
+)]
+pub struct L1BlockCommitment {
+    height: u64,
+    blkid: L1BlockId,
+}
+
+impl L1BlockCommitment {
+    pub fn new(height: u64, blkid: L1BlockId) -> Self {
+        Self { height, blkid }
+    }
+
+    pub fn height(&self) -> u64 {
+        self.height
+    }
+
+    pub fn blkid(&self) -> &L1BlockId {
+        &self.blkid
+    }
+}
+
 /// Reference to a transaction in a block.  This is the block index and the
 /// position of the transaction in the block.
 #[derive(
@@ -68,6 +101,7 @@ impl L1TxRef {
     pub fn blk_idx(&self) -> u64 {
         self.0
     }
+
     pub fn position(&self) -> u32 {
         self.1
     }

crates/primitives/src/l2.rs

@@ -23,3 +23,38 @@ use crate::{buf::Buf32, impl_buf_wrapper};
 pub struct L2BlockId(Buf32);
 
 impl_buf_wrapper!(L2BlockId, Buf32, 32);
+
+/// Commits to a specific block at some slot.
+#[derive(
+    Copy,
+    Clone,
+    Debug,
+    Eq,
+    PartialEq,
+    Ord,
+    PartialOrd,
+    Hash,
+    Arbitrary,
+    BorshDeserialize,
+    BorshSerialize,
+    Deserialize,
+    Serialize,
+)]
+pub struct L2BlockCommitment {
+    slot: u64,
+    blkid: L2BlockId,
+}
+
+impl L2BlockCommitment {
+    pub fn new(slot: u64, blkid: L2BlockId) -> Self {
+        Self { slot, blkid }
+    }
+
+    pub fn slot(&self) -> u64 {
+        self.slot
+    }
+
+    pub fn blkid(&self) -> &L2BlockId {
+        &self.blkid
+    }
+}

crates/primitives/src/lib.rs

@@ -14,6 +14,7 @@ pub mod l1;
 pub mod l2;
 #[macro_use]
 mod macros;
+pub mod epoch;
 pub mod keys;
 pub mod operator;
 pub mod params;

crates/rocksdb-store/src/chain_state/db.rs

@@ -56,22 +56,19 @@ impl ChainstateDatabase for ChainstateDb {
         Ok(self.db.get::<ChainstateSchema>(&idx)?)
     }
 
-    fn write_genesis_state(
-        &self,
-        toplevel: &strata_state::chain_state::Chainstate,
-    ) -> DbResult<()> {
+    fn write_genesis_state(&self, toplevel: strata_state::chain_state::Chainstate) -> DbResult<()> {
         let genesis_key = 0;
         if self.get_first_idx()?.is_some() || self.get_last_idx()?.is_some() {
             return Err(DbError::OverwriteStateUpdate(genesis_key));
         }
-        self.db.put::<ChainstateSchema>(&genesis_key, toplevel)?;
+        self.db.put::<ChainstateSchema>(&genesis_key, &toplevel)?;
         Ok(())
     }
 
     fn write_state_update(
         &self,
         idx: u64,
-        batch: &strata_state::state_op::WriteBatch,
+        batch: strata_state::state_op::WriteBatch,
     ) -> DbResult<()> {
         if self.db.get::<WriteBatchSchema>(&idx)?.is_some() {
             return Err(DbError::OverwriteStateUpdate(idx));
@@ -82,10 +79,10 @@ impl ChainstateDatabase for ChainstateDb {
             Some(state) => state,
             None => return Err(DbError::OooInsert("Chainstate", idx)),
         };
-        let post_state = state_op::apply_write_batch_to_chainstate(pre_state, batch);
+        let post_state = state_op::apply_write_batch_to_chainstate(pre_state, &batch);
 
         let mut write_batch = SchemaBatch::new();
-        write_batch.put::<WriteBatchSchema>(&idx, batch)?;
+        write_batch.put::<WriteBatchSchema>(&idx, &batch)?;
         write_batch.put::<ChainstateSchema>(&idx, &post_state)?;
         self.db.write_schemas(write_batch)?;