Merge branch 'unstable' into dev/pedro/unified_changes

beacon_chain/networking/README.md

@@ -1,10 +1,10 @@
 # Networking
 
-This folders hold a collection of modules to:
+These folders hold a collection of modules to:
 - configure the Eth2 P2P network
 - discover, connect, and maintain quality Eth2 peers
 
-Data received is handed other to the `../gossip_processing` modules for validation.
+Data received is handed over to the `../gossip_processing` modules for validation.
 
 ## Security concerns
 

beacon_chain/spec/datatypes/constants.nim

@@ -86,3 +86,10 @@ const
 
   # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.5/specs/electra/beacon-chain.md#withdrawal-prefixes
   COMPOUNDING_WITHDRAWAL_PREFIX* = 0x02
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/beacon-chain.md#execution-1
+  MAX_BLOBS_PER_BLOCK_ELECTRA* = 9'u64
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/p2p-interface.md#configuration
+  MAX_REQUEST_BLOB_SIDECARS_ELECTRA* =
+    MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_ELECTRA

beacon_chain/spec/network.nim

@@ -37,7 +37,7 @@ const
   # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.9/specs/_features/eip7594/p2p-interface.md#configuration
   MAX_REQUEST_DATA_COLUMN_SIDECARS*: uint64 =
     MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS
-  
+
   defaultEth2TcpPort* = 9000
   defaultEth2TcpPortDesc* = $defaultEth2TcpPort
 

beacon_chain/spec/presets.nim

@@ -824,6 +824,8 @@ proc readRuntimeConfig*(
   checkCompatibility MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK,
                      "MAX_REQUEST_BLOB_SIDECARS"
   checkCompatibility BLOB_SIDECAR_SUBNET_COUNT
+  checkCompatibility MAX_BLOBS_PER_BLOCK_ELECTRA
+  checkCompatibility MAX_REQUEST_BLOB_SIDECARS_ELECTRA
 
   # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/phase0/fork-choice.md#configuration
   # Isn't being used as a preset in the usual way: at any time, there's one correct value

beacon_chain/sync/sync_protocol.nim

@@ -91,10 +91,10 @@ proc readChunkPayload*(
     await conn.readExactly(addr contextBytes, sizeof contextBytes)
   except CatchableError:
     return neterr UnexpectedEOF
-  let contextFork = 
+  let contextFork =
     peer.network.forkDigests[].consensusForkForDigest(contextBytes).valueOr:
       return neterr InvalidContextBytes
-  
+
   withConsensusFork(contextFork):
     when consensusFork >= ConsensusFork.Fulu:
       let res = await readChunkPayload(conn, peer, DataColumnSidecar)
@@ -107,6 +107,99 @@ proc readChunkPayload*(
 
 {.pop.} # TODO fix p2p macro for raises
 
+template getBlobSidecarsByRoot(
+    versionNumber: static string, peer: Peer, dag: ChainDAGRef, response: auto,
+    blobIds: BlobIdentifierList) =
+  trace "got v" & versionNumber & " blobs range request",
+    peer, len = blobIds.len
+  if blobIds.len == 0:
+    raise newException(InvalidInputsError, "No blobs requested")
+
+  let count = blobIds.len
+
+  var
+    found = 0
+    bytes: seq[byte]
+
+  for i in 0..<count:
+    let blockRef = dag.getBlockRef(blobIds[i].block_root).valueOr:
+      continue
+    let index = blobIds[i].index
+    if dag.db.getBlobSidecarSZ(blockRef.bid.root, index, bytes):
+      let uncompressedLen = uncompressedLenFramed(bytes).valueOr:
+        warn "Cannot read blob size, database corrupt?",
+          bytes = bytes.len(), blck = shortLog(blockRef), blobindex = index
+        continue
+
+      peer.awaitQuota(
+        blobResponseCost, "blob_sidecars_by_root/" & versionNumber)
+      peer.network.awaitQuota(
+        blobResponseCost, "blob_sidecars_by_root/" & versionNumber)
+
+      await response.writeBytesSZ(
+        uncompressedLen, bytes,
+        peer.network.forkDigestAtEpoch(blockRef.slot.epoch).data)
+      inc found
+
+  debug "Blob root v" & versionNumber & " request done",
+    peer, roots = blobIds.len, count, found
+
+template getBlobSidecarsByRange(
+    versionNumber: static string, peer: Peer, dag: ChainDAGRef, response: auto,
+    startSlot: Slot, reqCount: uint64, blobsPerBlock: static uint64,
+    maxReqSidecars: static uint64) =
+  trace "got v" & versionNumber & " blobs range request",
+    peer, startSlot, count = reqCount
+  if reqCount == 0:
+    raise newException(InvalidInputsError, "Empty range requested")
+
+  let epochBoundary =
+    if dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS >= dag.head.slot.epoch:
+      GENESIS_EPOCH
+    else:
+      dag.head.slot.epoch - dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
+
+  if startSlot.epoch < epochBoundary:
+    raise newException(ResourceUnavailableError, BlobsOutOfRange)
+
+  var blockIds: array[int(maxReqSidecars), BlockId]
+  let
+    count = int min(reqCount, blockIds.lenu64)
+    endIndex = count - 1
+    startIndex =
+      dag.getBlockRange(startSlot, 1, blockIds.toOpenArray(0, endIndex))
+
+  var
+    found = 0
+    bytes: seq[byte]
+
+  for i in startIndex..endIndex:
+    for j in 0..<blobsPerBlock:
+      if dag.db.getBlobSidecarSZ(blockIds[i].root, BlobIndex(j), bytes):
+        if not dag.head.executionValid:
+          continue
+
+        let uncompressedLen = uncompressedLenFramed(bytes).valueOr:
+          warn "Cannot read blobs sidecar size, database corrupt?",
+            bytes = bytes.len(), blck = shortLog(blockIds[i])
+          continue
+
+        # TODO extract from libp2pProtocol
+        peer.awaitQuota(
+          blobResponseCost, "blobs_sidecars_by_range/" & versionNumber)
+        peer.network.awaitQuota(
+          blobResponseCost, "blobs_sidecars_by_range/" & versionNumber)
+
+        await response.writeBytesSZ(
+          uncompressedLen, bytes,
+          peer.network.forkDigestAtEpoch(blockIds[i].slot.epoch).data)
+        inc found
+      else:
+        break
+
+  debug "BlobSidecar v" & versionNumber & " range request done",
+    peer, startSlot, count = reqCount, found
+
 p2pProtocol BeaconSync(version = 1,
                        networkState = BeaconSyncNetworkState):
   proc beaconBlocksByRange_v2(
@@ -259,38 +352,7 @@ p2pProtocol BeaconSync(version = 1,
     #      client call that returns `seq[ref BlobSidecar]` will
     #      will be generated by the libp2p macro - we guarantee that seq items
     #      are `not-nil` in the implementation
-    trace "got blobs range request", peer, len = blobIds.len
-    if blobIds.len == 0:
-      raise newException(InvalidInputsError, "No blobs requested")
-
-    let
-      dag = peer.networkState.dag
-      count = blobIds.len
-
-    var
-      found = 0
-      bytes: seq[byte]
-
-    for i in 0..<count:
-      let blockRef = dag.getBlockRef(blobIds[i].block_root).valueOr:
-        continue
-      let index = blobIds[i].index
-      if dag.db.getBlobSidecarSZ(blockRef.bid.root, index, bytes):
-        let uncompressedLen = uncompressedLenFramed(bytes).valueOr:
-          warn "Cannot read blob size, database corrupt?",
-            bytes = bytes.len(), blck = shortLog(blockRef), blobindex = index
-          continue
-
-        peer.awaitQuota(blobResponseCost, "blob_sidecars_by_root/1")
-        peer.network.awaitQuota(blobResponseCost, "blob_sidecars_by_root/1")
-
-        await response.writeBytesSZ(
-          uncompressedLen, bytes,
-          peer.network.forkDigestAtEpoch(blockRef.slot.epoch).data)
-        inc found
-
-    debug "Blob root request done",
-      peer, roots = blobIds.len, count, found
+    getBlobSidecarsByRoot("1", peer, peer.networkState.dag, response, blobIds)
 
   # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blobsidecarsbyrange-v1
   proc blobSidecarsByRange(
@@ -308,61 +370,52 @@ p2pProtocol BeaconSync(version = 1,
     #      client call that returns `seq[ref BlobSidecar]` will
     #      will be generated by the libp2p macro - we guarantee that seq items
     #      are `not-nil` in the implementation
+    getBlobSidecarsByRange(
+      "1", peer, peer.networkState.dag, response, startSlot, reqCount,
+      MAX_BLOBS_PER_BLOCK, MAX_REQUEST_BLOB_SIDECARS)
 
-    trace "got blobs range request", peer, startSlot, count = reqCount
-    if reqCount == 0:
-      raise newException(InvalidInputsError, "Empty range requested")
-
-    let
-      dag = peer.networkState.dag
-      epochBoundary =
-        if dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS >= dag.head.slot.epoch:
-          GENESIS_EPOCH
-        else:
-          dag.head.slot.epoch - dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS
-
-    if startSlot.epoch < epochBoundary:
-      raise newException(ResourceUnavailableError, BlobsOutOfRange)
-
-    var blockIds: array[int(MAX_REQUEST_BLOB_SIDECARS), BlockId]
-    let
-      count = int min(reqCount, blockIds.lenu64)
-      endIndex = count - 1
-      startIndex =
-        dag.getBlockRange(startSlot, 1, blockIds.toOpenArray(0, endIndex))
-
-    var
-      found = 0
-      bytes: seq[byte]
-
-    for i in startIndex..endIndex:
-      for j in 0..<MAX_BLOBS_PER_BLOCK:
-        if dag.db.getBlobSidecarSZ(blockIds[i].root, BlobIndex(j), bytes):
-          # In general, there is not much intermediate time between post-merge
-          # blocks all being optimistic and none of them being optimistic. The
-          # EL catches up, tells the CL the head is verified, and that's it.
-          if  blockIds[i].slot.epoch >= dag.cfg.BELLATRIX_FORK_EPOCH and
-              not dag.head.executionValid:
-            continue
-
-          let uncompressedLen = uncompressedLenFramed(bytes).valueOr:
-            warn "Cannot read blobs sidecar size, database corrupt?",
-              bytes = bytes.len(), blck = shortLog(blockIds[i])
-            continue
-
-          # TODO extract from libp2pProtocol
-          peer.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/1")
-          peer.network.awaitQuota(blobResponseCost, "blobs_sidecars_by_range/1")
-
-          await response.writeBytesSZ(
-            uncompressedLen, bytes,
-            peer.network.forkDigestAtEpoch(blockIds[i].slot.epoch).data)
-          inc found
-        else:
-          break
+  # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blobsidecarsbyroot-v1
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/p2p-interface.md#blobsidecarsbyroot-v2
+  proc blobSidecarsByRoot_v2(
+      peer: Peer,
+      blobIds: BlobIdentifierList,
+      response: MultipleChunksResponse[
+        ref BlobSidecar, Limit(MAX_REQUEST_BLOB_SIDECARS_ELECTRA)])
+      {.async, libp2pProtocol("blob_sidecars_by_root", 2).} =
+    # TODO Semantically, this request should return a non-ref, but doing so
+    #      runs into extreme inefficiency due to the compiler introducing
+    #      hidden copies - in future nim versions with move support, this should
+    #      be revisited
+    # TODO This code is more complicated than it needs to be, since the type
+    #      of the multiple chunks response is not actually used in this server
+    #      implementation (it's used to derive the signature of the client
+    #      function, not in the code below!)
+    # TODO although you can't tell from this function definition, a magic
+    #      client call that returns `seq[ref BlobSidecar]` will
+    #      will be generated by the libp2p macro - we guarantee that seq items
+    #      are `not-nil` in the implementation
+    getBlobSidecarsByRoot("2", peer, peer.networkState.dag, response, blobIds)
 
-    debug "BlobSidecar range request done",
-      peer, startSlot, count = reqCount, found
+  # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/p2p-interface.md#blobsidecarsbyrange-v1
+  # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.10/specs/electra/p2p-interface.md#blobsidecarsbyrange-v2
+  proc blobSidecarsByRange_v2(
+      peer: Peer,
+      startSlot: Slot,
+      reqCount: uint64,
+      response: MultipleChunksResponse[
+        ref BlobSidecar, Limit(MAX_REQUEST_BLOB_SIDECARS_ELECTRA)])
+      {.async, libp2pProtocol("blob_sidecars_by_range", 2).} =
+    # TODO This code is more complicated than it needs to be, since the type
+    #      of the multiple chunks response is not actually used in this server
+    #      implementation (it's used to derive the signature of the client
+    #      function, not in the code below!)
+    # TODO although you can't tell from this function definition, a magic
+    #      client call that returns `seq[ref BlobSidecar]` will
+    #      will be generated by the libp2p macro - we guarantee that seq items
+    #      are `not-nil` in the implementation
+    getBlobSidecarsByRange(
+      "2", peer, peer.networkState.dag, response, startSlot, reqCount,
+      MAX_BLOBS_PER_BLOCK_ELECTRA, MAX_REQUEST_BLOB_SIDECARS_ELECTRA)
 
   # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.8/specs/_features/eip7594/p2p-interface.md#datacolumnsidecarsbyroot-v1
   proc dataColumnSidecarsByRoot(
@@ -388,7 +441,7 @@ p2pProtocol BeaconSync(version = 1,
       bytes: seq[byte]
 
     for i in 0..<count:
-      let blockRef = 
+      let blockRef =
         dag.getBlockRef(colIds[i].block_root).valueOr:
           continue
       let index =
@@ -423,17 +476,17 @@ p2pProtocol BeaconSync(version = 1,
       response: MultipleChunksResponse[
         ref DataColumnSidecar, Limit(MAX_REQUEST_DATA_COLUMN_SIDECARS)])
       {.async, libp2pProtocol("data_column_sidecars_by_range", 1).} =
-    
+
     trace "got data columns range request", peer, startSlot,
       count = reqCount, columns = reqColumns
-    
+
     if reqCount == 0 or reqColumns.len == 0:
       raise newException(InvalidInputsError, "Empty range requested")
 
     let
       dag = peer.networkState.dag
-      # Using MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS until 
-      # MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS is released in 
+      # Using MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS until
+      # MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS is released in
       # Fulu. Effectively both the values are same
       epochBoundary =
         if dag.cfg.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS >= dag.head.slot.epoch:
@@ -486,7 +539,7 @@ p2pProtocol BeaconSync(version = 1,
     debug "Data column range request done",
       peer, startSlot, count = reqCount, columns = reqColumns, found
 
-proc init*(T: type BeaconSync.NetworkState, dag: ChainDAGRef): T =
+func init*(T: type BeaconSync.NetworkState, dag: ChainDAGRef): T =
   T(
     dag: dag,
   )

docker/dist/base_image/README.md

@@ -1,4 +1,4 @@
-You probably don't want to recreate and push these base images to Docker Hub,
+You probably don't want to re-create and push these base images to Docker Hub,
 because when older images expire and get deleted, it will no longer be possible
 to reproduce old releases.