diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index 108c63c423..6a55f8b909 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -488,20 +488,16 @@ func BlockByNumber(txn db.Transaction, number uint64) (*core.Block, error) { } func TransactionsByBlockNumber(txn db.Transaction, number uint64) ([]core.Transaction, error) { - iterator, err := txn.NewIterator() + numBytes := core.MarshalBlockNumber(number) + prefix := db.TransactionsByBlockNumberAndIndex.Key(numBytes) + + iterator, err := txn.NewIterator(prefix, true) if err != nil { return nil, err } var txs []core.Transaction - numBytes := core.MarshalBlockNumber(number) - - prefix := db.TransactionsByBlockNumberAndIndex.Key(numBytes) - for iterator.Seek(prefix); iterator.Valid(); iterator.Next() { - if !bytes.HasPrefix(iterator.Key(), prefix) { - break - } - + for iterator.Next(); iterator.Valid(); iterator.Next() { val, vErr := iterator.Value() if vErr != nil { return nil, utils.RunAndWrapOnError(iterator.Close, vErr) @@ -523,16 +519,17 @@ func TransactionsByBlockNumber(txn db.Transaction, number uint64) ([]core.Transa } func receiptsByBlockNumber(txn db.Transaction, number uint64) ([]*core.TransactionReceipt, error) { - iterator, err := txn.NewIterator() + numBytes := core.MarshalBlockNumber(number) + prefix := db.ReceiptsByBlockNumberAndIndex.Key(numBytes) + + iterator, err := txn.NewIterator(prefix, true) if err != nil { return nil, err } var receipts []*core.TransactionReceipt - numBytes := core.MarshalBlockNumber(number) - prefix := db.ReceiptsByBlockNumberAndIndex.Key(numBytes) - for iterator.Seek(prefix); iterator.Valid(); iterator.Next() { + for iterator.First(); iterator.Valid(); iterator.Next() { if !bytes.HasPrefix(iterator.Key(), prefix) { break } diff --git a/blockchain/blockchain_test.go b/blockchain/blockchain_test.go index 1e1e07ada9..84b437d3a0 100644 --- a/blockchain/blockchain_test.go +++ b/blockchain/blockchain_test.go @@ -654,7 +654,7 @@ func TestRevert(t *testing.T) { t.Run("empty blockchain should mean empty db", func(t *testing.T) { require.NoError(t, testdb.View(func(txn db.Transaction) error { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) if err != nil { return err } diff --git a/core/history.go b/core/history.go index f14db1702e..62656e5a97 100644 --- a/core/history.go +++ b/core/history.go @@ -29,7 +29,7 @@ func (h *history) deleteLog(key []byte, height uint64) error { } func (h *history) valueAt(key []byte, height uint64) ([]byte, error) { - it, err := h.txn.NewIterator() + it, err := h.txn.NewIterator(nil, false) if err != nil { return nil, err } diff --git a/core/state_test.go b/core/state_test.go index 6b96d64b3b..6a4e773e7d 100644 --- a/core/state_test.go +++ b/core/state_test.go @@ -556,7 +556,7 @@ func TestRevert(t *testing.T) { t.Run("empty state should mean empty db", func(t *testing.T) { require.NoError(t, testDB.View(func(txn db.Transaction) error { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) if err != nil { return err } diff --git a/db/buffered_transaction.go b/db/buffered_transaction.go index d708633cfe..b19ba1ab0e 100644 --- a/db/buffered_transaction.go +++ b/db/buffered_transaction.go @@ -79,6 +79,6 @@ func (t *BufferedTransaction) Impl() any { } // NewIterator : see db.Transaction.NewIterator -func (t *BufferedTransaction) NewIterator() (Iterator, error) { +func (t *BufferedTransaction) NewIterator(_ []byte, _ bool) (Iterator, error) { return nil, errors.New("buffered transactions dont support iterators") } diff --git a/db/db.go b/db/db.go index 6475dc5e19..b65c35e620 100644 --- a/db/db.go +++ b/db/db.go @@ -42,6 +42,9 @@ type Iterator interface { // Valid returns true if the iterator is positioned at a valid key/value pair. Valid() bool + // First moves the iterator to the first key/value pair. + First() bool + // Next moves the iterator to the next key/value pair. It returns whether the // iterator is valid after the call. Once invalid, the iterator remains // invalid. @@ -63,7 +66,7 @@ type Iterator interface { // the transaction is committed. type Transaction interface { // NewIterator returns an iterator over the database's key/value pairs. - NewIterator() (Iterator, error) + NewIterator(lowerBound []byte, withUpperBound bool) (Iterator, error) // Discard discards all the changes done to the database with this transaction Discard() error // Commit flushes all the changes pending on this transaction to the database, making the changes visible to other diff --git a/db/memory_transaction.go b/db/memory_transaction.go index ba63cbef54..4e794260d5 100644 --- a/db/memory_transaction.go +++ b/db/memory_transaction.go @@ -14,7 +14,7 @@ func NewMemTransaction() Transaction { return &memTransaction{storage: make(map[string][]byte)} } -func (t *memTransaction) NewIterator() (Iterator, error) { +func (t *memTransaction) NewIterator(_ []byte, _ bool) (Iterator, error) { return nil, errors.New("not implemented") } diff --git a/db/pebble/batch.go b/db/pebble/batch.go index 60747d6a86..61101bd285 100644 --- a/db/pebble/batch.go +++ b/db/pebble/batch.go @@ -88,7 +88,7 @@ func (b *batch) Get(key []byte, cb func([]byte) error) error { } // NewIterator : see db.Transaction.NewIterator -func (b *batch) NewIterator() (db.Iterator, error) { +func (b *batch) NewIterator(lowerBound []byte, withUpperBound bool) (db.Iterator, error) { var iter *pebble.Iterator var err error @@ -96,7 +96,12 @@ func (b *batch) NewIterator() (db.Iterator, error) { return nil, ErrDiscardedTransaction } - iter, err = b.batch.NewIter(nil) + iterOpt := &pebble.IterOptions{LowerBound: lowerBound} + if withUpperBound { + iterOpt.UpperBound = upperBound(lowerBound) + } + + iter, err = b.batch.NewIter(iterOpt) if err != nil { return nil, err } diff --git a/db/pebble/db.go b/db/pebble/db.go index dc9a62d4c1..80c21db0f1 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -164,6 +164,14 @@ func CalculatePrefixSize(ctx context.Context, pDB *DB, prefix []byte, withUpperB return item, utils.RunAndWrapOnError(it.Close, err) } +// Calculates the next possible prefix after the given prefix bytes. +// It's used to establish an upper boundary for prefix-based database scans. +// Examples: +// +// [1] -> [2] +// [1, 255, 255] -> [2] +// [1, 2, 255] -> [1, 3] +// [255, 255] -> nil func upperBound(prefix []byte) []byte { var ub []byte diff --git a/db/pebble/db_test.go b/db/pebble/db_test.go index d1c5d72f7c..aff6d080c7 100644 --- a/db/pebble/db_test.go +++ b/db/pebble/db_test.go @@ -261,7 +261,7 @@ func TestSeek(t *testing.T) { require.NoError(t, txn.Set([]byte{3}, []byte{3})) t.Run("seeks to the next key in lexicographical order", func(t *testing.T) { - iter, err := txn.NewIterator() + iter, err := txn.NewIterator(nil, false) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, iter.Close()) @@ -275,7 +275,7 @@ func TestSeek(t *testing.T) { }) t.Run("key returns nil when seeking nonexistent data", func(t *testing.T) { - iter, err := txn.NewIterator() + iter, err := txn.NewIterator(nil, false) require.NoError(t, err) t.Cleanup(func() { @@ -314,7 +314,7 @@ func TestPrefixSearch(t *testing.T) { })) require.NoError(t, testDB.View(func(txn db.Transaction) error { - iter, err := txn.NewIterator() + iter, err := txn.NewIterator(nil, false) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, iter.Close()) @@ -359,7 +359,7 @@ func TestNext(t *testing.T) { require.NoError(t, txn.Set([]byte{2}, []byte{2})) t.Run("Next() on new iterator", func(t *testing.T) { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) require.NoError(t, err) t.Run("new iterator should be invalid", func(t *testing.T) { @@ -374,7 +374,7 @@ func TestNext(t *testing.T) { }) t.Run("Next() should work as expected after a Seek()", func(t *testing.T) { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) require.NoError(t, err) require.True(t, it.Seek([]byte{0})) diff --git a/db/pebble/iterator.go b/db/pebble/iterator.go index 471b186fdb..0885ff97e7 100644 --- a/db/pebble/iterator.go +++ b/db/pebble/iterator.go @@ -39,6 +39,11 @@ func (i *iterator) Value() ([]byte, error) { return buf, nil } +func (i *iterator) First() bool { + i.positioned = true + return i.iter.First() +} + // Next : see db.Transaction.Iterator.Next func (i *iterator) Next() bool { if !i.positioned { diff --git a/db/pebble/snapshot.go b/db/pebble/snapshot.go index b8ce545e3e..ff67139850 100644 --- a/db/pebble/snapshot.go +++ b/db/pebble/snapshot.go @@ -58,7 +58,7 @@ func (s *snapshot) Get(key []byte, cb func([]byte) error) error { } // NewIterator : see db.Transaction.NewIterator -func (s *snapshot) NewIterator() (db.Iterator, error) { +func (s *snapshot) NewIterator(lowerBound []byte, withUpperBound bool) (db.Iterator, error) { var iter *pebble.Iterator var err error @@ -66,7 +66,12 @@ func (s *snapshot) NewIterator() (db.Iterator, error) { return nil, ErrDiscardedTransaction } - iter, err = s.snapshot.NewIter(nil) + iterOpt := &pebble.IterOptions{LowerBound: lowerBound} + if withUpperBound { + iterOpt.UpperBound = upperBound(lowerBound) + } + + iter, err = s.snapshot.NewIter(iterOpt) if err != nil { return nil, err } diff --git a/db/remote/db_test.go b/db/remote/db_test.go index b3dc49da34..5034db4cbb 100644 --- a/db/remote/db_test.go +++ b/db/remote/db_test.go @@ -63,7 +63,7 @@ func TestRemote(t *testing.T) { t.Run("iterate", func(t *testing.T) { err := remoteDB.View(func(txn db.Transaction) error { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) if err != nil { return err } @@ -85,7 +85,7 @@ func TestRemote(t *testing.T) { t.Run("seek", func(t *testing.T) { err := remoteDB.View(func(txn db.Transaction) error { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) if err != nil { return err } diff --git a/db/remote/iterator.go b/db/remote/iterator.go index 49bd06dc30..1153424633 100644 --- a/db/remote/iterator.go +++ b/db/remote/iterator.go @@ -52,6 +52,13 @@ func (i *iterator) Value() ([]byte, error) { return i.currentV, nil } +func (i *iterator) First() bool { + if err := i.doOpAndUpdate(gen.Op_FIRST, nil); err != nil { + i.log.Debugw("Error", "op", gen.Op_FIRST, "err", err) + } + return len(i.currentK) > 0 || len(i.currentV) > 0 +} + func (i *iterator) Next() bool { if err := i.doOpAndUpdate(gen.Op_NEXT, nil); err != nil { i.log.Debugw("Error", "op", gen.Op_NEXT, "err", err) diff --git a/db/remote/transaction.go b/db/remote/transaction.go index 3884e305f1..7d65b814dc 100644 --- a/db/remote/transaction.go +++ b/db/remote/transaction.go @@ -16,7 +16,7 @@ type transaction struct { log utils.SimpleLogger } -func (t *transaction) NewIterator() (db.Iterator, error) { +func (t *transaction) NewIterator(_ []byte, _ bool) (db.Iterator, error) { err := t.client.Send(&gen.Cursor{ Op: gen.Op_OPEN, }) diff --git a/db/sync_transaction.go b/db/sync_transaction.go index a0dde13b51..f7232cf688 100644 --- a/db/sync_transaction.go +++ b/db/sync_transaction.go @@ -61,6 +61,6 @@ func (t *SyncTransaction) Impl() any { } // NewIterator : see db.Transaction.NewIterator -func (t *SyncTransaction) NewIterator() (Iterator, error) { +func (t *SyncTransaction) NewIterator(_ []byte, _ bool) (Iterator, error) { return nil, errors.New("sync transactions dont support iterators") } diff --git a/grpc/tx.go b/grpc/tx.go index f65e886eb6..4d01c5e537 100644 --- a/grpc/tx.go +++ b/grpc/tx.go @@ -23,7 +23,7 @@ func newTx(dbTx db.Transaction) *tx { } func (t *tx) newCursor() (uint32, error) { - it, err := t.dbTx.NewIterator() + it, err := t.dbTx.NewIterator(nil, false) if err != nil { return 0, err } diff --git a/migration/bucket_migrator.go b/migration/bucket_migrator.go index 9bd2899a88..4a109f92c3 100644 --- a/migration/bucket_migrator.go +++ b/migration/bucket_migrator.go @@ -78,7 +78,7 @@ func (m *BucketMigrator) Before(_ []byte) error { func (m *BucketMigrator) Migrate(ctx context.Context, txn db.Transaction, network *utils.Network, log utils.SimpleLogger) ([]byte, error) { remainingInBatch := m.batchSize - iterator, err := txn.NewIterator() + iterator, err := txn.NewIterator(nil, false) if err != nil { return nil, err } diff --git a/migration/migration.go b/migration/migration.go index 107bd40f10..043bf38366 100644 --- a/migration/migration.go +++ b/migration/migration.go @@ -183,7 +183,7 @@ func updateSchemaMetadata(txn db.Transaction, schema schemaMetadata) error { // migration0000 makes sure the targetDB is empty func migration0000(txn db.Transaction, _ *utils.Network) error { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) if err != nil { return err } @@ -202,7 +202,7 @@ func migration0000(txn db.Transaction, _ *utils.Network) error { // // This enables us to remove the db.ContractRootKey prefix. func relocateContractStorageRootKeys(txn db.Transaction, _ *utils.Network) error { - it, err := txn.NewIterator() + it, err := txn.NewIterator(nil, false) if err != nil { return err } @@ -429,7 +429,7 @@ func (m *changeTrieNodeEncoding) Migrate(_ context.Context, txn db.Transaction, return nil } - iterator, err := txn.NewIterator() + iterator, err := txn.NewIterator(nil, false) if err != nil { return nil, err } diff --git a/migration/migration_pkg_test.go b/migration/migration_pkg_test.go index e2d5613c48..cc85203e33 100644 --- a/migration/migration_pkg_test.go +++ b/migration/migration_pkg_test.go @@ -572,7 +572,7 @@ func TestChangeStateDiffStructEmptyDB(t *testing.T) { require.Nil(t, intermediateState) // DB is still empty. - iter, err := txn.NewIterator() + iter, err := txn.NewIterator(nil, false) defer func() { require.NoError(t, iter.Close()) }() @@ -654,7 +654,7 @@ func TestChangeStateDiffStruct(t *testing.T) { // - Both state diffs have been updated. // - There are no extraneous entries in the DB. require.NoError(t, testdb.View(func(txn db.Transaction) error { - iter, err := txn.NewIterator() + iter, err := txn.NewIterator(nil, false) require.NoError(t, err) defer func() { require.NoError(t, iter.Close()) diff --git a/p2p/p2p.go b/p2p/p2p.go index f0b54c3381..70ed7bc10c 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -353,15 +353,14 @@ func loadPeers(database db.DB) ([]peer.AddrInfo, error) { var peers []peer.AddrInfo err := database.View(func(txn db.Transaction) error { - it, err := txn.NewIterator() + it, err := txn.NewIterator(db.Peer.Key(), true) if err != nil { return fmt.Errorf("create iterator: %w", err) } defer it.Close() - prefix := db.Peer.Key() - for it.Seek(prefix); it.Valid(); it.Next() { - peerIDBytes := it.Key()[len(prefix):] + for it.First(); it.Valid(); it.Next() { + peerIDBytes := it.Key() peerID, err := peer.IDFromBytes(peerIDBytes) if err != nil { return fmt.Errorf("decode peer ID: %w", err)