From deba74fcb11b8e155fc33768c979c89c12c1b3a4 Mon Sep 17 00:00:00 2001 From: Lewin Bormann Date: Sat, 15 Jun 2024 12:26:12 +0200 Subject: [PATCH] Upgrade crc crate --- Cargo.toml | 4 ++-- examples/stresstest/src/main.rs | 2 +- src/crc.rs | 11 ++++++++++ src/db_impl.rs | 3 ++- src/lib.rs | 5 ++--- src/log.rs | 39 ++++++++++++++------------------- src/skipmap.rs | 4 +++- src/table_block.rs | 10 ++++----- src/table_builder.rs | 11 +++++----- 9 files changed, 48 insertions(+), 41 deletions(-) create mode 100644 src/crc.rs diff --git a/Cargo.toml b/Cargo.toml index 651ed3a..4c31396 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,14 +9,14 @@ readme = "README.md" keywords = ["LevelDB", "key-value", "database", "SSTable", "Google"] license = "MIT" publish = true -edition = "2018" +edition = "2021" include = ["src/**/*", "src/*", "Cargo.toml", "LICENSE", "README.md"] [lib] crate-type = ["cdylib", "rlib"] [dependencies] -crc = "1.8" +crc = "3.2" integer-encoding = "3.0" rand = "0.8.5" snap = "1.0" diff --git a/examples/stresstest/src/main.rs b/examples/stresstest/src/main.rs index 84782cd..18e074c 100644 --- a/examples/stresstest/src/main.rs +++ b/examples/stresstest/src/main.rs @@ -16,7 +16,7 @@ fn write(db: &mut DB, n: usize) { let (k, v) = (gen_string(KEY_LEN), gen_string(VAL_LEN)); db.put(k.as_bytes(), v.as_bytes()).unwrap(); - if i % (n/100) == 0 { + if i % (n / 100) == 0 { println!("{}/100 ...", i * 100 / n); db.flush().unwrap(); } diff --git a/src/crc.rs b/src/crc.rs new file mode 100644 index 0000000..6205d40 --- /dev/null +++ b/src/crc.rs @@ -0,0 +1,11 @@ +const CRC: crc::Crc> = crc::Crc::>::new(&crc::CRC_32_ISCSI); + +pub(crate) fn crc32(data: impl AsRef<[u8]>) -> u32 { + let mut digest = CRC.digest(); + digest.update(data.as_ref()); + digest.finalize() +} + +pub(crate) fn digest() -> crc::Digest<'static, u32> { + CRC.digest() +} diff --git a/src/db_impl.rs b/src/db_impl.rs index f85ed41..f7b1311 100644 --- a/src/db_impl.rs +++ b/src/db_impl.rs @@ -567,7 +567,8 @@ impl DB { /// make_room_for_write checks if the memtable has become too large, and triggers a compaction /// if it's the case. fn make_room_for_write(&mut self, force: bool) -> Result<()> { - if !force && self.mem.approx_mem_usage() < self.opt.write_buffer_size || self.mem.len() == 0 { + if !force && self.mem.approx_mem_usage() < self.opt.write_buffer_size || self.mem.len() == 0 + { Ok(()) } else { // Create new memtable. diff --git a/src/lib.rs b/src/lib.rs index 927d6d7..77a8de1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,8 +23,6 @@ #![allow(dead_code)] -extern crate crc; - #[cfg(feature = "fs")] extern crate errno; @@ -50,6 +48,7 @@ mod block_builder; mod blockhandle; mod cache; mod cmp; +mod crc; #[cfg(feature = "fs")] mod disk_env; @@ -80,8 +79,8 @@ mod write_batch; mod db_impl; mod db_iter; -pub mod env; pub mod compressor; +pub mod env; #[cfg(feature = "async")] pub use asyncdb::AsyncDB; diff --git a/src/log.rs b/src/log.rs index bc7321f..55dd72c 100644 --- a/src/log.rs +++ b/src/log.rs @@ -3,12 +3,11 @@ //! A record is a bytestring: [checksum: uint32, length: uint16, type: uint8, data: [u8]] //! checksum is the crc32 sum of type and data; type is one of RecordType::{Full/First/Middle/Last} +use crate::crc; use crate::error::{err, Result, StatusCode}; use std::io::{Read, Write}; -use crc::crc32; -use crc::Hasher32; use integer_encoding::FixedInt; use integer_encoding::FixedIntWriter; @@ -25,19 +24,16 @@ pub enum RecordType { pub struct LogWriter { dst: W, - digest: crc32::Digest, current_block_offset: usize, block_size: usize, } impl LogWriter { pub fn new(writer: W) -> LogWriter { - let digest = crc32::Digest::new(crc32::CASTAGNOLI); LogWriter { dst: writer, current_block_offset: 0, block_size: BLOCK_SIZE, - digest, } } @@ -49,8 +45,7 @@ impl LogWriter { w } - pub fn add_record(&mut self, r: &[u8]) -> Result { - let mut record = &r[..]; + pub fn add_record(&mut self, mut record: &[u8]) -> Result { let mut first_frag = true; let mut result = Ok(0); while result.is_ok() && !record.is_empty() { @@ -94,11 +89,11 @@ impl LogWriter { fn emit_record(&mut self, t: RecordType, data: &[u8], len: usize) -> Result { assert!(len < 256 * 256); - self.digest.reset(); - self.digest.write(&[t as u8]); - self.digest.write(&data[0..len]); + let mut digest = crc::digest(); + digest.update(&[t as u8]); + digest.update(&data[0..len]); - let chksum = mask_crc(self.digest.sum32()); + let chksum = mask_crc(digest.finalize()); let mut s = 0; s += self.dst.write(&chksum.encode_fixed_vec())?; @@ -119,7 +114,6 @@ impl LogWriter { pub struct LogReader { // TODO: Wrap src in a buffer to enhance read performance. src: R, - digest: crc32::Digest, blk_off: usize, blocksize: usize, head_scratch: [u8; 7], @@ -134,7 +128,6 @@ impl LogReader { blocksize: BLOCK_SIZE, checksums: chksum, head_scratch: [0; 7], - digest: crc32::Digest::new(crc32::CASTAGNOLI), } } @@ -195,10 +188,10 @@ impl LogReader { } fn check_integrity(&mut self, typ: u8, data: &[u8], expected: u32) -> bool { - self.digest.reset(); - self.digest.write(&[typ]); - self.digest.write(data); - unmask_crc(expected) == self.digest.sum32() + let mut digest = crc::digest(); + digest.update(&[typ]); + digest.update(data); + unmask_crc(expected) == digest.finalize() } } @@ -220,15 +213,17 @@ mod tests { #[test] fn test_crc_mask_crc() { - let crc = crc32::checksum_castagnoli("abcde".as_bytes()); - assert_eq!(crc, unmask_crc(mask_crc(crc))); - assert!(crc != mask_crc(crc)); + let mut digest = crc::digest(); + digest.update("abcde".as_bytes()); + let sum = digest.finalize(); + assert_eq!(sum, unmask_crc(mask_crc(sum))); + assert!(sum != mask_crc(sum)); } #[test] fn test_crc_sanity() { - assert_eq!(0x8a9136aa, crc32::checksum_castagnoli(&[0 as u8; 32])); - assert_eq!(0x62a8ab43, crc32::checksum_castagnoli(&[0xff as u8; 32])); + assert_eq!(0x8a9136aa, crc::crc32(&[0_u8; 32])); + assert_eq!(0x62a8ab43, crc::crc32(&[0xff_u8; 32])); } #[test] diff --git a/src/skipmap.rs b/src/skipmap.rs index 55d9b64..8d75b76 100644 --- a/src/skipmap.rs +++ b/src/skipmap.rs @@ -273,7 +273,9 @@ impl InnerSkipMap { // assigning its value to new.next... new.next = unsafe { (*current).next.take() }; // ...and then setting the previous element's next field to the new node - unsafe { let _ = replace(&mut (*current).next, Some(new)); }; + unsafe { + let _ = replace(&mut (*current).next, Some(new)); + }; } /// Runs through the skipmap and prints everything including addresses fn dbg_print(&self) { diff --git a/src/table_block.rs b/src/table_block.rs index 39f5357..66d6dcb 100644 --- a/src/table_block.rs +++ b/src/table_block.rs @@ -1,5 +1,6 @@ use crate::block::Block; use crate::blockhandle::BlockHandle; +use crate::crc; use crate::env::RandomAccess; use crate::error::{err, Result, StatusCode}; use crate::filter; @@ -8,7 +9,6 @@ use crate::log::unmask_crc; use crate::options::Options; use crate::table_builder; -use crc::crc32::{self, Hasher32}; use integer_encoding::FixedInt; /// Reads the data for the specified block handle from a file. @@ -79,8 +79,8 @@ pub fn read_table_block( /// Verify checksum of block fn verify_table_block(data: &[u8], compression: u8, want: u32) -> bool { - let mut digest = crc32::Digest::new(crc32::CASTAGNOLI); - digest.write(data); - digest.write(&[compression; 1]); - digest.sum32() == want + let mut digest = crc::digest(); + digest.update(data); + digest.update(&[compression; 1]); + digest.finalize() == want } diff --git a/src/table_builder.rs b/src/table_builder.rs index 6161d62..534d92e 100644 --- a/src/table_builder.rs +++ b/src/table_builder.rs @@ -3,6 +3,7 @@ use crate::block_builder::BlockBuilder; use crate::blockhandle::BlockHandle; use crate::cmp::InternalKeyCmp; use crate::compressor::{self, Compressor, CompressorId}; +use crate::crc; use crate::error::Result; use crate::filter::{InternalFilterPolicy, NoFilterPolicy}; use crate::filter_block::FilterBlockBuilder; @@ -14,8 +15,6 @@ use std::cmp::Ordering; use std::io::Write; use std::rc::Rc; -use crc::crc32; -use crc::Hasher32; use integer_encoding::FixedIntWriter; pub const FOOTER_LENGTH: usize = 40; @@ -209,14 +208,14 @@ impl TableBuilder { let (ctype, compressor) = compressor_id_pair; let data = compressor.encode(block)?; - let mut digest = crc32::Digest::new(crc32::CASTAGNOLI); + let mut digest = crc::digest(); - digest.write(&data); - digest.write(&[ctype; TABLE_BLOCK_COMPRESS_LEN]); + digest.update(&data); + digest.update(&[ctype; TABLE_BLOCK_COMPRESS_LEN]); self.dst.write(&data)?; self.dst.write(&[ctype; TABLE_BLOCK_COMPRESS_LEN])?; - self.dst.write_fixedint(mask_crc(digest.sum32()))?; + self.dst.write_fixedint(mask_crc(digest.finalize()))?; let handle = BlockHandle::new(self.offset, data.len()); self.offset += data.len() + TABLE_BLOCK_COMPRESS_LEN + TABLE_BLOCK_CKSUM_LEN;