diff --git a/cmd/db-exporter/src/command_decode_payload.rs b/cmd/db-exporter/src/command_decode_payload.rs index 57cd8d438b..1695c42231 100644 --- a/cmd/db-exporter/src/command_decode_payload.rs +++ b/cmd/db-exporter/src/command_decode_payload.rs @@ -6,14 +6,14 @@ use crate::command_progress::{ ParallelCommandReadBlockFromDB, }; use anyhow::Result; -use chrono::{TimeZone, Utc}; +use chrono::{DateTime, TimeZone, Utc}; use clap::Parser; use csv::{Writer, WriterBuilder}; use move_binary_format::errors::{Location, PartialVMError}; use serde::Serialize; use starcoin_abi_decoder; use starcoin_abi_decoder::DecodedTransactionPayload; -use starcoin_config::BuiltinNetworkID::Barnard; +use starcoin_config::BuiltinNetworkID::{Barnard, Main}; use starcoin_config::ChainNetwork; use starcoin_crypto::{hash::CryptoHash, HashValue}; use starcoin_statedb::ChainStateDB; @@ -22,6 +22,7 @@ use starcoin_types::{block::Block, transaction::TransactionPayload}; use starcoin_vm_types::errors::VMError; use std::fs::File; use std::sync::{Arc, Mutex}; +use std::time::{Duration, UNIX_EPOCH}; use std::{fmt::Debug, path::PathBuf}; const DECODE_PAYLOAD_COMMAND_NAME: &str = "decode_payload_command"; @@ -96,6 +97,7 @@ impl DecodePayloadCommandError { pub struct CSVHeaders { txn_hash: String, signer: String, + txn_type: String, func_name: String, ty_args: String, args: String, @@ -119,13 +121,21 @@ impl ParallelCommandObserver for CommandDecodePayload { } } +fn timestamp_to_datetime(timestamp: u64) -> String { + // Creates a new SystemTime from the specified number of whole seconds + let d = UNIX_EPOCH + Duration::from_secs(timestamp); + // Create DateTime from SystemTime + let datetime = DateTime::::from(d); + // Formats the combined date and time with the specified format string. + datetime.format("%Y-%m-%d %H:%M:%S.%f").to_string() +} + impl ParallelCommand for Block { fn execute(&self, command: &CommandDecodePayload) -> (usize, Vec) { // let errors = vec![]; // let mut success_module_size = 0; - let datetime = Utc.timestamp_opt(self.header.timestamp() as i64, 0); - let formatted_date = datetime.unwrap().format("%Y-%m-%d %H:%M:%s").to_string(); + let formatted_date = timestamp_to_datetime(self.header.timestamp() / 1000); let root = self.header.state_root(); let statedb = ChainStateDB::new(command.storage.clone(), Some(root)); @@ -136,31 +146,51 @@ impl ParallelCommand for Block starcoin_abi_decoder::decode_txn_payload(&statedb, txn.payload()) .expect("Decode transaction payload failed!"); + let mut writer = command.writer_mutex.lock().unwrap(); match decoded_txn_payload { - DecodedTransactionPayload::ScriptFunction(payload) => { - let mut writer = command.writer_mutex.lock().unwrap(); - writer - .serialize(CSVHeaders { - txn_hash: txn.hash().to_string(), - signer, - func_name: format!("{}::{}", payload.module, payload.function), - ty_args: payload - .ty_args - .iter() - .map(|a| a.to_string()) - .collect::>() - .join(","), - args: payload - .args - .iter() - .map(|a| a.0.to_string()) - .collect::>() - .join(","), - timestamp: formatted_date.clone(), - }) - .expect("Write into CSV failed!") - } - DecodedTransactionPayload::Script(_) | DecodedTransactionPayload::Package(_) => (), + DecodedTransactionPayload::ScriptFunction(payload) => writer + .serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + txn_type: String::from("ScriptFunction"), + signer, + func_name: format!("{}::{}", payload.module, payload.function), + ty_args: payload + .ty_args + .iter() + .map(|a| a.to_string()) + .collect::>() + .join("|"), + args: payload + .args + .iter() + .map(|a| a.0.to_string()) + .collect::>() + .join("|"), + timestamp: formatted_date.clone(), + }) + .expect("Write into CSV failed!"), + DecodedTransactionPayload::Script(script) => writer + .serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + txn_type: String::from("Script"), + signer, + func_name: "".to_string(), + ty_args: "".to_string(), + args: "".to_string(), + timestamp: formatted_date.clone(), + }) + .expect("Write into CSV failed!"), + DecodedTransactionPayload::Package(package) => writer + .serialize(CSVHeaders { + txn_hash: txn.hash().to_string(), + txn_type: String::from("Package"), + signer, + func_name: "".to_string(), + ty_args: "".to_string(), + args: "".to_string(), + timestamp: formatted_date.clone(), + }) + .expect("Write into CSV failed!"), } } //(success_module_size, errors) @@ -201,7 +231,7 @@ pub fn decode_payload( let (dbreader, storage) = ParallelCommandReadBlockFromDB::new( input_path, - ChainNetwork::from(Barnard), + ChainNetwork::from(Main), start_height.unwrap_or(0), end_height.unwrap_or(0), )?; @@ -223,7 +253,7 @@ pub fn decode_payload( #[test] pub fn test_decode_payload() -> Result<()> { decode_payload( - PathBuf::from("~/.starcoin/barnard"), + PathBuf::from("/Users/bobong/.starcoin/main"), PathBuf::from("/Users/bobong/Downloads/STC-DB-mainnet/output.csv"), None, None, diff --git a/cmd/db-exporter/src/command_progress.rs b/cmd/db-exporter/src/command_progress.rs index 00035e65fe..84fd59a900 100644 --- a/cmd/db-exporter/src/command_progress.rs +++ b/cmd/db-exporter/src/command_progress.rs @@ -10,9 +10,14 @@ use starcoin_storage::storage::StorageInstance; use starcoin_storage::{Storage, StorageVersion}; use starcoin_types::block::{Block, BlockNumber}; use starcoin_vm_types::language_storage::TypeTag; -use std::sync::Arc; -use std::{fs::File, io::{BufRead, BufReader}, path::PathBuf, time::SystemTime}; use std::io::{Seek, SeekFrom}; +use std::sync::Arc; +use std::{ + fs::File, + io::{BufRead, BufReader}, + path::PathBuf, + time::SystemTime, +}; struct CommandResult { succeed: usize, @@ -78,7 +83,6 @@ pub struct ParallelCommandReadBodyFromExportLine { } impl ParallelCommandReadBodyFromExportLine { - fn count_lines(reader: &mut BufReader) -> Result { let line_count = reader.lines().count(); reader.seek(SeekFrom::Start(0))?; @@ -87,11 +91,10 @@ impl ParallelCommandReadBodyFromExportLine { pub fn new(input_path: PathBuf) -> Result { let file = File::open(input_path.display().to_string())?; - let line_count = ParallelCommandReadBodyFromExportLine::count_lines(&mut BufReader::new(file.try_clone()?))?; - Ok(Self { - file, - line_count, - }) + let line_count = ParallelCommandReadBodyFromExportLine::count_lines(&mut BufReader::new( + file.try_clone()?, + ))?; + Ok(Self { file, line_count }) } } @@ -129,22 +132,27 @@ impl ParallelCommandReadBlockFromDB { let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), input_path.as_ref()) .expect("Failed init_and_check_storage"); - let chain = BlockChain::new(net.time_service(), chain_info.head().id(), storage.clone(), None) - .expect("Failed to initialize block chain"); + let chain = BlockChain::new( + net.time_service(), + chain_info.head().id(), + storage.clone(), + None, + ) + .expect("Failed to initialize block chain"); let cur_num = chain.status().head().number(); - let (start_num, end_num) = if start != 0 && end == 0 { + let (start_num, end_num) = if start == 0 && end == 0 { (0, cur_num) } else { - let end = if cur_num > end + BLOCK_GAP { + let final_end = if cur_num > end + BLOCK_GAP { end } else if cur_num > BLOCK_GAP { cur_num - BLOCK_GAP } else { end }; - (start, end) + (start, final_end) }; if start > cur_num || start > end { @@ -291,7 +299,6 @@ impl ParallelCommandProgress { .collect::>() }) .collect::>>(); - let result = excution_result.into_iter().flatten().fold( CommandResult { succeed: 0, @@ -302,6 +309,26 @@ impl ParallelCommandProgress { failed: acc.failed + result.failed, }, ); + // + // let excution_result = all_items + // .iter() + // .map(|item| { + // let (succeed, failed) = item.execute(command); + // progress_bar.inc(1); + // CommandResult::new(succeed, failed.len()) + // }) + // .collect::>(); + // + // let result = excution_result.into_iter().fold( + // CommandResult { + // succeed: 0, + // failed: 0, + // }, + // |acc, result| CommandResult { + // succeed: acc.succeed + result.succeed, + // failed: acc.failed + result.failed, + // }, + // ); progress_bar.finish();