From d4057c08edfdf28795a392b9c1e86d6279f327d7 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Thu, 26 May 2022 11:32:01 -0400 Subject: [PATCH 1/7] Re-added starting back from latest processed file # Conflicts: # src/cmd/console.rs # src/polling.rs --- .gitignore | 1 + Cargo.lock | 1 + Cargo.toml | 1 + src/cmd/console.rs | 23 ++++++------ src/polling.rs | 92 ++++++++++++++++++++++++++++++++++++++++++++-- src/result.rs | 2 + 6 files changed, 106 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index ecce718..4d5e9e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ arweave.ptr target +/thegarii \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 3fd164d..04b0871 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,6 +1207,7 @@ dependencies = [ name = "thegarii" version = "0.0.9" dependencies = [ + "anyhow", "async-trait", "base64-url", "bincode", diff --git a/Cargo.toml b/Cargo.toml index fbeeb62..a046b38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ name = "thegarii" path = "bin/thegarii.rs" [dependencies] +anyhow = "1" async-trait = "0.1.52" base64-url = "1.4.13" bincode = "1.3.3" diff --git a/src/cmd/console.rs b/src/cmd/console.rs index 217650b..8491c24 100644 --- a/src/cmd/console.rs +++ b/src/cmd/console.rs @@ -1,7 +1,6 @@ // Copyright 2021 ChainSafe Systems // SPDX-License-Identifier: LGPL-3.0-only use crate::{polling::Polling, Env, Result}; -use std::fs; use structopt::StructOpt; /// console service @@ -13,23 +12,25 @@ pub struct Console { /// if restarting service on failing automatically #[structopt(short = "f", long)] forever: bool, - /// polling start from, if `None`, thegarii will poll from the block height - /// stored in $PTR_FILE or 0 + /// polling start from, if `None`, polling from 0 #[structopt(short = "s", long)] start: Option, + /// data directory where to store latest block fully processed + #[structopt(short = "d", long, default_value = "./thegarii")] + data_directory: String, } impl Console { /// run as service pub async fn exec(&self, env: Env) -> Result<()> { - log::debug!("\n{:#?}", self); - let ptr = fs::read_to_string(&env.ptr_file) - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or_else(|| self.start.unwrap_or(0)); - - log::info!("start polling blocks from {}...", ptr); - let mut polling = Polling::new(self.end, env, self.forever, ptr).await?; + let mut polling = Polling::new( + self.data_directory.to_string(), + self.end, + env, + self.forever, + self.start, + ) + .await?; if let Err(e) = polling.start().await { log::error!("{:?}", e); diff --git a/src/polling.rs b/src/polling.rs index 6c5dbcb..a4228b5 100644 --- a/src/polling.rs +++ b/src/polling.rs @@ -7,8 +7,14 @@ use crate::{ types::{FirehoseBlock, U256}, Error, Result, }; +use anyhow::Context; use prost::Message; -use std::{collections::BTreeMap, fs, path::PathBuf, time::Duration}; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, + time::Duration, +}; #[derive(Debug, Clone)] struct BlockInfo { @@ -18,6 +24,7 @@ struct BlockInfo { /// polling service pub struct Polling { + last_processed_block_path: Box, batch: usize, block_time: u64, client: Client, @@ -32,11 +39,27 @@ pub struct Polling { impl Polling { /// new polling service - pub async fn new(end: Option, env: Env, forever: bool, ptr: u64) -> Result { + pub async fn new( + data_directory: String, + end: Option, + env: Env, + forever: bool, + ptr: Option, + ) -> Result { let client = Client::new(env.endpoints, Duration::from_millis(env.timeout), env.retry)?; let batch = env.batch_blocks as usize; + fs::create_dir_all(&data_directory).context( + format_args!("unable to create data directory {}", &data_directory).to_string(), + )?; + + let last_processed_block_path = + Path::new(&data_directory).join("latest_block_processed.txt"); + + let ptr = Self::determine_start_ptr(&last_processed_block_path, ptr)?; + Ok(Self { + last_processed_block_path: Box::new(last_processed_block_path), batch, block_time: env.block_time, confirms: env.confirms, @@ -50,6 +73,52 @@ impl Polling { }) } + fn determine_start_ptr( + last_processed_block_path: &PathBuf, + start_block_flag: Option, + ) -> Result { + match last_processed_block_path.exists() { + true => { + let content: String = fs::read_to_string(last_processed_block_path).context( + format_args!( + "unable to read content of last block processsed file {:?}", + last_processed_block_path, + ) + .to_string(), + )?; + + let value = content.parse::().context( + format_args!("content {} is not a valid u64 string value", &content) + .to_string(), + )?; + + log::info!( + "start block retrieved from last processed block state file, starting from block {}", + value + ); + + Ok(value) + } + false => { + if let Some(start) = start_block_flag { + log::info!( + "start block explicitely provided, starting from block {}", + start + ); + + Ok(start) + } else { + log::info!( + "no previous latest block processed file {:?} exists, starting from block 0", + &last_processed_block_path + ); + + Ok(0) + } + } + } + } + /// dm log to stdout /// /// DMLOG BLOCK @@ -174,13 +243,30 @@ impl Polling { // // Stores string for easy debugging self.ptr = cur + 1; - fs::write(&self.ptr_file, self.ptr.to_string())?; + + self.write_ptr().await?; } } Ok(()) } + async fn write_ptr(&self) -> Result<()> { + let ptr_string = self.ptr.to_string(); + + tokio::fs::write(self.last_processed_block_path.as_ref(), &ptr_string) + .await + .context( + format_args!( + "unable to write last processed block ptr to {:?}", + &self.last_processed_block_path, + ) + .to_string(), + )?; + + Ok(()) + } + /// poll to head async fn track_head(&mut self) -> Result<()> { self.latest = self.client.get_current_block().await?.height; diff --git a/src/result.rs b/src/result.rs index 5fcaeee..2dc4e5f 100644 --- a/src/result.rs +++ b/src/result.rs @@ -50,6 +50,8 @@ pub enum Error { Uint(#[from] uint::FromDecStrErr), #[error(transparent)] Var(#[from] VarError), + #[error(transparent)] + Generic(#[from] anyhow::Error), } /// result type From 582add8f351ffb6bb30a5b9a799fdcb0e3a95291 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Fri, 6 May 2022 13:39:28 -0400 Subject: [PATCH 2/7] Added ctrl-c signal handling and better error printing on exit --- bin/thegarii.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/bin/thegarii.rs b/bin/thegarii.rs index 69634df..75653b5 100644 --- a/bin/thegarii.rs +++ b/bin/thegarii.rs @@ -1,9 +1,31 @@ // Copyright 2021 ChainSafe Systems // SPDX-License-Identifier: LGPL-3.0-only +use std::process::exit; + use thegarii::Opt; +use tokio::signal; #[tokio::main] -async fn main() { - Opt::exec().await.expect("thegraii crashed") +async fn main() -> Result<(), anyhow::Error> { + tokio::spawn(async { + if let Err(err) = signal::ctrl_c().await { + log::error!("error waiting for SIGINT signal: {:?}", err); + exit(1); + } + + log::info!("received SIGINT signal, terminating"); + exit(0); + }); + + match Opt::exec().await { + Ok(_) => { + log::info!("completed"); + Ok(()) + } + Err(err) => { + log::error!("unexpected error occurred: {:?}", err); + exit(1); + } + } } From e51158741104680b145af21243a9f7fe213329ad Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Sat, 7 May 2022 00:15:15 -0400 Subject: [PATCH 3/7] Heavy refactoring of underlying polling logic - Now simply polling on "irrversible block" only, so we ditch all the fork handling code - Batching is smoother now as task are spawned as soon as first one finish new one starts (old code was waiting for the full batch to complete before starting new task(s)) - Improved logging to more easily understand where we are --- Cargo.lock | 1 + Cargo.toml | 1 + src/client.rs | 2 + src/cmd/console.rs | 13 +- src/env.rs | 2 +- src/polling.rs | 368 ++++++++++++++++++++------------------------- src/result.rs | 2 + 7 files changed, 184 insertions(+), 205 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04b0871..bcf5058 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1214,6 +1214,7 @@ dependencies = [ "dirs", "env_logger", "futures", + "hex", "log", "prost", "prost-types", diff --git a/Cargo.toml b/Cargo.toml index a046b38..eea2fd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ base64-url = "1.4.13" bincode = "1.3.3" dirs = "4.0.0" env_logger = "0.9.0" +hex = "0.4.3" futures = "0.3.21" log = "0.4.14" prost = "0.9" diff --git a/src/client.rs b/src/client.rs index 02c650d..ff942ce 100644 --- a/src/client.rs +++ b/src/client.rs @@ -210,6 +210,8 @@ impl Client { /// } /// ``` pub async fn get_firehose_block_by_height(&self, height: u64) -> Result { + log::info!("resolving firehose block {}", height); + let block = self.get_block_by_height(height).await?; let txs: Vec = join_all(block.txs.iter().map(|tx| self.get_tx_by_id(tx))) .await diff --git a/src/cmd/console.rs b/src/cmd/console.rs index 8491c24..5be4949 100644 --- a/src/cmd/console.rs +++ b/src/cmd/console.rs @@ -12,23 +12,30 @@ pub struct Console { /// if restarting service on failing automatically #[structopt(short = "f", long)] forever: bool, - /// polling start from, if `None`, polling from 0 + /// If never processed block before, start from -s if defined (use 'live' to start from "head" block) #[structopt(short = "s", long)] - start: Option, + start: Option, /// data directory where to store latest block fully processed #[structopt(short = "d", long, default_value = "./thegarii")] data_directory: String, + /// reduce deep mind block output by just showing the length (not good for production!) + #[structopt(short = "q", long)] + quiet: bool, } impl Console { /// run as service pub async fn exec(&self, env: Env) -> Result<()> { + log::debug!("\n{:?}", self); + log::info!("start polling blocks..."); + let mut polling = Polling::new( self.data_directory.to_string(), self.end, env, self.forever, - self.start, + self.start.clone(), + self.quiet, ) .await?; diff --git a/src/env.rs b/src/env.rs index 9de5f18..4f8fd50 100644 --- a/src/env.rs +++ b/src/env.rs @@ -28,7 +28,7 @@ pub struct EnvArguments { #[structopt(short = "B", long, default_value = "20")] pub batch_blocks: u16, /// time cost for producing a new block in arweave - #[structopt(short, long, default_value = "20000")] + #[structopt(short, long, default_value = "60000")] pub block_time: u64, /// safe blocks against to reorg in polling #[structopt(short, long, default_value = "20")] diff --git a/src/polling.rs b/src/polling.rs index a4228b5..5351e9f 100644 --- a/src/polling.rs +++ b/src/polling.rs @@ -1,26 +1,13 @@ // Copyright 2021 ChainSafe Systems // SPDX-License-Identifier: LGPL-3.0-only -use crate::{ - client::Client, - env::Env, - pb::Block, - types::{FirehoseBlock, U256}, - Error, Result, -}; +use crate::{client::Client, env::Env, pb::Block, Error, Result}; use anyhow::Context; +use futures::stream; +use futures::StreamExt; +use futures::TryFutureExt; use prost::Message; -use std::{ - collections::BTreeMap, - fs, - path::{Path, PathBuf}, - time::Duration, -}; - -#[derive(Debug, Clone)] -struct BlockInfo { - pub indep_hash: String, - pub cumulative_diff: U256, -} +use std::path::{Path, PathBuf}; +use std::{fs, time::Duration}; /// polling service pub struct Polling { @@ -32,9 +19,8 @@ pub struct Polling { end: Option, forever: bool, latest: u64, - live_blocks: BTreeMap, ptr: u64, - ptr_file: PathBuf, + quiet: bool, } impl Polling { @@ -44,9 +30,14 @@ impl Polling { end: Option, env: Env, forever: bool, - ptr: Option, + ptr: Option, + quiet: bool, ) -> Result { - let client = Client::new(env.endpoints, Duration::from_millis(env.timeout), env.retry)?; + let client = Client::new( + env.endpoints.clone(), + Duration::from_millis(env.timeout), + env.retry, + )?; let batch = env.batch_blocks as usize; fs::create_dir_all(&data_directory).context( @@ -56,9 +47,7 @@ impl Polling { let last_processed_block_path = Path::new(&data_directory).join("latest_block_processed.txt"); - let ptr = Self::determine_start_ptr(&last_processed_block_path, ptr)?; - - Ok(Self { + let mut poller = Self { last_processed_block_path: Box::new(last_processed_block_path), batch, block_time: env.block_time, @@ -67,184 +56,143 @@ impl Polling { end, forever, latest: 0, - live_blocks: Default::default(), - ptr, - ptr_file: env.ptr_file, - }) - } - - fn determine_start_ptr( - last_processed_block_path: &PathBuf, - start_block_flag: Option, - ) -> Result { - match last_processed_block_path.exists() { - true => { - let content: String = fs::read_to_string(last_processed_block_path).context( - format_args!( - "unable to read content of last block processsed file {:?}", - last_processed_block_path, - ) - .to_string(), - )?; - - let value = content.parse::().context( - format_args!("content {} is not a valid u64 string value", &content) - .to_string(), - )?; + ptr: 0, + quiet: quiet, + }; - log::info!( - "start block retrieved from last processed block state file, starting from block {}", - value - ); + poller.initialize_start_ptr(ptr).await?; - Ok(value) - } - false => { - if let Some(start) = start_block_flag { - log::info!( - "start block explicitely provided, starting from block {}", - start - ); + Ok(poller) + } - Ok(start) - } else { + async fn initialize_start_ptr(&mut self, start_block_flag: Option) -> Result<()> { + self.ptr = match self.last_processed_block_path.exists() { + true => self.start_ptr_from_state().await?, + false => match start_block_flag { + Some(value) if value == "live" => self.start_ptr_from_last_irreversible().await?, + Some(start) => self.start_ptr_from_flag_value(&start).await?, + _ => { log::info!( - "no previous latest block processed file {:?} exists, starting from block 0", - &last_processed_block_path - ); + "no previous latest block processed file {:?} exists, starting from block 0", + self.last_processed_block_path + ); - Ok(0) + 0 } - } - } - } - - /// dm log to stdout - /// - /// DMLOG BLOCK - fn dm_log(b: FirehoseBlock) -> Result<()> { - let height = b.height; - let proto: Block = b.try_into()?; - - println!( - "DMLOG BLOCK {} {}", - height, - proto - .encode_to_vec() - .into_iter() - .map(|b| format!("{:02x}", b)) - .reduce(|mut r, c| { - r.push_str(&c); - r - }) - .ok_or(Error::ParseBlockFailed)? - ); + }, + }; Ok(()) } - /// compare blocks with current live blocks - /// - /// # TODO - /// - /// - return the height of fork block if exists - /// - replace live_blocks field with a sorted stack - fn cmp_live_blocks(&mut self, blocks: &mut [FirehoseBlock]) -> Result<()> { - if blocks.is_empty() { - return Ok(()); - } + async fn start_ptr_from_state(&self) -> Result { + let content: String = tokio::fs::read_to_string(self.last_processed_block_path.as_ref()) + .await + .context( + format_args!( + "unable to read content of last block processsed file {:?}", + self.last_processed_block_path, + ) + .to_string(), + )?; - // # Safty - // - // this will never happen since we have an empty check above - let last = blocks.last().ok_or(Error::ParseBlockPtrFailed)?.clone(); - if last.height + self.confirms < self.latest { - return Ok(()); - } + content.parse::() + .context(format_args!("content {} is not a valid u64 string value", &content).to_string(), + ).map(|value| { + log::info!( + "start block retrieved from last processed block state file, starting from block {}", + value + ); - // - detect if have fork - // - add new live blocks - let mut dup_blocks = vec![]; - for b in blocks.iter() { - let cumulative_diff = - U256::from_dec_str(&b.cumulative_diff.clone().unwrap_or_else(|| "0".into()))?; + value + }).map_err(Into::into) + } - let block_info = BlockInfo { - indep_hash: b.indep_hash.clone(), - cumulative_diff, - }; + async fn start_ptr_from_last_irreversible(&self) -> Result { + log::info!("user requested 'live' block, retrieving it from endpoint"); - // detect fork - if let Some(value) = self.live_blocks.get(&b.height) { - // - comparing if have different `indep_hash` - // - comparing if the block belongs to a longer chain - if *value.indep_hash != b.indep_hash && cumulative_diff > value.cumulative_diff { - // TODO - // - // return fork number - } else { - dup_blocks.push(b.height); - continue; - } - } + self.latest_irreversible_block_num() + .await + .and_then(|live_block| { + log::info!( + "start block explicitely provided, starting from live block {}", + live_block + ); - // update live blocks - if b.height + self.confirms > self.latest { - self.live_blocks.insert(b.height, block_info); - } - } + Ok(live_block) + }) + } + + async fn start_ptr_from_flag_value(&self, value: &String) -> Result { + value + .parse::() + .and_then(|value| { + log::info!( + "start block explicitely provided, starting from block {}", + value + ); - // remove emitted live blocks - // blocks.retain(|b| !dup_blocks.contains(&b.height)); + Ok(value) + }) + .context(format_args!("start {} is not a valid u64 string value", value).to_string()) + .map_err(Into::into) + } - // trim irreversible blocks - self.live_blocks = self - .live_blocks - .clone() - .into_iter() - .filter(|(h, _)| *h + self.confirms > self.latest) - .collect(); + /// dm log to stdout + /// + /// DMLOG BLOCK + fn dm_log(&self, b: &(u64, Vec)) -> Result<()> { + let height = b.0; + + if self.quiet { + println!("DMLOG BLOCK {} ", height); + } else { + println!("DMLOG BLOCK {} {}", height, hex::encode(&b.1)); + } - log::trace!( - "live blocks: {:?}", - self.live_blocks.keys().into_iter().collect::>() - ); Ok(()) } /// poll blocks and write to stdout - async fn poll(&mut self, blocks: impl IntoIterator) -> Result<()> { - let mut blocks = blocks.into_iter().collect::>(); - + async fn poll(&mut self, blocks: Vec) -> Result<()> { if blocks.is_empty() { + log::info!("nothing to poll, blocks are empty"); return Ok(()); } - while !blocks.is_empty() { - let mut polling = blocks.clone(); - if polling.len() > self.batch { - blocks = polling.split_off(self.batch); - } else { - blocks.drain(..); - } + log::info!( + "polling from {} to {}", + blocks.first().expect("non-empty"), + blocks.last().expect("non-empty") + ); + + let mut tasks = stream::iter(blocks.into_iter().map(|block| { + self.client + .get_firehose_block_by_height(block) + .and_then(|block| async { + let height = block.height; + let proto: Block = block.try_into()?; + + Ok((height, proto.encode_to_vec())) + }) + })) + .buffered(self.batch); + + while let Some(item) = tasks.next().await { + let block = item?; + + self.dm_log(&block)?; + // # Safty + // + // only update ptr after dm_log + self.ptr = block.0 + 1; + + self.write_ptr().await?; - // poll blocks and dm logging - let mut blocks = self.client.poll(polling.into_iter()).await?; - self.cmp_live_blocks(&mut blocks)?; - for b in blocks { - let cur = b.height; - Self::dm_log(b)?; - - // # Safty - // - // only update ptr after dm_log - // - // # NOTE - // - // Stores string for easy debugging - self.ptr = cur + 1; - - self.write_ptr().await?; + if let Some(end) = self.end { + if block.0 == end { + return Err(Error::StopBlockReached); + } } } @@ -267,41 +215,59 @@ impl Polling { Ok(()) } + async fn latest_irreversible_block_num(&self) -> Result { + let head_block = self.client.get_current_block().await?.height; + if head_block < self.confirms { + return Ok(head_block); + } + + Ok(head_block - self.confirms) + } + /// poll to head async fn track_head(&mut self) -> Result<()> { - self.latest = self.client.get_current_block().await?.height; - self.poll(self.ptr..=self.latest).await?; + log::info!("fetching last irreversible block"); + self.latest = self.latest_irreversible_block_num().await?; + + log::info!("tracking head from {} to {}", self.ptr, self.latest); + self.poll((self.ptr..=self.latest).collect()).await?; Ok(()) } /// start polling service pub async fn start(&mut self) -> Result<()> { - if let Some(end) = self.end { - self.poll(self.ptr..=end).await?; - - return Ok(()); - } - loop { // restart when network error occurs - if let Err(e) = self.track_head().await { - log::error!("{:?}", e); - - if self.forever { - log::info!("restarting..."); - continue; - } else { - return Err(e); + let result = self.track_head().await; + match result { + Err(Error::StopBlockReached) => { + log::info!( + "stop block {} reached, stopping poller", + self.end.expect("stop block reached, must be set") + ); + return Ok(()); } - } - // sleep and waiting for new blocks - log::info!( - "waiting for new blocks for {}ms... current height: {}", - self.block_time, - self.latest, - ); - tokio::time::sleep(Duration::from_millis(self.block_time)).await; + Err(e) => { + log::error!("{:?}", e); + + if self.forever { + log::info!("restarting..."); + continue; + } else { + return Err(e); + } + } + + _ => { + log::info!( + "sleeping {}ms before checking for new blocks (last irrerversible block {})", + self.block_time, + self.latest, + ); + tokio::time::sleep(Duration::from_millis(self.block_time)).await; + } + }; } } } diff --git a/src/result.rs b/src/result.rs index 2dc4e5f..a0ccba7 100644 --- a/src/result.rs +++ b/src/result.rs @@ -28,6 +28,8 @@ pub enum Error { ParseBlockFailed, #[error("parse block ptr failed")] ParseBlockPtrFailed, + #[error("stop block reached")] + StopBlockReached, #[error(transparent)] AddrParseError(#[from] AddrParseError), #[error(transparent)] From 7115c3f14990ba22644d3b7041d7719d53e1edc7 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 28 Sep 2022 10:36:57 -0400 Subject: [PATCH 4/7] Updated `deep mind` and `DMLOG` to `Firehose` and `FIRE` --- src/cmd/console.rs | 2 +- src/polling.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cmd/console.rs b/src/cmd/console.rs index 5be4949..38a168f 100644 --- a/src/cmd/console.rs +++ b/src/cmd/console.rs @@ -18,7 +18,7 @@ pub struct Console { /// data directory where to store latest block fully processed #[structopt(short = "d", long, default_value = "./thegarii")] data_directory: String, - /// reduce deep mind block output by just showing the length (not good for production!) + /// reduce Firehose logs block output by just showing the length (not good for production!) #[structopt(short = "q", long)] quiet: bool, } diff --git a/src/polling.rs b/src/polling.rs index 5351e9f..7e72441 100644 --- a/src/polling.rs +++ b/src/polling.rs @@ -138,16 +138,16 @@ impl Polling { .map_err(Into::into) } - /// dm log to stdout + /// Firehose log to stdout /// - /// DMLOG BLOCK - fn dm_log(&self, b: &(u64, Vec)) -> Result<()> { + /// FIRE BLOCK + fn firehose_log(&self, b: &(u64, Vec)) -> Result<()> { let height = b.0; if self.quiet { - println!("DMLOG BLOCK {} ", height); + println!("FIRE BLOCK {} ", height); } else { - println!("DMLOG BLOCK {} {}", height, hex::encode(&b.1)); + println!("FIRE BLOCK {} {}", height, hex::encode(&b.1)); } Ok(()) @@ -181,10 +181,10 @@ impl Polling { while let Some(item) = tasks.next().await { let block = item?; - self.dm_log(&block)?; + self.firehose_log(&block)?; // # Safty // - // only update ptr after dm_log + // only update ptr after firehose_log has been emitted self.ptr = block.0 + 1; self.write_ptr().await?; From c4cf71a4af77344d5e258a9a868fad424c859ba4 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 28 Sep 2022 11:02:51 -0400 Subject: [PATCH 5/7] Added GitHub Actions to build `thegarii` --- .github/ISSUE_TEMPLATE.md | 7 ++ .github/generate_change_log.sh | 36 ++++++ .github/pull_request_template.md | 35 ------ .github/workflows/audit.yml | 13 +++ .github/workflows/check-license.yml | 28 ----- .github/workflows/push.yml | 79 +++++++++++++ .github/workflows/release.yml | 173 ++++++++++++++++++++++++++++ CHANGELOG.md | 7 ++ Dockerfile | 34 +++--- rust-toolchain.toml | 3 + 10 files changed, 331 insertions(+), 84 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE.md create mode 100755 .github/generate_change_log.sh create mode 100644 .github/workflows/audit.yml delete mode 100644 .github/workflows/check-license.yml create mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/release.yml create mode 100644 CHANGELOG.md create mode 100644 rust-toolchain.toml diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..cb67232 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,7 @@ +**Do you want to request a *feature* or report a *bug*?** + +**What is the current behavior?** + +**If the current behavior is a bug, please provide the steps to reproduce and if possible a minimal demo of the problem.** + +**What is the expected behavior?** diff --git a/.github/generate_change_log.sh b/.github/generate_change_log.sh new file mode 100755 index 0000000..c17477a --- /dev/null +++ b/.github/generate_change_log.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +checksum() { + echo $(sha256sum $@ | awk '{print $1}') +} + +change_log_file="./CHANGELOG.md" +version="## $@" +version_prefix="## [0-9]{1,2}\." +start=0 +CHANGE_LOG="" +while read line; do + if [[ $line == *"$version"* ]]; then + start=1 + continue + fi + if [[ $line =~ $version_prefix ]] && [ $start == 1 ]; then + break; + fi + if [ $start == 1 ]; then + CHANGE_LOG+="$line\n" + fi +done < ${change_log_file} + +LINUX_X86_64_BIN_SUM="$(checksum ./linux-x86_64-unknown-linux-gnu)" + +OUTPUT=$(cat <<-END +## Changelog\n +${CHANGE_LOG}\n +## Checksums\n +| Assets | Sha256 Checksum |\n +| :-----------: |------------|\n +| thegarii-linux-x86-64 | ${LINUX_X86_64_BIN_SUM} |\n +END +) + +echo -e ${OUTPUT} diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7eb6a04..e69de29 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,35 +0,0 @@ -## Changes - - - -- -- -- - -## Tests - - - -``` - -``` - -## Issues - - - -- diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..e6f3174 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,13 @@ +# See https://github.com/actions-rs/audit-check +name: Security audit +on: + schedule: + - cron: '0 0 */7 * *' +jobs: + security_audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/audit-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/check-license.yml b/.github/workflows/check-license.yml deleted file mode 100644 index 60bb899..0000000 --- a/.github/workflows/check-license.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Check Licence - -on: - pull_request: - branches: - - main - push: - branches: - - main - -jobs: - check-licence: - runs-on: ubuntu-18.04 - - steps: - - uses: actions/checkout@v2 - - name: Set up Golang - uses: actions/setup-go@v2 - with: - go-version: '^1.16' - - name: Install addlicense - run: | - export PATH=${PATH}:`go env GOPATH`/bin - go install github.com/google/addlicense@latest - - name: Check license - run: | - export PATH=${PATH}:`go env GOPATH`/bin - addlicense -check -c "ChainSafe Systems" -f ./res/header.txt -y 2021 $(find $PWD -type f -name '*.rs') diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..946da36 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,79 @@ +name: Continuous Integration + +on: + push: + branches: [master, develop] + pull_request: + types: [opened, synchronize, reopened] + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: full + +jobs: + rustfmt: + name: Check rustfmt style + strategy: + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Rust Toolchain(s) + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + components: rustfmt + override: true + + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: rustfmt-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Check formating + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-D warnings" + with: + command: fmt + args: --all -- --check + + release-check: + name: Build in release mode + strategy: + matrix: + rust: ["stable"] + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Install Rust Toolchain(s) + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + + - name: Cache cargo registry + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: release-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Cargo check (release) + uses: actions-rs/cargo@v1 + env: + RUSTFLAGS: "-D warnings" + with: + command: check + args: --release diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9b28861 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,173 @@ +name: Release + +on: + push: + tags: + - "*" + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + name: Build Release + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + strategy: + matrix: + rust: [stable] + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + components: rustfmt + override: true + + - uses: actions/cache@v2 + name: Cache cargo registry + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: release-cargo-${{ hashFiles('**/Cargo.toml') }} + + - name: Cache LLVM and Clang + uses: actions/cache@v2 + id: cache-llvm + with: + path: | + ./llvm + key: llvm-10 + + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + version: "10" + cached: ${{ steps.cache-llvm.outputs.cache-hit }} + + - name: Build target + uses: actions-rs/cargo@v1 + env: + CARGO_PROFILE_RELEASE_CODEGEN_UNITS: '1' + CARGO_PROFILE_RELEASE_LTO: 'fat' + with: + # We cannot use `cross` tool right now. The reason is that we require some + # local libraries, `libclang.so` specifically. The `cross` tool runs a Docker + # container which does not have the library in question. We will need to wait to + # have support of https://github.com/cross-rs/cross/pull/635 to be able to cross + # compile properly. + # use-cross: true + command: build + args: --release + + - name: Upload Build + uses: actions/upload-artifact@v2 + with: + name: linux-x86_64-unknown-linux-gnu + path: ./target/release/thegarii + + release: + name: Release + needs: [build] + runs-on: ubuntu-latest + + permissions: + contents: write + packages: write + + steps: + - name: Set Env + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Checkout Code + uses: actions/checkout@v2 + + - name: Generate Change Log + id: changelog + if: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + chmod 755 ./.github/generate_change_log.sh + CHANGELOG=$(./.github/generate_change_log.sh ${{ env.RELEASE_VERSION }}) + + echo "CHANGELOG<> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + - name: Download All Artifacts + id: download-artifacts + uses: actions/download-artifact@v2 + with: + path: ./binaries + + - name: Sanitize Downloaded Files + run: | + # We downloaded all the artifacts previously uploaded and we put them in + # the 'binaries' folder. In this folder, the layout is: + # + # binaries + # ├── linux-arm64-unknown-linux-gnu + # │ └── + # └── linux-x86_64-unknown-linux-gnu + # └── + # + # The sub-folder name comes from the 'name' field of the 'actions/upload-artifact@v2' + # step. The '' file name is the filename of the uploaded 'path' field, + # we used './target/release/' in the upload step so the file name here + # is ''. + + download_path="${{steps.download-artifacts.outputs.download-path}}" + chmod +x "${download_path}/linux-x86_64-unknown-linux-gnu/thegarii" + mv "$download_path/linux-x86_64-unknown-linux-gnu/thegarii" "$download_path/thegarii-x86_64-unknown-linux-gnu" + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate docker tags/labels from github build context + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=tag + type=sha,prefix=,enable=true + flavor: | + latest=${{ startsWith(github.ref, 'refs/tags/') }} + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: ${{steps.download-artifacts.outputs.download-path}} + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Create Release + uses: softprops/action-gh-release@v1 + if: ${{ startsWith(github.ref, 'refs/tags/') }} + with: + name: ${{ env.RELEASE_VERSION }} + tag_name: ${{ env.RELEASE_VERSION }} + draft: false + prerelease: false + body: ${{ env.CHANGELOG }} + token: ${{ secrets.GITHUB_TOKEN }} + fail_on_unmatched_files: true + generate_release_notes: true + files: | + ${{steps.download-artifacts.outputs.download-path}}/* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1b329a5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +## 0.2.0 + +- Re-branded under Firehose (remove all usage of `deep mind`, `DMLOG` and other similar names). + + **Important** https://github.com/streamingfast/firehose-arweave/commit/5f34975b59911eefdf0f5c3862b2f01f4d34738a + or older is required as the Firehose logs exchanged format has changed and requires a recent enough version + of `firehose-arweave`. diff --git a/Dockerfile b/Dockerfile index 589f37c..67c30b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,14 @@ -FROM debian:bullseye -RUN apt-get update && apt-get install -y git curl build-essential cmake pkg-config libssl-dev libsqlite3-dev libgmp-dev ncurses-bin libncurses-dev net-tools ufw -WORKDIR /usr/src -RUN git clone --depth=1 -b maint-24 https://github.com/erlang/otp erlang-otp -WORKDIR /usr/src/erlang-otp -RUN ./configure && make -RUN make install -WORKDIR /usr/src/ -RUN git clone -b N.2.5.1.0 --recursive https://github.com/ArweaveTeam/arweave.git -WORKDIR /usr/src/arweave -RUN ./rebar3 as prod tar -WORKDIR /opt/arweave -RUN cp /usr/src/arweave/_build/prod/rel/arweave/arweave-2.5.1.0.tar.gz /opt/arweave -RUN tar -xzvf arweave-2.5.1.0.tar.gz -WORKDIR /opt/ +FROM ubuntu:20.04 -COPY scripts/run_docker.sh /opt/arweave/run.sh -RUN echo -n "fs.file-max=100000000" >> /etc/sysctl.conf -RUN echo -n "DefaultLimitNOFILE=10000000" >> /etc/systemd/user.conf -RUN echo -n "DefaultLimitNOFILE=10000000" >> /etc/systemd/system.conf -EXPOSE 1984 -CMD ["/opt/arweave/run.sh"] +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get -y install -y \ + ca-certificates libssl1.1 vim htop iotop sysstat wget \ + dstat strace lsof curl jq tzdata && \ + rm -rf /var/cache/apt /var/lib/apt/lists/* + +RUN rm /etc/localtime && ln -snf /usr/share/zoneinfo/America/Montreal /etc/localtime && dpkg-reconfigure -f noninteractive tzdata + +COPY thegarii-x86_64-unknown-linux-gnu /app/thegarii +RUN chmod +x /app/thegarii + +ENV PATH "$PATH:/app" diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..45d559f --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.64.0" +components = [ "rustfmt" ] \ No newline at end of file From 538600bcd05399a2dacdba35dd5d59d4aa2390d2 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 28 Sep 2022 11:37:24 -0400 Subject: [PATCH 6/7] Fixed changelog generation script A few bash-fu problems where fiddling with markdown formatting, fixed that. --- .github/generate_change_log.sh | 55 ++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/.github/generate_change_log.sh b/.github/generate_change_log.sh index c17477a..1b66542 100755 --- a/.github/generate_change_log.sh +++ b/.github/generate_change_log.sh @@ -1,36 +1,41 @@ #!/usr/bin/env bash -checksum() { - echo $(sha256sum $@ | awk '{print $1}') -} -change_log_file="./CHANGELOG.md" -version="## $@" -version_prefix="## [0-9]{1,2}\." -start=0 -CHANGE_LOG="" -while read line; do + +main() { + change_log_file="./CHANGELOG.md" + version="## $@" + version_prefix="## [0-9]{1,2}\." + start=0 + CHANGE_LOG="" + while IFS= read line; do if [[ $line == *"$version"* ]]; then - start=1 - continue + start=1 + continue fi if [[ $line =~ $version_prefix ]] && [ $start == 1 ]; then - break; + break; fi if [ $start == 1 ]; then - CHANGE_LOG+="$line\n" + CHANGE_LOG+="$line\n" fi -done < ${change_log_file} + done < ${change_log_file} + LINUX_X86_64_BIN_SUM="$(checksum ./linux-x86_64-unknown-linux-gnu)" -LINUX_X86_64_BIN_SUM="$(checksum ./linux-x86_64-unknown-linux-gnu)" - -OUTPUT=$(cat <<-END -## Changelog\n -${CHANGE_LOG}\n -## Checksums\n -| Assets | Sha256 Checksum |\n -| :-----------: |------------|\n -| thegarii-linux-x86-64 | ${LINUX_X86_64_BIN_SUM} |\n + OUTPUT="$(cat <<-END +## Changelog +${CHANGE_LOG} +## Checksums +|Assets | Checksum (sha256)| +|-|-| +|thegarii-linux-x86-64 | ${LINUX_X86_64_BIN_SUM}| END -) +)" + + echo -e "${OUTPUT}" +} + +checksum() { + echo $(sha256sum $@ | awk '{print $1}') +} -echo -e ${OUTPUT} +main $@ \ No newline at end of file From 1f2f62a3e7c3a7fe48157630fe73c3ebdb8163f2 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Wed, 28 Sep 2022 13:33:46 -0400 Subject: [PATCH 7/7] Fixed up automatic changelog generation. --- .github/generate_change_log.sh | 11 +++++++---- .github/workflows/release.yml | 22 +++++++++++----------- CHANGELOG.md | 4 ++++ 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/generate_change_log.sh b/.github/generate_change_log.sh index 1b66542..b0f3bf1 100755 --- a/.github/generate_change_log.sh +++ b/.github/generate_change_log.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash - main() { + version="$1" + binaries_dir="$2" + change_log_file="./CHANGELOG.md" - version="## $@" + version=`printf "## $version" | tr -d 'v'` version_prefix="## [0-9]{1,2}\." start=0 CHANGE_LOG="" @@ -19,7 +21,8 @@ main() { CHANGE_LOG+="$line\n" fi done < ${change_log_file} - LINUX_X86_64_BIN_SUM="$(checksum ./linux-x86_64-unknown-linux-gnu)" + + LINUX_X86_64_BIN_SUM="$(checksum "$binaries_dir/thegarii-x86_64-unknown-linux-gnu")" OUTPUT="$(cat <<-END ## Changelog @@ -27,7 +30,7 @@ ${CHANGE_LOG} ## Checksums |Assets | Checksum (sha256)| |-|-| -|thegarii-linux-x86-64 | ${LINUX_X86_64_BIN_SUM}| +|thegarii-x86_64-unknown-linux-gnu | ${LINUX_X86_64_BIN_SUM}| END )" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b28861..bb3cd13 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,17 +93,6 @@ jobs: - name: Checkout Code uses: actions/checkout@v2 - - name: Generate Change Log - id: changelog - if: ${{ startsWith(github.ref, 'refs/tags/') }} - run: | - chmod 755 ./.github/generate_change_log.sh - CHANGELOG=$(./.github/generate_change_log.sh ${{ env.RELEASE_VERSION }}) - - echo "CHANGELOG<> $GITHUB_ENV - echo "$CHANGELOG" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - name: Download All Artifacts id: download-artifacts uses: actions/download-artifact@v2 @@ -130,6 +119,17 @@ jobs: chmod +x "${download_path}/linux-x86_64-unknown-linux-gnu/thegarii" mv "$download_path/linux-x86_64-unknown-linux-gnu/thegarii" "$download_path/thegarii-x86_64-unknown-linux-gnu" + - name: Generate Change Log + id: changelog + if: ${{ startsWith(github.ref, 'refs/tags/') }} + run: | + chmod 755 ./.github/generate_change_log.sh + CHANGELOG=$(./.github/generate_change_log.sh "${{ env.RELEASE_VERSION }}" "${{steps.download-artifacts.outputs.download-path}}") + + echo "CHANGELOG<> $GITHUB_ENV + echo "$CHANGELOG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + - name: Log in to the Container registry uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b329a5..9a43f25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.1 + +- Fixed up automatic changelog generation. + ## 0.2.0 - Re-branded under Firehose (remove all usage of `deep mind`, `DMLOG` and other similar names).