From 4f227e9e41748addb073e3707a34e472c1491025 Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Mon, 26 Feb 2024 15:51:31 -0500 Subject: [PATCH] Added automatic support for `reader-node-bootstrap-url` flag Tarball and bash supported today, refers to changelog for documentation. --- CHANGELOG.md | 26 +++ chain.go | 5 +- cmd/main.go | 8 +- node-manager/boot/eos_bp/config.ini | 29 ---- node-manager/boot/eos_bp/genesis.json | 4 - node-manager/boot/eos_jungle/config.ini | 44 ----- node-manager/boot/eos_jungle/genesis.json | 24 --- node-manager/boot/eos_mainnet/config.ini | 27 ---- node-manager/boot/eos_mainnet/genesis.json | 23 --- node-manager/operator/bootstrap.go | 5 + node-manager/operator/operator.go | 4 - reader_node.go | 22 +++ reader_node_bootstrap.go | 126 +++++++++++++++ reader_node_bootstrapper_bash.go | 177 +++++++++++++++++++++ reader_node_bootstrapper_tarball.go | 98 ++++++++++++ 15 files changed, 464 insertions(+), 158 deletions(-) delete mode 100644 node-manager/boot/eos_bp/config.ini delete mode 100644 node-manager/boot/eos_bp/genesis.json delete mode 100644 node-manager/boot/eos_jungle/config.ini delete mode 100644 node-manager/boot/eos_jungle/genesis.json delete mode 100644 node-manager/boot/eos_mainnet/config.ini delete mode 100644 node-manager/boot/eos_mainnet/genesis.json create mode 100644 node-manager/operator/bootstrap.go create mode 100644 reader_node.go create mode 100644 reader_node_bootstrap.go create mode 100644 reader_node_bootstrapper_bash.go create mode 100644 reader_node_bootstrapper_tarball.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 37897f5..f7a9d9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,32 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s ## Unreleased +* The `reader-node-bootstrap-url` gained the ability to be bootstrapped from a `bash` script. + + If the bootstrap URL is of the form `bash:///?`, the bash script at + `` will be executed. The script is going to receive in environment variables the resolved + reader node variables in the form of `READER_NODE_`. The fully resolved node arguments + (from `reader-node-arguments`) are passed as args to the bash script. The query parameters accepted are: + + - `arg=` | Pass as extra argument to the script, prepended to the list of resolved node arguments + - `env=%3d` | Pass as extra environment variable as `=` with key being upper-cased (multiple(s) allowed) + - `env_=` | Pass as extra environment variable as `=` with key being upper-cased (multiple(s) allowed) + - `cwd=` | Change the working directory to `` before running the script + - `interpreter=` | Use `` as the interpreter to run the script + - `interpreter_arg=` | Pass `` as arguments to the interpreter before the script path (multiple(s) allowed) + + > [!NOTE] + > The `bash:///` script support is currently experimental and might change in upcoming releases, the behavior changes will be + clearly documented here. + +* The `reader-node-bootstrap-url` gained the ability to be bootstrapped from a pre-made archive file ending with `tar.zst` or `tar.zstd`. + +* The `reader-node-bootstrap-data-url` is now added automatically if `firecore.Chain#ReaderNodeBootstrapperFactory` is `non-nil`. + + If the bootstrap URL ends with `tar.zst` or `tar.zstd`, the archive is read and extracted into the + `reader-node-data-dir` location. The archive is expected to contain the full content of the 'reader-node-data-dir' + and is expanded as is. + * Added `Beacon` to known list of Block model. ## v1.2.3 diff --git a/chain.go b/chain.go index 8268aa2..2099e5a 100644 --- a/chain.go +++ b/chain.go @@ -289,8 +289,11 @@ func (c *Chain[B]) Validate() { // **Caveats** Two chain in the same Go binary will not work today as `bstream` uses global // variables to store configuration which presents multiple chain to exist in the same process. func (c *Chain[B]) Init() { - c.BlockEncoder = NewBlockEncoder() + + if c.ReaderNodeBootstrapperFactory == nil { + c.ReaderNodeBootstrapperFactory = DefaultReaderNodeBootstrapper(noOpReaderNodeBootstrapperFactory) + } } // BinaryName represents the binary name for your Firehose on is the [ShortName] diff --git a/cmd/main.go b/cmd/main.go index d554753..46f1706 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -108,11 +108,15 @@ func Main[B firecore.Block](chain *firecore.Chain[B]) { apps.RegisterIndexBuilderApp(chain, rootLog) } + startFlags := apps.StartCmd.Flags() + if chain.RegisterExtraStartFlags != nil { - chain.RegisterExtraStartFlags(apps.StartCmd.Flags()) + chain.RegisterExtraStartFlags(startFlags) } - apps.ConfigureStartCmd(chain, binaryName, rootLog) + if chain.ReaderNodeBootstrapperFactory != nil && startFlags.Lookup("reader-node-bootstrap-data-url") == nil { + startFlags.String("reader-node-bootstrap-data-url", "", firecore.DefaultReaderNodeBootstrapDataURLFlagDescription()) + } if err := tools.ConfigureToolsCmd(chain, rootLog, rootTracer); err != nil { exitWithError("registering tools command", err) diff --git a/node-manager/boot/eos_bp/config.ini b/node-manager/boot/eos_bp/config.ini deleted file mode 100644 index b388df7..0000000 --- a/node-manager/boot/eos_bp/config.ini +++ /dev/null @@ -1,29 +0,0 @@ -# Chain -abi-serializer-max-time-ms = 500000 -chain-state-db-size-mb = 5000 -max-transaction-time = 5000 - -# P2P -agent-name = eos_bp -p2p-server-address = 0.0.0.0:9876 -p2p-max-nodes-per-host = 5 -connection-cleanup-period = 15 - -# HTTP -access-control-allow-origin = * -http-server-address = 0.0.0.0:8888 -http-max-response-time-ms = 1000 -http-validate-host = 0 -verbose-http-errors = true - -plugin = eosio::db_size_api_plugin -plugin = eosio::net_api_plugin -plugin = eosio::chain_api_plugin -plugin = eosio::producer_api_plugin - -# We want to produce the block logs, no deep-mind instrumentation here. -producer-name = eosio -producer-name = eosio2 -producer-name = eosio3 -enable-stale-production = true -signature-provider = EOS5MHPYyhjBjnQZejzZHqHewPWhGTfQWSVTWYEhDmJu4SXkzgweP=KEY:5JpjqdhVCQTegTjrLtCSXHce7c9M8w7EXYZS7xC13jVFF4Phcrx \ No newline at end of file diff --git a/node-manager/boot/eos_bp/genesis.json b/node-manager/boot/eos_bp/genesis.json deleted file mode 100644 index 4d648b8..0000000 --- a/node-manager/boot/eos_bp/genesis.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "initial_key": "EOS5MHPYyhjBjnQZejzZHqHewPWhGTfQWSVTWYEhDmJu4SXkzgweP", - "initial_timestamp": "2019-05-23T19:18:34" -} \ No newline at end of file diff --git a/node-manager/boot/eos_jungle/config.ini b/node-manager/boot/eos_jungle/config.ini deleted file mode 100644 index f55ba3d..0000000 --- a/node-manager/boot/eos_jungle/config.ini +++ /dev/null @@ -1,44 +0,0 @@ -http-server-address = 0.0.0.0:8888 -p2p-listen-endpoint = 0.0.0.0:9876 -p2p-server-address = 0.0.0.0:9876 -p2p-max-nodes-per-host = 2 -connection-cleanup-period = 60 -verbose-http-errors = true -chain-state-db-size-mb = 64000 -# shared-memory-size-mb = 2048 -reversible-blocks-db-size-mb = 2048 -http-validate-host = false -max-transaction-time = 5000 -abi-serializer-max-time-ms = 500000 -# read-mode = read-only # we want internal connectivity - -plugin = eosio::net_api_plugin -plugin = eosio::chain_api_plugin -plugin = eosio::db_size_api_plugin -plugin = eosio::producer_api_plugin - -# Max speed for replay -# validation-mode = light -# wasm-runtime = wavm - -# Enable deep mind -# deep-mind = 1 -# deep-mind-console = 1 - -agent-name = dfuse dev -p2p-peer-address = 145.239.133.201:9876 -p2p-peer-address = 163.172.34.128:9876 -p2p-peer-address = 34.73.143.228:9876 -p2p-peer-address = 47.244.11.76:9876 -p2p-peer-address = 88.99.193.44:9876 -p2p-peer-address = bp4-d3.eos42.io:9876 -p2p-peer-address = jungle2.cryptolions.io:19876 -p2p-peer-address = jungle2.cryptolions.io:9876 -p2p-peer-address = jungle2-eos.blckchnd.com:9876 -p2p-peer-address = jungle2.eosdac.io:9872 -p2p-peer-address = jungle.eosamsterdam.net:9876 -p2p-peer-address = jungle.eoscafeblock.com:9876 -p2p-peer-address = jungle.eosusa.news:19876 -p2p-peer-address = junglepeer.eossweden.se:9876 -p2p-peer-address = peer1-jungle.eosphere.io:9876 -p2p-peer-address = peer.jungle.alohaeos.com:9876 diff --git a/node-manager/boot/eos_jungle/genesis.json b/node-manager/boot/eos_jungle/genesis.json deleted file mode 100644 index 5d1a4d6..0000000 --- a/node-manager/boot/eos_jungle/genesis.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "initial_configuration": { - "base_per_transaction_net_usage": 12, - "context_free_discount_net_usage_den": 100, - "context_free_discount_net_usage_num": 20, - "deferred_trx_expiration_window": 600, - "max_authority_depth": 6, - "max_block_cpu_usage": 200000, - "max_block_net_usage": 1048576, - "max_inline_action_depth": 4, - "max_inline_action_size": 4096, - "max_ram_size": 34359738368, - "max_transaction_cpu_usage": 150000, - "max_transaction_delay": 3888000, - "max_transaction_lifetime": 3600, - "max_transaction_net_usage": 524288, - "min_transaction_cpu_usage": 100, - "net_usage_leeway": 500, - "target_block_cpu_usage_pct": 2000, - "target_block_net_usage_pct": 1000 - }, - "initial_key": "EOS8bRkmrfsQSmb87ix1EuFSe2NDsepKGCjUNgLEt1SDqw1fuhG4v", - "initial_timestamp": "2018-11-23T16:20:00" -} \ No newline at end of file diff --git a/node-manager/boot/eos_mainnet/config.ini b/node-manager/boot/eos_mainnet/config.ini deleted file mode 100644 index 26bd919..0000000 --- a/node-manager/boot/eos_mainnet/config.ini +++ /dev/null @@ -1,27 +0,0 @@ -http-server-address = 0.0.0.0:8888 -agent-name = dfuse dev -p2p-server-address = 0.0.0.0:9876 -p2p-max-nodes-per-host = 2 -connection-cleanup-period = 60 -verbose-http-errors = true -chain-state-db-size-mb = 64000 -reversible-blocks-db-size-mb = 2048 -# shared-memory-size-mb = 2048 -http-validate-host = false -max-transaction-time = 5000 -abi-serializer-max-time-ms = 500000 -read-mode = read-only - -plugin = eosio::net_api_plugin -plugin = eosio::chain_api_plugin -plugin = eosio::db_size_api_plugin -plugin = eosio::producer_api_plugin - -# Enable deep mind -# deep-mind = 1 -# deep-mind-console = 1 - -p2p-peer-address = publicnode.cypherglass.com:9876 -p2p-peer-address = mars.fnp2p.eosbixin.com:443 -p2p-peer-address = fullnode.eoslaomao.com:443 -p2p-peer-address = peer.main.alohaeos.com:9876 \ No newline at end of file diff --git a/node-manager/boot/eos_mainnet/genesis.json b/node-manager/boot/eos_mainnet/genesis.json deleted file mode 100644 index 1d784c7..0000000 --- a/node-manager/boot/eos_mainnet/genesis.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "initial_timestamp": "2018-06-08T08:08:08.888", - "initial_key": "EOS7EarnUhcyYqmdnPon8rm7mBCTnBoot6o7fE2WzjvEX2TdggbL3", - "initial_configuration": { - "max_block_net_usage": 1048576, - "target_block_net_usage_pct": 1000, - "max_transaction_net_usage": 524288, - "base_per_transaction_net_usage": 12, - "net_usage_leeway": 500, - "context_free_discount_net_usage_num": 20, - "context_free_discount_net_usage_den": 100, - "max_block_cpu_usage": 200000, - "target_block_cpu_usage_pct": 1000, - "max_transaction_cpu_usage": 150000, - "min_transaction_cpu_usage": 100, - "max_transaction_lifetime": 3600, - "deferred_trx_expiration_window": 600, - "max_transaction_delay": 3888000, - "max_inline_action_size": 4096, - "max_inline_action_depth": 4, - "max_authority_depth": 6 - } -} \ No newline at end of file diff --git a/node-manager/operator/bootstrap.go b/node-manager/operator/bootstrap.go new file mode 100644 index 0000000..5f69afc --- /dev/null +++ b/node-manager/operator/bootstrap.go @@ -0,0 +1,5 @@ +package operator + +type Bootstrapper interface { + Bootstrap() error +} diff --git a/node-manager/operator/operator.go b/node-manager/operator/operator.go index 03428e3..1667a99 100644 --- a/node-manager/operator/operator.go +++ b/node-manager/operator/operator.go @@ -48,10 +48,6 @@ type Operator struct { zlogger *zap.Logger } -type Bootstrapper interface { - Bootstrap() error -} - type Options struct { Bootstrapper Bootstrapper diff --git a/reader_node.go b/reader_node.go new file mode 100644 index 0000000..3ef2197 --- /dev/null +++ b/reader_node.go @@ -0,0 +1,22 @@ +package firecore + +import "golang.org/x/exp/maps" + +var ReaderNodeVariablesDocumentation = map[string]string{ + "{data-dir}": "The current data-dir path defined by the flag 'data-dir'", + "{node-data-dir}": "The node data dir path defined by the flag 'reader-node-data-dir'", + "{hostname}": "The machine's hostname", + "{start-block-num}": "The resolved start block number defined by the flag 'reader-node-start-block-num' (can be overwritten)", + "{stop-block-num}": "The stop block number defined by the flag 'reader-node-stop-block-num'", +} + +var ReaderNodeVariables = maps.Keys(ReaderNodeVariablesDocumentation) + +func ReaderNodeVariablesValues(resolver ReaderNodeArgumentResolver) map[string]string { + values := make(map[string]string, len(ReaderNodeVariables)) + for _, variable := range ReaderNodeVariables { + values[variable] = resolver(variable) + } + + return values +} diff --git a/reader_node_bootstrap.go b/reader_node_bootstrap.go new file mode 100644 index 0000000..f7136ae --- /dev/null +++ b/reader_node_bootstrap.go @@ -0,0 +1,126 @@ +package firecore + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + "github.com/streamingfast/cli" + "github.com/streamingfast/cli/sflags" + "github.com/streamingfast/firehose-core/node-manager/operator" + "go.uber.org/zap" +) + +type ReaderNodeBootstrapperFactory func( + ctx context.Context, + logger *zap.Logger, + cmd *cobra.Command, + resolvedNodeArguments []string, + resolver ReaderNodeArgumentResolver, +) (operator.Bootstrapper, error) + +// noOpReaderNodeBootstrapperFactory is a factory that returns always `nil, nil` and is used +// as an empty override for the default bootstrapper logic. +func noOpReaderNodeBootstrapperFactory(ctx context.Context, logger *zap.Logger, cmd *cobra.Command, resolvedNodeArguments []string, resolver ReaderNodeArgumentResolver) (operator.Bootstrapper, error) { + return nil, nil +} + +func DefaultReaderNodeBootstrapDataURLFlagDescription() string { + return cli.Dedent(` + When specified, if the reader node is emtpy (e.g. that 'reader-node-data-dir' location doesn't exist + or has no file within it), the 'reader-node' is going to be boostrapped from it. The exact bootstrapping + behavior depends on the URL received. + + If the bootstrap URL is of the form 'bash:///?', the bash script at + '' will be executed. The script is going to receive in environment variables the resolved + reader node variables in the form of 'READER_NODE_'. The fully resolved node arguments + (from 'reader-node-arguments') are passed as args to the bash script. The query parameters accepted are: + + - arg= | Pass as extra argument to the script, prepended to the list of resolved node arguments + - env=%%3d | Pass as extra environment variable as = with key being upper-cased (multiple(s) allowed) + - env_= | Pass as extra environment variable as = with key being upper-cased (multiple(s) allowed) + - cwd= | Change the working directory to before running the script + - interpreter= | Use as the interpreter to run the script + - interpreter_arg= | Pass as arguments to the interpreter before the script path (multiple(s) allowed) + + If the bootstrap URL ends with 'tar.zst' or 'tar.zstd', the archive is read and extracted into the + 'reader-node-data-dir' location. The archive is expected to contain the full content of the 'reader-node-data-dir' + and is expanded as is. + `) + "\n" +} + +// DefaultReaderNodeBootstrapper is a constrtuction you can when you want the default bootstrapper logic to be applied +// but you need support new bootstrap data URL(s) format or override the default behavior for some type. +// +// The `overrideFactory` argument is a factory function that will be called first, if it returns a non-nil bootstrapper, +// it will be used and the default logic will be skipped. If it returns nil, the default logic will be applied. +func DefaultReaderNodeBootstrapper( + overrideFactory ReaderNodeBootstrapperFactory, +) ReaderNodeBootstrapperFactory { + return func( + ctx context.Context, + logger *zap.Logger, + cmd *cobra.Command, + resolvedNodeArguments []string, + resolver ReaderNodeArgumentResolver, + ) (operator.Bootstrapper, error) { + bootstrapDataURL := sflags.MustGetString(cmd, "reader-node-bootstrap-data-url") + if bootstrapDataURL == "" { + return nil, nil + } + + nodeDataDir := resolver("{node-data-dir}") + + if overrideFactory == nil { + panic("overrideFactory argument must be set") + } + + bootstrapper, err := overrideFactory(ctx, logger, cmd, resolvedNodeArguments, resolver) + if err != nil { + return nil, fmt.Errorf("override factory failed: %w", err) + } + + if bootstrapper != nil { + return bootstrapper, nil + } + + // Otherwise apply the default logic + switch { + case strings.HasSuffix(bootstrapDataURL, "tar.zst") || strings.HasSuffix(bootstrapDataURL, "tar.zstd"): + // There could be a mistmatch here if the user override `--datadir` manually, we live it for now + return NewTarballReaderNodeBootstrapper(bootstrapDataURL, nodeDataDir, logger), nil + + case strings.HasPrefix(bootstrapDataURL, "bash://"): + return NewBashNodeReaderBootstrapper(cmd, bootstrapDataURL, resolver, resolvedNodeArguments, logger), nil + + default: + return nil, fmt.Errorf("'reader-node-bootstrap-data-url' config should point to either an archive ending in '.tar.zstd' or a genesis file ending in '.json', not %s", bootstrapDataURL) + } + } +} + +func isBootstrapped(dataDir string, logger *zap.Logger) bool { + var foundFile bool + err := filepath.Walk(dataDir, + func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + // As soon as there is a file, we assume it's bootstrapped + foundFile = true + return io.EOF + }) + if err != nil && !os.IsNotExist(err) && err != io.EOF { + logger.Warn("error while checking for bootstrapped status", zap.Error(err)) + } + + return foundFile +} diff --git a/reader_node_bootstrapper_bash.go b/reader_node_bootstrapper_bash.go new file mode 100644 index 0000000..e086010 --- /dev/null +++ b/reader_node_bootstrapper_bash.go @@ -0,0 +1,177 @@ +package firecore + +import ( + "context" + "fmt" + "net/url" + "os" + "os/exec" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/streamingfast/cli/sflags" + "go.uber.org/zap" +) + +func NewBashNodeReaderBootstrapper( + cmd *cobra.Command, + url string, + resolver ReaderNodeArgumentResolver, + resolvedNodeArguments []string, + logger *zap.Logger, +) *BashNodeBootstrapper { + dataDir := resolver("{node-data-dir}") + binaryPath := sflags.MustGetString(cmd, "reader-node-path") + + return &BashNodeBootstrapper{ + url: url, + dataDir: dataDir, + binaryPath: binaryPath, + resolver: resolver, + resolvedNodeArguments: resolvedNodeArguments, + logger: logger, + } +} + +type BashNodeBootstrapper struct { + url string + dataDir string + binaryPath string + resolver ReaderNodeArgumentResolver + resolvedNodeArguments []string + logger *zap.Logger +} + +func (b *BashNodeBootstrapper) isBootstrapped() bool { + return isBootstrapped(b.dataDir, b.logger) +} + +func (b *BashNodeBootstrapper) Bootstrap() error { + if b.isBootstrapped() { + return nil + } + + b.logger.Info("bootstrapping chain data from bash script", zap.String("bootstrap_data_url", b.url)) + + url, err := url.Parse(b.url) + if err != nil { + return fmt.Errorf("cannot parse bootstrap data URL %q: %w", b.url, err) + } + + // Should not happen but let's play safe and check the scheme for now + if url.Scheme != "bash" { + return fmt.Errorf("unsupported bootstrap data URL scheme %q", url.Scheme) + } + + parameters := url.Query() + scriptPath := b.scriptPath(url) + + interpreter := "bash" + if interpreterOverride := parameters.Get("interpreter"); interpreterOverride != "" { + interpreter = interpreterOverride + } + workingDirectory := parameters.Get("cwd") + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + args := []string{} + // First arguments are those to be passed to the interpeter + args = append(args, parameters["interpreter_arg"]...) + // Then the argument is always the script path + args = append(args, scriptPath) + // Next arguments are those provided in the URL query part with the `arg` key (multiple(s) allowed) + args = append(args, parameters["arg"]...) + // Then we append the resolved node arguments + args = append(args, b.resolvedNodeArguments...) + + cmd := exec.CommandContext(ctx, interpreter, args...) + if workingDirectory != "" { + cmd.Dir = workingDirectory + } + + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, b.scriptCustomEnv(url, parameters)...) + cmd.Env = append(cmd.Env, b.nodeVariablesToEnv()...) + cmd.Env = append(cmd.Env, fmt.Sprintf("READER_NODE_BINARY_PATH=%s", b.binaryPath)) + + // Everything goes to `os.Stderr`, standard logging in nodes is to go to `os.Stderr` + // so by default, we output everything to `os.Stderr`. + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("bootstrap script %q failed: %w", cmd.String(), err) + } + + return nil +} + +func (b *BashNodeBootstrapper) scriptPath(bootstrapURL *url.URL) string { + scriptPath := bootstrapURL.EscapedPath() + + // We handle relative paths differently to ensure they are properly + // resolved to the current working directory. + if strings.HasPrefix(scriptPath, "/.") { + return scriptPath[1:] + } + + return scriptPath +} + +// scriptCustomEnv returns the custom environment variables to be set when running the bootstrap script +// based on the URL query parameters. +// +// We support actually two form(s): `env=%3d` and `env_=`. When ecountering +// the first form, the key is made uppercase and the value is URL-decoded and we appendd them to the +// environment. When encountering the second form, we trim the `env_` prefix, make the key uppercase +// and append the value to the environment. +func (b *BashNodeBootstrapper) scriptCustomEnv(bootstrapURL *url.URL, parameters url.Values) (out []string) { + for k, values := range parameters { + for _, value := range values { + if k == "env" { + // We assume the full element is already URL-decoded + parts := strings.SplitN(value, "=", 2) + if len(parts) != 2 { + b.logger.Warn("invalid env URL query parameter", zap.String("key", k), zap.String("value", value)) + continue + } + + out = append(out, fmt.Sprintf("%s=%s", strings.ToUpper(parts[0]), parts[1])) + } + + if strings.HasPrefix(k, "env_") { + out = append(out, fmt.Sprintf("%s=%s", strings.ToUpper(k[4:]), value)) + } + } + } + + return +} + +func (b *BashNodeBootstrapper) nodeVariablesToEnv() []string { + variablesValues := ReaderNodeVariablesValues(b.resolver) + if len(variablesValues) == 0 { + return nil + } + + i := 0 + env := make([]string, len(variablesValues)) + for k, v := range variablesValues { + env[i] = fmt.Sprintf("%s=%s", variableNameToEnvName(k), v) + i++ + } + + return env +} + +func variableNameToEnvName(variable string) string { + name := strings.ToUpper(variable) + name = strings.ReplaceAll(name, "-", "_") + name = strings.TrimSpace(name) + name = strings.TrimPrefix(name, "{") + name = strings.TrimSuffix(name, "}") + + return "READER_NODE_" + name +} diff --git a/reader_node_bootstrapper_tarball.go b/reader_node_bootstrapper_tarball.go new file mode 100644 index 0000000..8d2010b --- /dev/null +++ b/reader_node_bootstrapper_tarball.go @@ -0,0 +1,98 @@ +package firecore + +import ( + "archive/tar" + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/streamingfast/dstore" + "go.uber.org/zap" +) + +func NewTarballReaderNodeBootstrapper( + url string, + dataDir string, + logger *zap.Logger, +) *TarballNodeBootstrapper { + return &TarballNodeBootstrapper{ + url: url, + dataDir: dataDir, + logger: logger, + } +} + +type TarballNodeBootstrapper struct { + url string + dataDir string + logger *zap.Logger +} + +func (b *TarballNodeBootstrapper) isBootstrapped() bool { + return isBootstrapped(b.dataDir, b.logger) +} + +func (b *TarballNodeBootstrapper) Bootstrap() error { + if b.isBootstrapped() { + return nil + } + + b.logger.Info("bootstrapping geth chain data from pre-built data", zap.String("bootstrap_data_url", b.url)) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + reader, _, _, err := dstore.OpenObject(ctx, b.url, dstore.Compression("zstd")) + if err != nil { + return fmt.Errorf("cannot get snapshot from gstore: %w", err) + } + defer reader.Close() + + b.createChainData(reader) + return nil +} + +func (b *TarballNodeBootstrapper) createChainData(reader io.Reader) error { + err := os.MkdirAll(b.dataDir, os.ModePerm) + if err != nil { + return fmt.Errorf("unable to create blocks log file: %w", err) + } + + b.logger.Info("extracting bootstrapping data into node data directory", zap.String("data_dir", b.dataDir)) + tr := tar.NewReader(reader) + for { + header, err := tr.Next() + if err != nil { + if err == io.EOF { + return nil + } + + return err + } + + path := filepath.Join(b.dataDir, header.Name) + b.logger.Debug("about to write content of entry", zap.String("name", header.Name), zap.String("path", path), zap.Bool("is_dir", header.FileInfo().IsDir())) + if header.FileInfo().IsDir() { + err = os.MkdirAll(path, os.ModePerm) + if err != nil { + return fmt.Errorf("unable to create directory: %w", err) + } + + continue + } + + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("unable to create file: %w", err) + } + + if _, err := io.Copy(file, tr); err != nil { + file.Close() + return err + } + file.Close() + } +}