From f2a81b9a58182bb302b06c503d608c8e80aec380 Mon Sep 17 00:00:00 2001 From: Jim McDonald Date: Fri, 20 Sep 2024 23:37:29 +0100 Subject: [PATCH 1/3] Add log encoding option for JSON logging. This option allows selection of either CONSOLE (the default, and the current behaviour) or JSON for log output, allowing for easier parsing of output for structued log backends. --- cmd/juno/juno.go | 12 ++++++-- cmd/juno/juno_test.go | 17 +++++++++++ db/pebble/db.go | 4 +-- node/node.go | 47 ++++++++++++++++--------------- utils/log.go | 65 +++++++++++++++++++++++++++++++++++++++++-- utils/log_test.go | 4 +-- vm/vm_test.go | 2 +- 7 files changed, 117 insertions(+), 34 deletions(-) diff --git a/cmd/juno/juno.go b/cmd/juno/juno.go index c61ee40529..2839e8ed09 100644 --- a/cmd/juno/juno.go +++ b/cmd/juno/juno.go @@ -38,6 +38,7 @@ Juno is a Go implementation of a Starknet full-node client created by Nethermind const ( configF = "config" logLevelF = "log-level" + logEncodingF = "log-encoding" httpF = "http" httpHostF = "http-host" httpPortF = "http-port" @@ -122,6 +123,7 @@ const ( configFlagUsage = "The YAML configuration file." logLevelFlagUsage = "Options: trace, debug, info, warn, error." + logEncodingFlagUsage = "The encoding of the log: console, json." httpUsage = "Enables the HTTP RPC server on the default port and interface." httpHostUsage = "The interface on which the HTTP RPC server will listen for requests." httpPortUsage = "The port on which the HTTP server will listen for requests." @@ -191,9 +193,11 @@ func main() { config := new(node.Config) cmd := NewCmd(config, func(cmd *cobra.Command, _ []string) error { - _, err := fmt.Fprintf(cmd.OutOrStdout(), greeting, Version) - if err != nil { - return err + if config.LogEncoding == utils.CONSOLE { + _, err := fmt.Fprintf(cmd.OutOrStdout(), greeting, Version) + if err != nil { + return err + } } n, err := node.New(config, Version) @@ -302,12 +306,14 @@ func NewCmd(config *node.Config, run func(*cobra.Command, []string) error) *cobr // For testing purposes, these variables cannot be declared outside the function because Cobra // may mutate their values. defaultLogLevel := utils.INFO + defaultLogEncoding := utils.CONSOLE defaultNetwork := utils.Mainnet defaultMaxVMs := 3 * runtime.GOMAXPROCS(0) defaultCNUnverifiableRange := []int{} // Uint64Slice is not supported in Flags() junoCmd.Flags().StringVar(&cfgFile, configF, defaultConfig, configFlagUsage) junoCmd.Flags().Var(&defaultLogLevel, logLevelF, logLevelFlagUsage) + junoCmd.Flags().Var(&defaultLogEncoding, logEncodingF, logEncodingFlagUsage) junoCmd.Flags().Bool(httpF, defaultHTTP, httpUsage) junoCmd.Flags().String(httpHostF, defaulHost, httpHostUsage) junoCmd.Flags().Uint16(httpPortF, defaultHTTPPort, httpPortUsage) diff --git a/cmd/juno/juno_test.go b/cmd/juno/juno_test.go index 162f41a525..35f8db5090 100644 --- a/cmd/juno/juno_test.go +++ b/cmd/juno/juno_test.go @@ -31,6 +31,7 @@ func TestConfigPrecedence(t *testing.T) { // checks on the config, those will be checked by the node implementation. defaultHost := "localhost" defaultLogLevel := utils.INFO + defaultLogEncoding := utils.CONSOLE defaultHTTP := false defaultHTTPPort := uint16(6060) defaultWS := false @@ -84,6 +85,7 @@ func TestConfigPrecedence(t *testing.T) { }, expectedConfig: &node.Config{ LogLevel: utils.DEBUG, + LogEncoding: utils.CONSOLE, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 4576, @@ -115,6 +117,7 @@ func TestConfigPrecedence(t *testing.T) { "custom network config file": { cfgFile: true, cfgFileContents: `log-level: debug +log-encoding: JSON http-host: 0.0.0.0 http-port: 4576 db-path: /home/.juno @@ -129,6 +132,7 @@ cn-unverifiable-range: [0,10] `, expectedConfig: &node.Config{ LogLevel: utils.DEBUG, + LogEncoding: utils.JSON, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 4576, @@ -161,6 +165,7 @@ cn-unverifiable-range: [0,10] inputArgs: []string{""}, expectedConfig: &node.Config{ LogLevel: defaultLogLevel, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: defaultHost, HTTPPort: defaultHTTPPort, @@ -193,6 +198,7 @@ cn-unverifiable-range: [0,10] inputArgs: []string{"--config", ""}, expectedConfig: &node.Config{ LogLevel: defaultLogLevel, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: defaultHost, HTTPPort: defaultHTTPPort, @@ -230,6 +236,7 @@ cn-unverifiable-range: [0,10] cfgFileContents: "\n", expectedConfig: &node.Config{ LogLevel: defaultLogLevel, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: defaultHost, HTTPPort: defaultHTTPPort, @@ -269,6 +276,7 @@ pprof: true `, expectedConfig: &node.Config{ LogLevel: utils.DEBUG, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 4576, @@ -305,6 +313,7 @@ http-port: 4576 `, expectedConfig: &node.Config{ LogLevel: utils.DEBUG, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 4576, @@ -340,6 +349,7 @@ http-port: 4576 }, expectedConfig: &node.Config{ LogLevel: utils.DEBUG, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 4576, @@ -375,6 +385,7 @@ http-port: 4576 }, expectedConfig: &node.Config{ LogLevel: utils.DEBUG, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 4576, @@ -434,6 +445,7 @@ db-cache-size: 8 }, expectedConfig: &node.Config{ LogLevel: utils.ERROR, + LogEncoding: defaultLogEncoding, HTTP: true, HTTPHost: "127.0.0.1", HTTPPort: 4577, @@ -472,6 +484,7 @@ network: sepolia inputArgs: []string{"--db-path", "/home/flag/.juno"}, expectedConfig: &node.Config{ LogLevel: utils.WARN, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 4576, @@ -506,6 +519,7 @@ network: sepolia inputArgs: []string{"--db-path", "/home/flag/.juno", "--pprof"}, expectedConfig: &node.Config{ LogLevel: defaultLogLevel, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: defaultHost, HTTPPort: defaultHTTPPort, @@ -538,6 +552,7 @@ network: sepolia env: []string{"JUNO_HTTP_PORT", "8080", "JUNO_WS", "true", "JUNO_HTTP_HOST", "0.0.0.0"}, expectedConfig: &node.Config{ LogLevel: defaultLogLevel, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: "0.0.0.0", HTTPPort: 8080, @@ -571,6 +586,7 @@ network: sepolia inputArgs: []string{"--db-path", "/home/flag/.juno"}, expectedConfig: &node.Config{ LogLevel: defaultLogLevel, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: defaultHost, HTTPPort: defaultHTTPPort, @@ -604,6 +620,7 @@ network: sepolia env: []string{"JUNO_DB_PATH", "/home/env/.juno", "JUNO_GW_API_KEY", "apikey"}, expectedConfig: &node.Config{ LogLevel: defaultLogLevel, + LogEncoding: defaultLogEncoding, HTTP: defaultHTTP, HTTPHost: defaultHost, HTTPPort: defaultHTTPPort, diff --git a/db/pebble/db.go b/db/pebble/db.go index 6592a602f0..3ecc3beff6 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -31,11 +31,11 @@ func New(path string) (db.DB, error) { return newPebble(path, nil) } -func NewWithOptions(path string, cacheSizeMB uint, maxOpenFiles int, colouredLogger bool) (db.DB, error) { +func NewWithOptions(path string, cacheSizeMB uint, maxOpenFiles int, colouredLogger bool, logEncoding utils.LogEncoding) (db.DB, error) { // Ensure that the specified cache size meets a minimum threshold. cacheSizeMB = max(cacheSizeMB, minCacheSizeMB) - dbLog, err := utils.NewZapLogger(utils.ERROR, colouredLogger) + dbLog, err := utils.NewZapLogger(utils.ERROR, colouredLogger, logEncoding) if err != nil { return nil, fmt.Errorf("create DB logger: %w", err) } diff --git a/node/node.go b/node/node.go index abd0b93990..1764321329 100644 --- a/node/node.go +++ b/node/node.go @@ -45,27 +45,28 @@ const ( // Config is the top-level juno configuration. type Config struct { - LogLevel utils.LogLevel `mapstructure:"log-level"` - HTTP bool `mapstructure:"http"` - HTTPHost string `mapstructure:"http-host"` - HTTPPort uint16 `mapstructure:"http-port"` - RPCCorsEnable bool `mapstructure:"rpc-cors-enable"` - Websocket bool `mapstructure:"ws"` - WebsocketHost string `mapstructure:"ws-host"` - WebsocketPort uint16 `mapstructure:"ws-port"` - GRPC bool `mapstructure:"grpc"` - GRPCHost string `mapstructure:"grpc-host"` - GRPCPort uint16 `mapstructure:"grpc-port"` - DatabasePath string `mapstructure:"db-path"` - Network utils.Network `mapstructure:"network"` - EthNode string `mapstructure:"eth-node"` - Pprof bool `mapstructure:"pprof"` - PprofHost string `mapstructure:"pprof-host"` - PprofPort uint16 `mapstructure:"pprof-port"` - Colour bool `mapstructure:"colour"` - PendingPollInterval time.Duration `mapstructure:"pending-poll-interval"` - RemoteDB string `mapstructure:"remote-db"` - VersionedConstantsFile string `mapstructure:"versioned-constants-file"` + LogLevel utils.LogLevel `mapstructure:"log-level"` + LogEncoding utils.LogEncoding `mapstructure:"log-encoding"` + HTTP bool `mapstructure:"http"` + HTTPHost string `mapstructure:"http-host"` + HTTPPort uint16 `mapstructure:"http-port"` + RPCCorsEnable bool `mapstructure:"rpc-cors-enable"` + Websocket bool `mapstructure:"ws"` + WebsocketHost string `mapstructure:"ws-host"` + WebsocketPort uint16 `mapstructure:"ws-port"` + GRPC bool `mapstructure:"grpc"` + GRPCHost string `mapstructure:"grpc-host"` + GRPCPort uint16 `mapstructure:"grpc-port"` + DatabasePath string `mapstructure:"db-path"` + Network utils.Network `mapstructure:"network"` + EthNode string `mapstructure:"eth-node"` + Pprof bool `mapstructure:"pprof"` + PprofHost string `mapstructure:"pprof-host"` + PprofPort uint16 `mapstructure:"pprof-port"` + Colour bool `mapstructure:"colour"` + PendingPollInterval time.Duration `mapstructure:"pending-poll-interval"` + RemoteDB string `mapstructure:"remote-db"` + VersionedConstantsFile string `mapstructure:"versioned-constants-file"` Metrics bool `mapstructure:"metrics"` MetricsHost string `mapstructure:"metrics-host"` @@ -105,7 +106,7 @@ type Node struct { // New sets the config and logger to the StarknetNode. // Any errors while parsing the config on creating logger will be returned. func New(cfg *Config, version string) (*Node, error) { //nolint:gocyclo,funlen - log, err := utils.NewZapLogger(cfg.LogLevel, cfg.Colour) + log, err := utils.NewZapLogger(cfg.LogLevel, cfg.Colour, cfg.LogEncoding) if err != nil { return nil, err } @@ -115,7 +116,7 @@ func New(cfg *Config, version string) (*Node, error) { //nolint:gocyclo,funlen if dbIsRemote { database, err = remote.New(cfg.RemoteDB, context.TODO(), log, grpc.WithTransportCredentials(insecure.NewCredentials())) } else { - database, err = pebble.NewWithOptions(cfg.DatabasePath, cfg.DBCacheSize, cfg.DBMaxHandles, cfg.Colour) + database, err = pebble.NewWithOptions(cfg.DatabasePath, cfg.DBCacheSize, cfg.DBMaxHandles, cfg.Colour, cfg.LogEncoding) } if err != nil { diff --git a/utils/log.go b/utils/log.go index 3d88ef3cbd..fe1ff4599b 100644 --- a/utils/log.go +++ b/utils/log.go @@ -86,6 +86,65 @@ func (l *LogLevel) UnmarshalText(text []byte) error { return l.Set(string(text)) } +var ErrUnknownLogEncoding = fmt.Errorf( + "unknown log encoding (known: %s, %s)", + CONSOLE, JSON, +) + +type LogEncoding int + +// The following are necessary for Cobra and Viper, respectively, to unmarshal log level +// CLI/config parameters properly. +var ( + _ pflag.Value = (*LogEncoding)(nil) + _ encoding.TextUnmarshaler = (*LogEncoding)(nil) +) + +const ( + CONSOLE LogEncoding = iota + JSON +) + +func (l LogEncoding) String() string { + switch l { + case CONSOLE: + return "console" + case JSON: + return "json" + default: + // Should not happen. + panic(ErrUnknownLogEncoding) + } +} + +func (l LogEncoding) MarshalYAML() (interface{}, error) { + return l.String(), nil +} + +func (l *LogEncoding) Set(s string) error { + switch s { + case "CONSOLE", "console": + *l = CONSOLE + case "JSON", "json": + *l = JSON + default: + return ErrUnknownLogEncoding + } + return nil +} + +func (l *LogEncoding) Type() string { + return "LogEncoding" +} + +func (l *LogEncoding) MarshalText() ([]byte, error) { + return []byte(l.String()), nil +} + +func (l *LogEncoding) UnmarshalText(text []byte) error { + return l.Set(string(text)) +} + type Logger interface { SimpleLogger pebble.Logger @@ -127,12 +186,12 @@ func NewNopZapLogger() *ZapLogger { return &ZapLogger{zap.NewNop().Sugar()} } -func NewZapLogger(logLevel LogLevel, colour bool) (*ZapLogger, error) { +func NewZapLogger(logLevel LogLevel, colour bool, encoding LogEncoding) (*ZapLogger, error) { config := zap.NewProductionConfig() config.Sampling = nil - config.Encoding = "console" + config.Encoding = encoding.String() config.EncoderConfig.EncodeLevel = capitalColorLevelEncoder - if !colour { + if encoding == JSON || !colour { config.EncoderConfig.EncodeLevel = capitalLevelEncoder } config.EncoderConfig.EncodeTime = func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { diff --git a/utils/log_test.go b/utils/log_test.go index 8a5954b872..99ba300315 100644 --- a/utils/log_test.go +++ b/utils/log_test.go @@ -93,7 +93,7 @@ func TestLogLevelType(t *testing.T) { func TestZapWithColour(t *testing.T) { for level, str := range levelStrings { t.Run("level: "+str, func(t *testing.T) { - _, err := utils.NewZapLogger(level, true) + _, err := utils.NewZapLogger(level, true, utils.CONSOLE) assert.NoError(t, err) }) } @@ -102,7 +102,7 @@ func TestZapWithColour(t *testing.T) { func TestZapWithoutColour(t *testing.T) { for level, str := range levelStrings { t.Run("level: "+str, func(t *testing.T) { - _, err := utils.NewZapLogger(level, false) + _, err := utils.NewZapLogger(level, false, utils.CONSOLE) assert.NoError(t, err) }) } diff --git a/vm/vm_test.go b/vm/vm_test.go index 86e4cbe35f..47aca3490f 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -110,7 +110,7 @@ func TestV1Call(t *testing.T) { *classHash: simpleClass, })) - log, err := utils.NewZapLogger(utils.ERROR, false) + log, err := utils.NewZapLogger(utils.ERROR, false, utils.CONSOLE) require.NoError(t, err) // test_storage_read From 1ce063888472f246b4fce16243af972523cd331f Mon Sep 17 00:00:00 2001 From: Jim McDonald Date: Tue, 26 Nov 2024 12:22:34 +0000 Subject: [PATCH 2/3] Linting. --- utils/log.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/log.go b/utils/log.go index fe1ff4599b..683f1972f6 100644 --- a/utils/log.go +++ b/utils/log.go @@ -186,12 +186,12 @@ func NewNopZapLogger() *ZapLogger { return &ZapLogger{zap.NewNop().Sugar()} } -func NewZapLogger(logLevel LogLevel, colour bool, encoding LogEncoding) (*ZapLogger, error) { +func NewZapLogger(logLevel LogLevel, colour bool, logEncoding LogEncoding) (*ZapLogger, error) { config := zap.NewProductionConfig() config.Sampling = nil - config.Encoding = encoding.String() + config.Encoding = logEncoding.String() config.EncoderConfig.EncodeLevel = capitalColorLevelEncoder - if encoding == JSON || !colour { + if logEncoding == JSON || !colour { config.EncoderConfig.EncodeLevel = capitalLevelEncoder } config.EncoderConfig.EncodeTime = func(t time.Time, enc zapcore.PrimitiveArrayEncoder) { From d653ca6eb07cad6eaa53a4254137d984b969e058 Mon Sep 17 00:00:00 2001 From: Jim McDonald Date: Fri, 20 Dec 2024 23:21:45 +0000 Subject: [PATCH 3/3] Pass logger to NewWithOptions --- db/pebble/db.go | 10 ++-------- node/node.go | 6 +++++- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/db/pebble/db.go b/db/pebble/db.go index b98e7393f8..ce499e3e84 100644 --- a/db/pebble/db.go +++ b/db/pebble/db.go @@ -3,7 +3,6 @@ package pebble import ( "context" "errors" - "fmt" "sync" "testing" @@ -36,17 +35,12 @@ func New(path string) (db.DB, error) { return newPebble(path, nil) } -func NewWithOptions(path string, cacheSizeMB uint, maxOpenFiles int, colouredLogger bool, logEncoding utils.LogEncoding) (db.DB, error) { +func NewWithOptions(path string, cacheSizeMB uint, maxOpenFiles int, logger pebble.Logger) (db.DB, error) { // Ensure that the specified cache size meets a minimum threshold. cacheSizeMB = max(cacheSizeMB, minCacheSizeMB) - dbLog, err := utils.NewZapLogger(utils.ERROR, colouredLogger, logEncoding) - if err != nil { - return nil, fmt.Errorf("create DB logger: %w", err) - } - return newPebble(path, &pebble.Options{ - Logger: dbLog, + Logger: logger, Cache: pebble.NewCache(int64(cacheSizeMB * utils.Megabyte)), MaxOpenFiles: maxOpenFiles, }) diff --git a/node/node.go b/node/node.go index e58446adcb..8c314dc243 100644 --- a/node/node.go +++ b/node/node.go @@ -120,7 +120,11 @@ func New(cfg *Config, version string) (*Node, error) { //nolint:gocyclo,funlen if dbIsRemote { database, err = remote.New(cfg.RemoteDB, context.TODO(), log, grpc.WithTransportCredentials(insecure.NewCredentials())) } else { - database, err = pebble.NewWithOptions(cfg.DatabasePath, cfg.DBCacheSize, cfg.DBMaxHandles, cfg.Colour, cfg.LogEncoding) + dbLogger, err := utils.NewZapLogger(utils.ERROR, cfg.Colour, cfg.LogEncoding) + if err != nil { + return nil, errors.Join(errors.New("failed to create database logger"), err) + } + database, err = pebble.NewWithOptions(cfg.DatabasePath, cfg.DBCacheSize, cfg.DBMaxHandles, dbLogger) } if err != nil {