diff --git a/.gitignore b/.gitignore index 4385204..fa5476a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ magellan +emulator/rf-emulator **/*.db dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md index f999cdb..ebc8586 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Ability to update firmware * Refactored connection handling for faster scanning - * Updated to refelct home at github.com/OpenCHAMI + * Updated to reflect home at github.com/OpenCHAMI * Updated to reflect ghcr.io as container home ## [Unreleased] diff --git a/Makefile b/Makefile index c21ed53..f8de63c 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,12 @@ $(error VERSION is not set. Please review and copy config.env.default to config endif SHELL := /bin/bash +GOPATH ?= $(shell echo $${GOPATH:-~/go}) .DEFAULT_GOAL := all .PHONY: all all: ## build pipeline -all: mod inst build spell lint test +all: mod inst build lint test .PHONY: ci ci: ## CI build pipeline @@ -47,27 +48,30 @@ inst: ## go install tools go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.52.2 go install github.com/goreleaser/goreleaser@v1.18.2 +.PHONY: goreleaser +release: ## goreleaser build + $(call print-target) + $(GOPATH)/bin/goreleaser build --clean --single-target --snapshot + .PHONY: build build: ## goreleaser build -build: - $(call print-target) - goreleaser build --clean --single-target --snapshot + go build --tags=all .PHONY: docker -docker: ## docker build -docker: +container: ## docker build +container: $(call print-target) docker build . --build-arg REGISTRY_HOST=${REGISTRY_HOST} --no-cache --pull --tag '${NAME}:${VERSION}' .PHONY: spell spell: ## misspell $(call print-target) - misspell -error -locale=US -w **.md + $(GOPATH)/bin/misspell -error -locale=US -w **.md .PHONY: lint lint: ## golangci-lint $(call print-target) - golangci-lint run --fix + $(GOPATH)/bin/golangci-lint run --fix .PHONY: test test: ## go test @@ -88,6 +92,11 @@ docs: ## go docs go doc github.com/OpenCHAMI/magellan/internal go doc github.com/OpenCHAMI/magellan/pkg/crawler +.PHONY: emulator +emulator: + $(call print-target) + ./emulator/setup.sh + define print-target @printf "Executing target: \033[36m$@\033[0m\n" endef diff --git a/README.md b/README.md index 52db387..a9e9cc1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,20 @@ The `magellan` CLI tool is a Redfish-based, board management controller (BMC) discovery tool designed to scan networks and is written in Go. The tool collects information from BMC nodes using the provided Redfish RESTful API with [`gofish`](https://github.com/stmcginnis/gofish) and loads the queried data into an [SMD](https://github.com/OpenCHAMI/smd/tree/master) instance. The tool strives to be more flexible by implementing multiple methods of discovery to work for a wider range of systems (WIP) and is capable of using independently of other tools or services. +**Note: `magellan` v0.1.0 is incompatible with SMD v2.15.3 and earlier.** + +## Main Features + +The `magellan` tool comes packed with a handleful of features for doing discovery, such as: + +- Simple network scanning +- Redfish-based inventory collection +- Redfish-based firmware updating +- Integration with OpenCHAMI SMD +- Write inventory data to JSON + +See the [TODO](#todo) section for a list of soon-ish goals planned. + ## Getting Started [Build](#building) and [run on bare metal](#running-the-tool) or run and test with Docker using the [latest prebuilt image](#running-with-docker). For quick testing, the repository integrates a Redfish emulator that can be ran by executing the `emulator/setup.sh` script or running `make emulator`. @@ -52,10 +66,6 @@ docker pull ghcr.io/openchami/magellan:latest See the ["Running with Docker"](#running-with-docker) section below about running with the Docker container. - - - - ## Usage The sections below assume that the BMC nodes have an IP address available to query Redfish. Currently, `magellan` does not support discovery with MAC addresses although that may change in the future. @@ -89,7 +99,9 @@ This should return a JSON response with general information. The output below ha } ``` -To see all of the available commands, run `magellan` with the `help` subcommand: +### Running the Tool + +There are three main commands to use with the tool: `scan`, `list`, and `collect`. To see all of the available commands, run `magellan` with the `help` subcommand: ```bash ./magellan help @@ -120,9 +132,7 @@ Flags: Use "magellan [command] --help" for more information about a command. ``` -### Running the Tool - -There are three main commands to use with the tool: `scan`, `list`, and `collect`. To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe ports 623 and 443 by default: +To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe the common Redfish port 443 by default: ```bash ./magellan scan \ @@ -162,7 +172,7 @@ Note: If the `cache` flag is not set, `magellan` will use "/tmp/$USER/magellan.d ### Updating Firmware -The `magellan` tool is capable of updating firmware with using the `update` subcommand via the Redfish API. This may sometimes necessary if some of the `collect` output is missing or is not including what is expected. The subcommand expects there to be a running HTTP/HTTPS server running that has an accessbile URL path to the firmware download. Specify the URL with the `--firmware-path` flag and the firmware type with the `--component` flag with all the other usual arguments like in the example below: +The `magellan` tool is capable of updating firmware with using the `update` subcommand via the Redfish API. This may sometimes necessary if some of the `collect` output is missing or is not including what is expected. The subcommand expects there to be a running HTTP/HTTPS server running that has an accessible URL path to the firmware download. Specify the URL with the `--firmware-path` flag and the firmware type with the `--component` flag with all the other usual arguments like in the example below: ```bash ./magellan update \ @@ -220,7 +230,7 @@ At its core, `magellan` is designed to do three basic things: First, the tool performs a scan to find running services on a network. This is done by sending a raw TCP packet to all specified hosts (either IP or host name) and taking note which services respond. At this point, `magellan` has no way of knowing whether this is a Redfish service or not, so another HTTP request is made to verify. Once the BMC responds with an OK status code, `magellan` will store the necessary information in a local cache database to allow collecting more information about the node later. This allows for users to only have to scan their cluster once to find systems that are currently available and scannable. -Next, the tool queries information about the BMC node using `gofish` API functions, but requires access to BMC node found in the scanning step mentioned above to work. If the node requires basic authentication, a user name and password is required to be supplied as well. Once the BMC information is retrived from each node, the info is aggregated and a HTTP request is made to a SMD instance to be stored. Optionally, the information can be written to disk for inspection and debugging purposes. +Next, the tool queries information about the BMC node using `gofish` API functions, but requires access to BMC node found in the scanning step mentioned above to work. If the node requires basic authentication, a user name and password is required to be supplied as well. Once the BMC information is retrieved from each node, the info is aggregated and a HTTP request is made to a SMD instance to be stored. Optionally, the information can be written to disk for inspection and debugging purposes. In summary, `magellan` needs at minimum the following configured to work on each node: @@ -234,13 +244,13 @@ See the [issue list](https://github.com/OpenCHAMI/magellan/issues) for plans for * [X] Confirm loading different components into SMD * [X] Add ability to set subnet mask for scanning -* [ ] Add ability to scan with other protocols like LLDP -* [ ] Add more debugging messages with the `-v/--verbose` flag +* [ ] Add ability to scan with other protocols like LLDP and SSDP +* [X] Add more debugging messages with the `-v/--verbose` flag * [ ] Separate `collect` subcommand with making request to endpoint * [X] Support logging in with `opaal` to get access token * [X] Support using CA certificates with HTTP requests to SMD -* [ ] Add unit tests for `scan`, `list`, and `collect` commands -* [ ] Clean up, remove unused, and tidy code +* [ ] Add tests for the regressions and compatibility +* [X] Clean up, remove unused, and tidy code (first round) ## Copyright diff --git a/cmd/collect.go b/cmd/collect.go index 3641a55..1f3288a 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -5,19 +5,15 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/api/smd" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" - "github.com/OpenCHAMI/magellan/internal/log" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" + urlx "github.com/OpenCHAMI/magellan/internal/url" + "github.com/OpenCHAMI/magellan/pkg/auth" "github.com/cznic/mathutil" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) -var ( - forceUpdate bool -) - // The `collect` command fetches data from a collection of BMC nodes. // This command should be ran after the `scan` to find available hosts // on a subnet. @@ -30,36 +26,39 @@ var collectCmd = &cobra.Command{ " magellan collect --cache ./assets.db --output ./logs --timeout 30 --cacert cecert.pem\n" + " magellan collect --host smd.example.com --port 27779 --username username --password password", Run: func(cmd *cobra.Command, args []string) { - // make application logger - l := log.NewLogger(logrus.New(), logrus.DebugLevel) - // get probe states stored in db from scan - probeStates, err := sqlite.GetProbeResults(cachePath) + scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { - l.Log.Errorf("failed toget states: %v", err) + log.Error().Err(err).Msgf("failed to get scanned results from cache") + } + + // URL sanitanization for host argument + host, err = urlx.Sanitize(host) + if err != nil { + log.Error().Err(err).Msg("failed to sanitize host") } // try to load access token either from env var, file, or config if var not set if accessToken == "" { var err error - accessToken, err = LoadAccessToken() - if err != nil { - l.Log.Errorf("failed to load access token: %v", err) + accessToken, err = auth.LoadAccessToken(tokenPath) + if err != nil && verbose { + log.Warn().Err(err).Msgf("could not load access token") } } if verbose { - fmt.Printf("access token: %v\n", accessToken) + log.Debug().Str("Access Token", accessToken) } // if concurrency <= 0 { - concurrency = mathutil.Clamp(len(probeStates), 1, 255) + concurrency = mathutil.Clamp(len(scannedResults), 1, 10000) } - q := &magellan.QueryParams{ + err = magellan.CollectInventory(&scannedResults, &magellan.CollectParams{ + URI: host, Username: username, Password: password, - Protocol: protocol, Timeout: timeout, Concurrency: concurrency, Verbose: verbose, @@ -67,45 +66,37 @@ var collectCmd = &cobra.Command{ OutputPath: outputPath, ForceUpdate: forceUpdate, AccessToken: accessToken, - } - err = magellan.CollectAll(&probeStates, l, q) + }) if err != nil { - l.Log.Errorf("failed to collect data: %v", err) - } - - // add necessary headers for final request (like token) - headers := make(map[string]string) - if q.AccessToken != "" { - headers["Authorization"] = "Bearer " + q.AccessToken + log.Error().Err(err).Msgf("failed to collect data") } }, } func init() { currentUser, _ = user.Current() - collectCmd.PersistentFlags().StringVar(&smd.Host, "host", smd.Host, "set the host to the SMD API") - collectCmd.PersistentFlags().IntVarP(&smd.Port, "port", "p", smd.Port, "set the port to the SMD API") - collectCmd.PersistentFlags().StringVar(&username, "username", "", "set the BMC user") - collectCmd.PersistentFlags().StringVar(&password, "password", "", "set the BMC password") - collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "https", "set the protocol used to query") - collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/data/", currentUser.Username+"/"), "set the path to store collection data") - collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "set flag to force update data sent to SMD") - collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "path to CA cert. (defaults to system CAs)") + collectCmd.PersistentFlags().StringVar(&host, "host", "", "Set the URI to the SMD root endpoint") + collectCmd.PersistentFlags().StringVar(&username, "username", "", "Set the BMC user") + collectCmd.PersistentFlags().StringVar(&password, "password", "", "Set the BMC password") + collectCmd.PersistentFlags().StringVar(&scheme, "scheme", "https", "Set the scheme used to query") + collectCmd.PersistentFlags().StringVar(&protocol, "protocol", "tcp", "Set the protocol used to query") + collectCmd.PersistentFlags().StringVarP(&outputPath, "output", "o", fmt.Sprintf("/tmp/%smagellan/inventory/", currentUser.Username+"/"), "Set the path to store collection data") + collectCmd.PersistentFlags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD") + collectCmd.PersistentFlags().StringVar(&cacertPath, "cacert", "", "Path to CA cert. (defaults to system CAs)") // set flags to only be used together collectCmd.MarkFlagsRequiredTogether("username", "password") // bind flags to config properties - viper.BindPFlag("collect.driver", collectCmd.Flags().Lookup("driver")) - viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("host")) - viper.BindPFlag("collect.port", collectCmd.Flags().Lookup("port")) - viper.BindPFlag("collect.username", collectCmd.Flags().Lookup("username")) - viper.BindPFlag("collect.password", collectCmd.Flags().Lookup("password")) - viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("protocol")) - viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("output")) - viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("force-update")) - viper.BindPFlag("collect.cacert", collectCmd.Flags().Lookup("secure-tls")) - viper.BindPFlags(collectCmd.Flags()) + checkBindFlagError(viper.BindPFlag("collect.host", collectCmd.Flags().Lookup("host"))) + checkBindFlagError(viper.BindPFlag("collect.username", collectCmd.Flags().Lookup("username"))) + checkBindFlagError(viper.BindPFlag("collect.password", collectCmd.Flags().Lookup("password"))) + checkBindFlagError(viper.BindPFlag("collect.scheme", collectCmd.Flags().Lookup("scheme"))) + checkBindFlagError(viper.BindPFlag("collect.protocol", collectCmd.Flags().Lookup("protocol"))) + checkBindFlagError(viper.BindPFlag("collect.output", collectCmd.Flags().Lookup("output"))) + checkBindFlagError(viper.BindPFlag("collect.force-update", collectCmd.Flags().Lookup("force-update"))) + checkBindFlagError(viper.BindPFlag("collect.cacert", collectCmd.Flags().Lookup("cacert"))) + checkBindFlagError(viper.BindPFlags(collectCmd.Flags())) rootCmd.AddCommand(collectCmd) } diff --git a/cmd/crawl.go b/cmd/crawl.go index a4956bb..8069c9b 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -4,11 +4,11 @@ import ( "encoding/json" "fmt" "log" - "net/url" - "strings" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/OpenCHAMI/magellan/pkg/crawler" "github.com/spf13/cobra" + "github.com/spf13/viper" ) // The `crawl` command walks a collection of Redfish endpoints to collect @@ -17,25 +17,21 @@ import ( var crawlCmd = &cobra.Command{ Use: "crawl [uri]", Short: "Crawl a single BMC for inventory information", - Long: "Crawl a single BMC for inventory information\n" + - "\n" + - "Example:\n" + - " magellan crawl https://bmc.example.com", + Long: "Crawl a single BMC for inventory information. This command does NOT store information\n" + + "about the scan into cache after completion. To do so, use the 'collect' command instead\n\n" + + "Examples:\n" + + " magellan crawl https://bmc.example.com\n" + + " magellan crawl https://bmc.example.com -i -u username -p password", Args: func(cmd *cobra.Command, args []string) error { // Validate that the only argument is a valid URI + var err error if err := cobra.ExactArgs(1)(cmd, args); err != nil { return err } - parsedURI, err := url.ParseRequestURI(args[0]) + args[0], err = urlx.Sanitize(args[0]) if err != nil { - return fmt.Errorf("invalid URI specified: %s", args[0]) + return fmt.Errorf("failed to sanitize URI: %w", err) } - // Remove any trailing slashes - parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/") - // Collapse any doubled slashes - parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/") - // Update the URI in the args slice - args[0] = parsedURI.String() return nil }, Run: func(cmd *cobra.Command, args []string) { @@ -61,9 +57,13 @@ var crawlCmd = &cobra.Command{ } func init() { - crawlCmd.Flags().StringP("username", "u", "", "Username for the BMC") - crawlCmd.Flags().StringP("password", "p", "", "Password for the BMC") + crawlCmd.Flags().StringP("username", "u", "", "Set the username for the BMC") + crawlCmd.Flags().StringP("password", "p", "", "Set the password for the BMC") crawlCmd.Flags().BoolP("insecure", "i", false, "Ignore SSL errors") + checkBindFlagError(viper.BindPFlag("crawl.username", crawlCmd.Flags().Lookup("username"))) + checkBindFlagError(viper.BindPFlag("crawl.password", crawlCmd.Flags().Lookup("password"))) + checkBindFlagError(viper.BindPFlag("crawl.insecure", crawlCmd.Flags().Lookup("insecure"))) + rootCmd.AddCommand(crawlCmd) } diff --git a/cmd/list.go b/cmd/list.go index ed02dfc..e09299f 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -6,12 +6,16 @@ import ( "strings" "time" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" + "github.com/rs/zerolog/log" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) +var ( + showCache bool +) + // The `list` command provides an easy way to show what was found // and stored in a cache database from a scan. The data that's stored // is what is consumed by the `collect` command with the --cache flag. @@ -24,23 +28,34 @@ var listCmd = &cobra.Command{ " magellan list\n" + " magellan list --cache ./assets.db", Run: func(cmd *cobra.Command, args []string) { - probeResults, err := sqlite.GetProbeResults(cachePath) + // check if we just want to show cache-related info and exit + if showCache { + fmt.Printf("cache: %s\n", cachePath) + return + } + + // load the assets found from scan + scannedResults, err := sqlite.GetScannedAssets(cachePath) if err != nil { - logrus.Errorf("failed toget probe results: %v\n", err) + log.Error().Err(err).Msg("failed to get scanned assets") } format = strings.ToLower(format) if format == "json" { - b, _ := json.Marshal(probeResults) + b, err := json.Marshal(scannedResults) + if err != nil { + log.Error().Err(err).Msgf("failed to unmarshal scanned results") + } fmt.Printf("%s\n", string(b)) } else { - for _, r := range probeResults { - fmt.Printf("%s:%d (%s) @ %s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) + for _, r := range scannedResults { + fmt.Printf("%s:%d (%s) @%s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) } } }, } func init() { - listCmd.Flags().StringVar(&format, "format", "", "set the output format") + listCmd.Flags().StringVar(&format, "format", "", "Set the output format (json|default)") + listCmd.Flags().BoolVar(&showCache, "cache-info", false, "Show cache information and exit") rootCmd.AddCommand(listCmd) } diff --git a/cmd/login.go b/cmd/login.go index bd23e29..47aec06 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -7,9 +7,9 @@ import ( "os" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/log" + "github.com/OpenCHAMI/magellan/pkg/auth" "github.com/lestrrat-go/jwx/jwt" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" ) @@ -27,58 +27,61 @@ var loginCmd = &cobra.Command{ Short: "Log in with identity provider for access token", Long: "", Run: func(cmd *cobra.Command, args []string) { - // make application logger - l := log.NewLogger(logrus.New(), logrus.DebugLevel) - // check if we have a valid JWT before starting login if !forceLogin { // try getting the access token from env var - testToken, err := LoadAccessToken() + testToken, err := auth.LoadAccessToken(tokenPath) if err != nil { - l.Log.Errorf("failed to load access token: %v", err) + log.Error().Err(err).Msgf("failed to load access token") } // parse into jwt.Token to validate token, err := jwt.Parse([]byte(testToken)) if err != nil { - fmt.Printf("failed to parse access token contents: %v\n", err) + log.Error().Err(err).Msgf("failed to parse access token contents") return } // check if the token is invalid and we need a new one err = jwt.Validate(token) if err != nil { - fmt.Printf("failed to validate access token...fetching a new one") + log.Error().Err(err).Msgf("failed to validate access token...fetching a new one") } else { - fmt.Printf("found a valid token...skipping login (use the '-f/--force' flag to login anyway)") + log.Printf("found a valid token...skipping login (use the '-f/--force' flag to login anyway)") return } } + if verbose { + log.Printf("Listening for token on %s:%d", targetHost, targetPort) + } + // start the login flow var err error accessToken, err = magellan.Login(loginUrl, targetHost, targetPort) if errors.Is(err, http.ErrServerClosed) { - fmt.Printf("\n=========================================\nServer closed.\n=========================================\n\n") + if verbose { + fmt.Printf("\n=========================================\nServer closed.\n=========================================\n\n") + } } else if err != nil { - fmt.Printf("failed to start server: %v\n", err) + log.Error().Err(err).Msgf("failed to start server") } // if we got a new token successfully, save it to the token path if accessToken != "" && tokenPath != "" { err := os.WriteFile(tokenPath, []byte(accessToken), os.ModePerm) if err != nil { - fmt.Printf("failed to write access token to file: %v\n", err) + log.Error().Err(err).Msgf("failed to write access token to file") } } }, } func init() { - loginCmd.Flags().StringVar(&loginUrl, "url", "http://127.0.0.1:3333/login", "set the login URL") - loginCmd.Flags().StringVar(&targetHost, "target-host", "127.0.0.1", "set the target host to return the access code") - loginCmd.Flags().IntVar(&targetPort, "target-port", 5000, "set the target host to return the access code") - loginCmd.Flags().BoolVarP(&forceLogin, "force", "f", false, "start the login process even with a valid token") - loginCmd.Flags().StringVar(&tokenPath, "token-path", ".ochami-token", "set the path the load/save the access token") - loginCmd.Flags().BoolVar(&noBrowser, "no-browser", false, "prevent the default browser from being opened automatically") + loginCmd.Flags().StringVar(&loginUrl, "url", "http://127.0.0.1:3333/login", "Set the login URL") + loginCmd.Flags().StringVar(&targetHost, "target-host", "127.0.0.1", "Set the target host to return the access code") + loginCmd.Flags().IntVar(&targetPort, "target-port", 5000, "Set the target host to return the access code") + loginCmd.Flags().BoolVarP(&forceLogin, "force", "f", false, "Start the login process even with a valid token") + loginCmd.Flags().StringVar(&tokenPath, "token-path", ".ochami-token", "Set the path to load/save the access token") + loginCmd.Flags().BoolVar(&noBrowser, "no-browser", false, "Prevent the default browser from being opened automatically") rootCmd.AddCommand(loginCmd) } diff --git a/cmd/root.go b/cmd/root.go index 40f7407..b21e22d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -21,7 +21,7 @@ import ( "os/user" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/api/smd" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -42,17 +42,22 @@ var ( outputPath string configPath string verbose bool + debug bool + forceUpdate bool ) // The `root` command doesn't do anything on it's own except display // a help message and then exits. var rootCmd = &cobra.Command{ Use: "magellan", - Short: "Tool for BMC discovery", + Short: "Redfish-based BMC discovery tool", Long: "", Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { - cmd.Help() + err := cmd.Help() + if err != nil { + log.Error().Err(err).Msg("failed to print help") + } os.Exit(0) } }, @@ -66,51 +71,30 @@ func Execute() { } } -// LoadAccessToken() tries to load a JWT string from an environment -// variable, file, or config in that order. If loading the token -// fails with one options, it will fallback to the next option until -// all options are exhausted. -// -// Returns a token as a string with no error if successful. -// Alternatively, returns an empty string with an error if a token is -// not able to be loaded. -func LoadAccessToken() (string, error) { - // try to load token from env var - testToken := os.Getenv("ACCESS_TOKEN") - if testToken != "" { - return testToken, nil - } - - // try reading access token from a file - b, err := os.ReadFile(tokenPath) - if err == nil { - return string(b), nil - } - - // TODO: try to load token from config - testToken = viper.GetString("access_token") - if testToken != "" { - return testToken, nil - } - return "", fmt.Errorf("failed toload token from environment variable, file, or config") -} - func init() { currentUser, _ = user.Current() cobra.OnInitialize(InitializeConfig) rootCmd.PersistentFlags().IntVar(&concurrency, "concurrency", -1, "set the number of concurrent processes") - rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 30, "set the timeout") + rootCmd.PersistentFlags().IntVar(&timeout, "timeout", 5, "set the timeout") rootCmd.PersistentFlags().StringVarP(&configPath, "config", "c", "", "set the config file path") - rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set output verbosity") + rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "set to enable/disable verbose output") + rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "set to enable/disable debug messages") rootCmd.PersistentFlags().StringVar(&accessToken, "access-token", "", "set the access token") - rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%smagellan/magellan.db", currentUser.Username+"/"), "set the scanning result cache path") + rootCmd.PersistentFlags().StringVar(&cachePath, "cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username), "set the scanning result cache path") // bind viper config flags with cobra - viper.BindPFlag("concurrency", rootCmd.Flags().Lookup("concurrency")) - viper.BindPFlag("timeout", rootCmd.Flags().Lookup("timeout")) - viper.BindPFlag("verbose", rootCmd.Flags().Lookup("verbose")) - viper.BindPFlag("cache", rootCmd.Flags().Lookup("cache")) - viper.BindPFlags(rootCmd.Flags()) + checkBindFlagError(viper.BindPFlag("concurrency", rootCmd.PersistentFlags().Lookup("concurrency"))) + checkBindFlagError(viper.BindPFlag("timeout", rootCmd.PersistentFlags().Lookup("timeout"))) + checkBindFlagError(viper.BindPFlag("verbose", rootCmd.PersistentFlags().Lookup("verbose"))) + checkBindFlagError(viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug"))) + checkBindFlagError(viper.BindPFlag("access-token", rootCmd.PersistentFlags().Lookup("verbose"))) + checkBindFlagError(viper.BindPFlag("cache", rootCmd.PersistentFlags().Lookup("cache"))) +} + +func checkBindFlagError(err error) { + if err != nil { + log.Error().Err(err).Msg("failed to bind flag") + } } // InitializeConfig() initializes a new config object by loading it @@ -119,7 +103,10 @@ func init() { // See the 'LoadConfig' function in 'internal/config' for details. func InitializeConfig() { if configPath != "" { - magellan.LoadConfig(configPath) + err := magellan.LoadConfig(configPath) + if err != nil { + log.Error().Err(err).Msg("failed to load config") + } } } @@ -129,35 +116,33 @@ func InitializeConfig() { // TODO: This function should probably be moved to 'internal/config.go' // instead of in this file. func SetDefaults() { + currentUser, _ = user.Current() viper.SetDefault("threads", 1) - viper.SetDefault("timeout", 30) + viper.SetDefault("timeout", 5) viper.SetDefault("config", "") viper.SetDefault("verbose", false) - viper.SetDefault("cache", "/tmp/magellan/magellan.db") + viper.SetDefault("debug", false) + viper.SetDefault("cache", fmt.Sprintf("/tmp/%s/magellan/assets.db", currentUser.Username)) viper.SetDefault("scan.hosts", []string{}) viper.SetDefault("scan.ports", []int{}) viper.SetDefault("scan.subnets", []string{}) viper.SetDefault("scan.subnet-masks", []net.IP{}) viper.SetDefault("scan.disable-probing", false) - viper.SetDefault("collect.driver", []string{"redfish"}) - viper.SetDefault("collect.host", smd.Host) - viper.SetDefault("collect.port", smd.Port) - viper.SetDefault("collect.user", "") - viper.SetDefault("collect.pass", "") - viper.SetDefault("collect.protocol", "https") + viper.SetDefault("scan.disable-cache", false) + viper.SetDefault("collect.host", host) + viper.SetDefault("collect.username", "") + viper.SetDefault("collect.password", "") + viper.SetDefault("collect.protocol", "tcp") viper.SetDefault("collect.output", "/tmp/magellan/data/") viper.SetDefault("collect.force-update", false) - viper.SetDefault("collect.ca-cert", "") - viper.SetDefault("bmc-host", "") - viper.SetDefault("bmc-port", 443) - viper.SetDefault("user", "") - viper.SetDefault("pass", "") - viper.SetDefault("transfer-protocol", "HTTP") - viper.SetDefault("protocol", "https") - viper.SetDefault("firmware-url", "") - viper.SetDefault("firmware-version", "") - viper.SetDefault("component", "") - viper.SetDefault("secure-tls", false) - viper.SetDefault("status", false) + viper.SetDefault("collect.cacert", "") + viper.SetDefault("update.username", "") + viper.SetDefault("update.password", "") + viper.SetDefault("update.transfer-protocol", "https") + viper.SetDefault("update.protocol", "tcp") + viper.SetDefault("update.firmware.url", "") + viper.SetDefault("update.firmware.version", "") + viper.SetDefault("update.component", "") + viper.SetDefault("update.status", false) } diff --git a/cmd/scan.go b/cmd/scan.go index 9fafdfe..fedc691 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -6,23 +6,24 @@ import ( "net" "os" "path" - "strings" - "time" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/db/sqlite" + "github.com/OpenCHAMI/magellan/internal/cache/sqlite" + "github.com/rs/zerolog/log" + urlx "github.com/OpenCHAMI/magellan/internal/url" "github.com/cznic/mathutil" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( - begin uint8 - end uint8 + scheme string subnets []string - subnetMasks []net.IP + subnetMask net.IPMask + targetHosts [][]string disableProbing bool + disableCache bool ) // The `scan` command is usually the first step to using the CLI tool. @@ -32,91 +33,165 @@ var ( // See the `ScanForAssets()` function in 'internal/scan.go' for details // related to the implementation. var scanCmd = &cobra.Command{ - Use: "scan", - Short: "Scan for BMC nodes on a network", - Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response. " + - "If the '--disable-probe` flag is used, the tool will not send another request to probe for available " + - "Redfish services.\n\n" + - "Example:\n" + - " magellan scan --subnet 172.16.0.0/24 --add-host 10.0.0.101\n" + - " magellan scan --subnet 172.16.0.0 --subnet-mask 255.255.255.0 --cache ./assets.db", + Use: "scan urls...", + Short: "Scan to discover BMC nodes on a network", + Long: "Perform a net scan by attempting to connect to each host and port specified and getting a response.\n" + + "Each host is passed *with a full URL* including the protocol and port. Additional subnets can be added\n" + + "by using the '--subnet' flag and providing an IP address on the subnet as well as a CIDR. If no CIDR is\n" + + "provided, then the subnet mask specified with the '--subnet-mask' flag will be used instead (will use\n" + + "default mask if not set).\n\n" + + "Similarly, any host provided with no port will use either the ports specified\n" + + "with `--port` or the default port used with each specified protocol. The default protocol is 'tcp' unless\n" + + "specified. The `--scheme` flag works similarly and the default value is 'https' in the host URL or with the\n" + + "'--protocol' flag.\n\n" + + "If the '--disable-probe` flag is used, the tool will not send another request to probe for available.\n" + + "Redfish services. This is not recommended, since the extra request makes the scan a bit more reliable\n" + + "for determining which hosts to collect inventory data.\n\n" + + "Examples:\n" + + // assumes host https://10.0.0.101:443 + " magellan scan 10.0.0.101\n" + + // assumes subnet using HTTPS and port 443 except for specified host + " magellan scan http://10.0.0.101:80 https://user:password@10.0.0.102:443 http://172.16.0.105:8080 --subnet 172.16.0.0/24\n" + + // assumes hosts http://10.0.0.101:8080 and http://10.0.0.102:8080 + " magellan scan 10.0.0.101 10.0.0.102 https://172.16.0.10:443 --port 8080 --protocol tcp\n" + + // assumes subnet using default unspecified subnet-masks + " magellan scan --subnet 10.0.0.0\n" + + // assumes subnet using HTTPS and port 443 with specified CIDR + " magellan scan --subnet 10.0.0.0/16\n" + + // assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16 + " magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0\n" + + // assumes subnet without CIDR has a subnet-mask of 255.255.0.0 + " magellan scan --subnet 10.0.0.0/24 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db\n", Run: func(cmd *cobra.Command, args []string) { - var ( - hostsToScan []string - portsToScan []int - ) - - // start by adding `--host` supplied to scan - if len(hosts) > 0 { - hostsToScan = hosts + // add default ports for hosts if none are specified with flag + if len(ports) == 0 { + if debug { + log.Debug().Msg("adding default ports") + } + ports = magellan.GetDefaultPorts() } - // add hosts from `--subnets` and `--subnet-mask` - for i, subnet := range subnets { + // format and combine flag and positional args + targetHosts = append(targetHosts, urlx.FormatHosts(args, ports, scheme, verbose)...) + targetHosts = append(targetHosts, urlx.FormatHosts(hosts, ports, scheme, verbose)...) + + // add more hosts specified with `--subnet` flag + if debug { + log.Debug().Msg("adding hosts from subnets") + } + for _, subnet := range subnets { // subnet string is empty so nothing to do here if subnet == "" { continue } - // NOTE: should we check if subnet is valid here or is it done elsewhere (maybe in GenerateHosts)? + // generate a slice of all hosts to scan from subnets + subnetHosts := magellan.GenerateHostsWithSubnet(subnet, &subnetMask, ports, scheme) + targetHosts = append(targetHosts, subnetHosts...) + } - // no subnet masks supplied so add a default one for class C private networks - if len(subnetMasks) < i+1 { - subnetMasks = append(subnetMasks, net.IP{255, 255, 255, 0}) + // if there are no target hosts, then there's nothing to do + if len(targetHosts) <= 0 { + log.Warn().Msg("nothing to do (no valid target hosts)") + return + } else { + if len(targetHosts[0]) <= 0 { + log.Warn().Msg("nothing to do (no valid target hosts)") + return } + } - // generate a slice of all hosts to scan from subnets - hostsToScan = append(hostsToScan, magellan.GenerateHosts(subnet, &subnetMasks[i])...) + // show the parameters going into the scan + if debug { + combinedTargetHosts := []string{} + for _, targetHost := range targetHosts { + combinedTargetHosts = append(combinedTargetHosts, targetHost...) + } + c := map[string]any{ + "hosts": combinedTargetHosts, + "cache": cachePath, + "concurrency": concurrency, + "protocol": protocol, + "subnets": subnets, + "subnet-mask": subnetMask.String(), + "cert": cacertPath, + "disable-probing": disableProbing, + "disable-caching": disableCache, + } + b, _ := json.MarshalIndent(c, "", " ") + fmt.Printf("%s", string(b)) } - // add ports to use for scanning - if len(ports) > 0 { - portsToScan = ports + // set the number of concurrent requests (1 request per BMC node) + // + // NOTE: The number of concurrent job is equal to the number of hosts by default. + // The max concurrent jobs cannot be greater than the number of hosts. + if concurrency <= 0 { + concurrency = len(targetHosts) } else { - // no ports supplied so only use defaults - portsToScan = magellan.GetDefaultPorts() + concurrency = mathutil.Clamp(len(targetHosts), 1, len(targetHosts)) } // scan and store scanned data in cache - if concurrency <= 0 { - concurrency = mathutil.Clamp(len(hostsToScan), 1, 255) + foundAssets := magellan.ScanForAssets(&magellan.ScanParams{ + TargetHosts: targetHosts, + Scheme: scheme, + Protocol: protocol, + Concurrency: concurrency, + Timeout: timeout, + DisableProbing: disableProbing, + Verbose: verbose, + Debug: debug, + }) + + if len(foundAssets) > 0 && debug { + log.Info().Any("assets", foundAssets).Msgf("found assets from scan") } - probeStates := magellan.ScanForAssets(hostsToScan, portsToScan, concurrency, timeout, disableProbing, verbose) - if verbose { - format = strings.ToLower(format) - if format == "json" { - b, _ := json.Marshal(probeStates) - fmt.Printf("%s\n", string(b)) - } else { - for _, r := range probeStates { - fmt.Printf("%s:%d (%s) @ %s\n", r.Host, r.Port, r.Protocol, r.Timestamp.Format(time.UnixDate)) - } + + if !disableCache && cachePath != "" { + // make the cache directory path if needed + err := os.MkdirAll(path.Dir(cachePath), 0755) + if err != nil { + log.Printf("failed to make cache directory: %v", err) } - } - // make the dbpath dir if needed - err := os.MkdirAll(path.Dir(cachePath), 0766) - if err != nil { - fmt.Printf("failed tomake database directory: %v", err) + // TODO: change this to use an extensible plugin system for storage solutions + // (i.e. something like cache.InsertScannedAssets(path, assets) which implements a Cache interface) + if len(foundAssets) > 0 { + err = sqlite.InsertScannedAssets(cachePath, foundAssets...) + if err != nil { + log.Error().Err(err).Msg("failed to write scanned assets to cache") + } + if verbose { + log.Info().Msgf("saved assets to cache: %s", cachePath) + } + } else { + log.Warn().Msg("no assets found to save") + } } - sqlite.InsertProbeResults(cachePath, &probeStates) }, } func init() { - scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") - scanCmd.Flags().IntSliceVar(&ports, "port", []int{}, "set the ports to scan") - scanCmd.Flags().StringVar(&format, "format", "", "set the output format") - scanCmd.Flags().StringSliceVar(&subnets, "subnet", []string{}, "set additional subnets") - scanCmd.Flags().IPSliceVar(&subnetMasks, "subnet-mask", []net.IP{}, "set the subnet masks to use for network (must match number of subnets)") - scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "disable probing scanned results for BMC nodes") - - viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host")) - viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port")) - viper.BindPFlag("scan.subnets", scanCmd.Flags().Lookup("subnet")) - viper.BindPFlag("scan.subnet-masks", scanCmd.Flags().Lookup("subnet-mask")) - viper.BindPFlag("scan.disable-probing", scanCmd.Flags().Lookup("disable-probing")) + // scanCmd.Flags().StringSliceVar(&hosts, "host", []string{}, "set additional hosts to scan") + scanCmd.Flags().StringSliceVar(&hosts, "host", nil, "Add individual hosts to scan. (example: https://my.bmc.com:5000; same as using positional args)") + scanCmd.Flags().IntSliceVar(&ports, "port", nil, "Adds additional ports to scan for each host with unspecified ports.") + scanCmd.Flags().StringVar(&scheme, "scheme", "https", "Set the default scheme to use if not specified in host URI. (default is 'https')") + scanCmd.Flags().StringVar(&protocol, "protocol", "tcp", "Set the default protocol to use in scan. (default is 'tcp')") + scanCmd.Flags().StringSliceVar(&subnets, "subnet", nil, "Add additional hosts from specified subnets to scan.") + scanCmd.Flags().IPMaskVar(&subnetMask, "subnet-mask", net.IPv4Mask(255, 255, 255, 0), "Set the default subnet mask to use for with all subnets not using CIDR notation.") + scanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "Disable probing found assets for Redfish service(s) running on BMC nodes") + scanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "Disable saving found assets to a cache database specified with 'cache' flag") + + checkBindFlagError(viper.BindPFlag("scan.hosts", scanCmd.Flags().Lookup("host"))) + checkBindFlagError(viper.BindPFlag("scan.ports", scanCmd.Flags().Lookup("port"))) + checkBindFlagError(viper.BindPFlag("scan.scheme", scanCmd.Flags().Lookup("scheme"))) + checkBindFlagError(viper.BindPFlag("scan.protocol", scanCmd.Flags().Lookup("protocol"))) + checkBindFlagError(viper.BindPFlag("scan.subnets", scanCmd.Flags().Lookup("subnet"))) + checkBindFlagError(viper.BindPFlag("scan.subnet-masks", scanCmd.Flags().Lookup("subnet-mask"))) + checkBindFlagError(viper.BindPFlag("scan.disable-probing", scanCmd.Flags().Lookup("disable-probing"))) + checkBindFlagError(viper.BindPFlag("scan.disable-cache", scanCmd.Flags().Lookup("disable-cache"))) rootCmd.AddCommand(scanCmd) } diff --git a/cmd/update.go b/cmd/update.go index 87f2282..d5f9a50 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -1,101 +1,98 @@ package cmd import ( + "os" + "strings" + magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/log" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" ) var ( host string - port int firmwareUrl string firmwareVersion string component string transferProtocol string - status bool + showStatus bool ) // The `update` command provides an interface to easily update firmware // using Redfish. It also provides a simple way to check the status of // an update in-progress. var updateCmd = &cobra.Command{ - Use: "update", + Use: "update hosts...", Short: "Update BMC node firmware", - Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n" + + Long: "Perform an firmware update using Redfish by providing a remote firmware URL and component.\n\n" + "Examples:\n" + - " magellan update --host 172.16.0.108 --port 443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + - " magellan update --status --host 172.16.0.108 --port 443 --username bmc_username --password bmc_password", + " magellan update 172.16.0.108:443 --username bmc_username --password bmc_password --firmware-url http://172.16.0.200:8005/firmware/bios/image.RBU --component BIOS\n" + + " magellan update 172.16.0.108:443 --status --username bmc_username --password bmc_password", Run: func(cmd *cobra.Command, args []string) { - l := log.NewLogger(logrus.New(), logrus.DebugLevel) - q := &magellan.UpdateParams{ - FirmwarePath: firmwareUrl, - FirmwareVersion: firmwareVersion, - Component: component, - TransferProtocol: transferProtocol, - QueryParams: magellan.QueryParams{ - Drivers: []string{"redfish"}, - Preferred: "redfish", - Protocol: protocol, - Host: host, - Username: username, - Password: password, - Timeout: timeout, - Port: port, - }, - } - - // check if required params are set - if host == "" || username == "" || password == "" { - l.Log.Fatal("requires host, user, and pass to be set") + // check that we have at least one host + if len(args) <= 0 { + log.Error().Msg("update requires at least one host") + os.Exit(1) } // get status if flag is set and exit - if status { - err := magellan.GetUpdateStatus(q) - if err != nil { - l.Log.Errorf("failed toget update status: %v", err) + for _, arg := range args { + if showStatus { + err := magellan.GetUpdateStatus(&magellan.UpdateParams{ + FirmwarePath: firmwareUrl, + FirmwareVersion: firmwareVersion, + Component: component, + TransferProtocol: transferProtocol, + CollectParams: magellan.CollectParams{ + URI: arg, + Username: username, + Password: password, + Timeout: timeout, + }, + }) + if err != nil { + log.Error().Err(err).Msgf("failed to get update status") + } + return } - return - } - // client, err := magellan.NewClient(l, &q.QueryParams) - // if err != nil { - // l.Log.Errorf("failed tomake client: %v", err) - // } - // err = magellan.UpdateFirmware(client, l, q) - err := magellan.UpdateFirmwareRemote(q) - if err != nil { - l.Log.Errorf("failed toupdate firmware: %v", err) + // initiate a remote update + err := magellan.UpdateFirmwareRemote(&magellan.UpdateParams{ + FirmwarePath: firmwareUrl, + FirmwareVersion: firmwareVersion, + Component: component, + TransferProtocol: strings.ToUpper(transferProtocol), + CollectParams: magellan.CollectParams{ + URI: host, + Username: username, + Password: password, + Timeout: timeout, + }, + }) + if err != nil { + log.Error().Err(err).Msgf("failed to update firmware") + } } }, } func init() { - updateCmd.Flags().StringVar(&host, "bmc-host", "", "set the BMC host") - updateCmd.Flags().IntVar(&port, "bmc-port", 443, "set the BMC port") - updateCmd.Flags().StringVar(&username, "user", "", "set the BMC user") - updateCmd.Flags().StringVar(&password, "pass", "", "set the BMC password") - updateCmd.Flags().StringVar(&transferProtocol, "transfer-protocol", "HTTP", "set the transfer protocol") - updateCmd.Flags().StringVar(&protocol, "protocol", "https", "set the Redfish protocol") - updateCmd.Flags().StringVar(&firmwareUrl, "firmware-url", "", "set the path to the firmware") - updateCmd.Flags().StringVar(&firmwareVersion, "firmware-version", "", "set the version of firmware to be installed") - updateCmd.Flags().StringVar(&component, "component", "", "set the component to upgrade") - updateCmd.Flags().BoolVar(&status, "status", false, "get the status of the update") + updateCmd.Flags().StringVar(&username, "username", "", "Set the BMC user") + updateCmd.Flags().StringVar(&password, "password", "", "Set the BMC password") + updateCmd.Flags().StringVar(&transferProtocol, "scheme", "https", "Set the transfer protocol") + updateCmd.Flags().StringVar(&firmwareUrl, "firmware-url", "", "Set the path to the firmware") + updateCmd.Flags().StringVar(&firmwareVersion, "firmware-version", "", "Set the version of firmware to be installed") + updateCmd.Flags().StringVar(&component, "component", "", "Set the component to upgrade (BMC|BIOS)") + updateCmd.Flags().BoolVar(&showStatus, "status", false, "Get the status of the update") - viper.BindPFlag("host", updateCmd.Flags().Lookup("host")) - viper.BindPFlag("port", updateCmd.Flags().Lookup("port")) - viper.BindPFlag("username", updateCmd.Flags().Lookup("user")) - viper.BindPFlag("password", updateCmd.Flags().Lookup("pass")) - viper.BindPFlag("transfer-protocol", updateCmd.Flags().Lookup("transfer-protocol")) - viper.BindPFlag("protocol", updateCmd.Flags().Lookup("protocol")) - viper.BindPFlag("firmware.url", updateCmd.Flags().Lookup("firmware.url")) - viper.BindPFlag("firmware.version", updateCmd.Flags().Lookup("firmware.version")) - viper.BindPFlag("component", updateCmd.Flags().Lookup("component")) - viper.BindPFlag("secure-tls", updateCmd.Flags().Lookup("secure-tls")) - viper.BindPFlag("status", updateCmd.Flags().Lookup("status")) + checkBindFlagError(viper.BindPFlag("update.username", updateCmd.Flags().Lookup("username"))) + checkBindFlagError(viper.BindPFlag("update.password", updateCmd.Flags().Lookup("password"))) + checkBindFlagError(viper.BindPFlag("update.scheme", updateCmd.Flags().Lookup("scheme"))) + checkBindFlagError(viper.BindPFlag("update.firmware-url", updateCmd.Flags().Lookup("firmware-url"))) + checkBindFlagError(viper.BindPFlag("update.firmware-version", updateCmd.Flags().Lookup("firmware-version"))) + checkBindFlagError(viper.BindPFlag("update.component", updateCmd.Flags().Lookup("component"))) + checkBindFlagError(viper.BindPFlag("update.status", updateCmd.Flags().Lookup("status"))) rootCmd.AddCommand(updateCmd) } diff --git a/config.yaml b/config.yaml index 30f244c..7d817ed 100644 --- a/config.yaml +++ b/config.yaml @@ -5,15 +5,18 @@ scan: - "172.16.0.0" - "172.16.0.0/24" subnet-masks: + - "255.255.255.0" ports: - - 433 + - 443 disable-probing: false + disable-caching: false + protocol: "tcp" + scheme: "https" collect: - # host: smd-host - # port: smd-port username: "admin" password: "password" - protocol: "https" + protocol: "tcp" + scheme: "https" output: "/tmp/magellan/data/" threads: 1 force-update: false @@ -23,8 +26,7 @@ update: port: 443 username: "admin" password: "password" - transfer-protocol: "HTTP" - protocol: "https" + transfer-protocol: "https" firmware: url: version: diff --git a/go.mod b/go.mod index 14c6c29..e00981d 100644 --- a/go.mod +++ b/go.mod @@ -10,17 +10,19 @@ require ( github.com/lestrrat-go/jwx v1.2.29 github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c - github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stmcginnis/gofish v0.19.0 - golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 ) +require github.com/rs/zerolog v1.33.0 + require ( github.com/google/go-cmp v0.6.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + ) require ( @@ -39,7 +41,6 @@ require ( github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rs/zerolog v1.33.0 github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect @@ -49,8 +50,8 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 8180af8..befbed6 100644 --- a/go.sum +++ b/go.sum @@ -87,8 +87,6 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= @@ -109,7 +107,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -125,10 +122,11 @@ go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTV golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= -golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -144,7 +142,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -153,8 +150,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/internal/api/dora/dora.go b/internal/api/dora/dora.go deleted file mode 100644 index b113cf6..0000000 --- a/internal/api/dora/dora.go +++ /dev/null @@ -1,75 +0,0 @@ -package dora - -import ( - "encoding/json" - "fmt" - - "github.com/OpenCHAMI/magellan/internal/util" - - "github.com/jmoiron/sqlx" -) - -const ( - Host = "http://localhost" - DbType = "sqlite3" - DbPath = "../data/assets.db" - BaseEndpoint = "/v1" - Port = 8000 -) - -type ScannedResult struct { - id string - site any - cidr string - ip string - port int - protocol string - scanner string - state string - updated string -} - -func makeEndpointUrl(endpoint string) string { - return Host + ":" + fmt.Sprint(Port) + BaseEndpoint + endpoint -} - -// Scan for BMC assets uing dora scanner -func ScanForAssets() error { - - return nil -} - -// Query dora API to get scanned ports -func QueryScannedPorts() error { - // Perform scan and collect from dora server - url := makeEndpointUrl("/scanned_ports") - _, body, err := util.MakeRequest(nil, url, "GET", nil, nil) - if err != nil { - return fmt.Errorf("failed todiscover assets: %v", err) - } - - // get data from JSON - var res map[string]any - if err := json.Unmarshal(body, &res); err != nil { - return fmt.Errorf("failed tounmarshal response body: %v", err) - } - data := res["data"] - - fmt.Println(data) - - return nil -} - -// Loads scanned ports directly from DB -func LoadScannedPortsFromDB(dbPath string, dbType string) { - db, _ := sqlx.Open(dbType, dbPath) - sql := `SELECT * FROM scanned_port WHERE state='open'` - rows, _ := db.Query(sql) - for rows.Next() { - var r ScannedResult - rows.Scan( - &r.id, &r.site, &r.cidr, &r.ip, &r.port, &r.protocol, &r.scanner, - &r.state, &r.updated, - ) - } -} diff --git a/internal/api/smd/smd.go b/internal/api/smd/smd.go deleted file mode 100644 index 370841d..0000000 --- a/internal/api/smd/smd.go +++ /dev/null @@ -1,137 +0,0 @@ -package smd - -// See ref for API docs: -// https://github.com/OpenCHAMI/hms-smd/blob/master/docs/examples.adoc -// https://github.com/OpenCHAMI/hms-smd -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "net" - "net/http" - "os" - "time" - - "github.com/OpenCHAMI/magellan/internal/util" -) - -var ( - Host = "http://localhost" - BaseEndpoint = "/hsm/v2" - Port = 27779 -) - -type Option func(*Client) - -type Client struct { - *http.Client - CACertPool *x509.CertPool -} - -func NewClient(opts ...Option) *Client { - client := &Client{ - Client: http.DefaultClient, - } - for _, opt := range opts { - opt(client) - } - return client -} - -func WithHttpClient(httpClient *http.Client) Option { - return func(c *Client) { - c.Client = httpClient - } -} - -// This MakeRequest function is a wrapper around the util.MakeRequest function -// with a couple of niceties with using a smd.Client -func (c *Client) MakeRequest(url string, method string, body []byte, headers map[string]string) (*http.Response, []byte, error) { - return util.MakeRequest(c.Client, url, method, body, headers) -} - -func WithCertPool(certPool *x509.CertPool) Option { - return func(c *Client) { - c.Client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: certPool, - InsecureSkipVerify: true, - }, - DisableKeepAlives: true, - Dial: (&net.Dialer{ - Timeout: 120 * time.Second, - KeepAlive: 120 * time.Second, - }).Dial, - TLSHandshakeTimeout: 120 * time.Second, - ResponseHeaderTimeout: 120 * time.Second, - } - } -} - -func WithSecureTLS(certPath string) Option { - cacert, _ := os.ReadFile(certPath) - certPool := x509.NewCertPool() - certPool.AppendCertsFromPEM(cacert) - return WithCertPool(certPool) -} - -func (c *Client) GetRedfishEndpoints(headers map[string]string, opts ...Option) error { - url := makeEndpointUrl("/Inventory/RedfishEndpoints") - _, body, err := c.MakeRequest(url, "GET", nil, headers) - if err != nil { - return fmt.Errorf("failed toget endpoint: %v", err) - } - // fmt.Println(res) - fmt.Println(string(body)) - return nil -} - -func (c *Client) GetComponentEndpoint(xname string) error { - url := makeEndpointUrl("/Inventory/ComponentsEndpoints/" + xname) - res, body, err := c.MakeRequest(url, "GET", nil, nil) - if err != nil { - return fmt.Errorf("failed toget endpoint: %v", err) - } - fmt.Println(res) - fmt.Println(string(body)) - return nil -} - -func (c *Client) AddRedfishEndpoint(data []byte, headers map[string]string) error { - if data == nil { - return fmt.Errorf("failed toadd redfish endpoint: no data found") - } - - // Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint - url := makeEndpointUrl("/Inventory/RedfishEndpoints") - res, body, err := c.MakeRequest(url, "POST", data, headers) - if res != nil { - statusOk := res.StatusCode >= 200 && res.StatusCode < 300 - if !statusOk { - return fmt.Errorf("returned status code %d when adding endpoint", res.StatusCode) - } - fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) - } - return err -} - -func (c *Client) UpdateRedfishEndpoint(xname string, data []byte, headers map[string]string) error { - if data == nil { - return fmt.Errorf("failed to add redfish endpoint: no data found") - } - // Update redfish endpoint via PUT `/hsm/v2/Inventory/RedfishEndpoints` endpoint - url := makeEndpointUrl("/Inventory/RedfishEndpoints/" + xname) - res, body, err := c.MakeRequest(url, "PUT", data, headers) - fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) - if res != nil { - statusOk := res.StatusCode >= 200 && res.StatusCode < 300 - if !statusOk { - return fmt.Errorf("failed to update redfish endpoint (returned %s)", res.Status) - } - } - return err -} - -func makeEndpointUrl(endpoint string) string { - return Host + ":" + fmt.Sprint(Port) + BaseEndpoint + endpoint -} diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..7b4eea1 --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,13 @@ +package cache + +import ( + "database/sql/driver" +) + +// TODO: implement extendable storage drivers using cache interface (sqlite, duckdb, etc.) +type Cache[T any] interface { + CreateIfNotExists(path string) (driver.Connector, error) + Insert(path string, data ...T) error + Delete(path string, data ...T) error + Get(path string) ([]T, error) +} diff --git a/internal/db/postgresql/postgresql.go b/internal/cache/postgresql/postgresql.go similarity index 100% rename from internal/db/postgresql/postgresql.go rename to internal/cache/postgresql/postgresql.go diff --git a/internal/cache/sqlite/sqlite.go b/internal/cache/sqlite/sqlite.go new file mode 100644 index 0000000..7a04978 --- /dev/null +++ b/internal/cache/sqlite/sqlite.go @@ -0,0 +1,107 @@ +package sqlite + +import ( + "fmt" + + magellan "github.com/OpenCHAMI/magellan/internal" + "github.com/OpenCHAMI/magellan/internal/util" + + "github.com/jmoiron/sqlx" +) + +const TABLE_NAME = "magellan_scanned_assets" + +func CreateScannedAssetIfNotExists(path string) (*sqlx.DB, error) { + schema := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + host TEXT NOT NULL, + port INTEGER NOT NULL, + protocol TEXT, + state INTEGER, + timestamp TIMESTAMP, + PRIMARY KEY (host, port) + ); + `, TABLE_NAME) + // TODO: it may help with debugging to check for file permissions here first + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return nil, fmt.Errorf("failed to open database: %v", err) + } + db.MustExec(schema) + return db, nil +} + +func InsertScannedAssets(path string, assets ...magellan.RemoteAsset) error { + if assets == nil { + return fmt.Errorf("states == nil") + } + + // create database if it doesn't already exist + db, err := CreateScannedAssetIfNotExists(path) + if err != nil { + return err + } + + // insert all probe states into db + tx := db.MustBegin() + for _, state := range assets { + sql := fmt.Sprintf(`INSERT OR REPLACE INTO %s (host, port, protocol, state, timestamp) + VALUES (:host, :port, :protocol, :state, :timestamp);`, TABLE_NAME) + _, err := tx.NamedExec(sql, &state) + if err != nil { + fmt.Printf("failed to execute transaction: %v\n", err) + } + } + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %v", err) + } + return nil +} + +func DeleteScannedAssets(path string, results ...magellan.RemoteAsset) error { + if results == nil { + return fmt.Errorf("no assets found") + } + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return fmt.Errorf("failed to open database: %v", err) + } + tx := db.MustBegin() + for _, state := range results { + sql := fmt.Sprintf(`DELETE FROM %s WHERE host = :host, port = :port;`, TABLE_NAME) + _, err := tx.NamedExec(sql, &state) + if err != nil { + fmt.Printf("failed to execute transaction: %v\n", err) + } + } + + err = tx.Commit() + if err != nil { + return fmt.Errorf("failed to commit transaction: %v", err) + } + return nil +} + +func GetScannedAssets(path string) ([]magellan.RemoteAsset, error) { + // check if path exists first to prevent creating the database + exists, err := util.PathExists(path) + if !exists { + return nil, fmt.Errorf("no file found") + } else if err != nil { + return nil, err + } + + // now check if the file is the SQLite database + db, err := sqlx.Open("sqlite3", path) + if err != nil { + return nil, fmt.Errorf("failed to open database: %v", err) + } + + results := []magellan.RemoteAsset{} + err = db.Select(&results, fmt.Sprintf("SELECT * FROM %s ORDER BY host ASC, port ASC;", TABLE_NAME)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve assets: %v", err) + } + return results, nil +} diff --git a/internal/collect.go b/internal/collect.go index 45665a1..524c5c6 100644 --- a/internal/collect.go +++ b/internal/collect.go @@ -2,54 +2,38 @@ package magellan import ( - "crypto/tls" "encoding/json" "fmt" - "net/http" "os" "path" "sync" "time" - "github.com/OpenCHAMI/magellan/internal/log" + "github.com/OpenCHAMI/magellan/pkg/client" + "github.com/OpenCHAMI/magellan/pkg/crawler" - "github.com/OpenCHAMI/magellan/internal/api/smd" "github.com/OpenCHAMI/magellan/internal/util" + "github.com/rs/zerolog/log" "github.com/Cray-HPE/hms-xname/xnames" _ "github.com/mattn/go-sqlite3" - "github.com/stmcginnis/gofish" _ "github.com/stmcginnis/gofish" - "github.com/stmcginnis/gofish/redfish" "golang.org/x/exp/slices" ) -const ( - IPMI_PORT = 623 - SSH_PORT = 22 - HTTPS_PORT = 443 -) - -// QueryParams is a collections of common parameters passed to the CLI. -// Each CLI subcommand has a corresponding implementation function that -// takes an object as an argument. However, the implementation may not -// use all of the properties within the object. -type QueryParams struct { - Host string // set by the 'host' flag - Port int // set by the 'port' flag - Protocol string // set by the 'protocol' flag - Username string // set the BMC username with the 'username' flag - Password string // set the BMC password with the 'password' flag - Drivers []string // DEPRECATED: TO BE REMOVED!!! - Concurrency int // set the of concurrent jobs with the 'concurrency' flag - Preferred string // DEPRECATED: TO BE REMOVED!!! - Timeout int // set the timeout with the 'timeout' flag - CaCertPath string // set the cert path with the 'cacert' flag - Verbose bool // set whether to include verbose output with 'verbose' flag - IpmitoolPath string // DEPRECATED: TO BE REMOVE!!! - OutputPath string // set the path to save output with 'output' flag - ForceUpdate bool // set whether to force updating SMD with 'force-update' flag - AccessToken string // set the access token to include in request with 'access-token' flag +// CollectParams is a collection of common parameters passed to the CLI +// for the 'collect' subcommand. +type CollectParams struct { + URI string // set by the 'host' flag + Username string // set the BMC username with the 'username' flag + Password string // set the BMC password with the 'password' flag + Concurrency int // set the of concurrent jobs with the 'concurrency' flag + Timeout int // set the timeout with the 'timeout' flag + CaCertPath string // set the cert path with the 'cacert' flag + Verbose bool // set whether to include verbose output with 'verbose' flag + OutputPath string // set the path to save output with 'output' flag + ForceUpdate bool // set whether to force updating SMD with 'force-update' flag + AccessToken string // set the access token to include in request with 'access-token' flag } // This is the main function used to collect information from the BMC nodes via Redfish. @@ -57,46 +41,41 @@ type QueryParams struct { // // Requests can be made to several of the nodes using a goroutine by setting the q.Concurrency // property value between 1 and 255. -func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) error { +func CollectInventory(assets *[]RemoteAsset, params *CollectParams) error { // check for available probe states - if probeStates == nil { - return fmt.Errorf("no probe states found") + if assets == nil { + return fmt.Errorf("no assets found") } - if len(*probeStates) <= 0 { - return fmt.Errorf("no probe states found") - } - - // make the output directory to store files - outputPath := path.Clean(q.OutputPath) - outputPath, err := util.MakeOutputDirectory(outputPath) - if err != nil { - l.Log.Errorf("failed to make output directory: %v", err) + if len(*assets) <= 0 { + return fmt.Errorf("no assets found") } // collect bmc information asynchronously var ( - offset = 0 - wg sync.WaitGroup - found = make([]string, 0, len(*probeStates)) - done = make(chan struct{}, q.Concurrency+1) - chanProbeState = make(chan ScannedResult, q.Concurrency+1) - client = smd.NewClient( - smd.WithSecureTLS(q.CaCertPath), + offset = 0 + wg sync.WaitGroup + found = make([]string, 0, len(*assets)) + done = make(chan struct{}, params.Concurrency+1) + chanAssets = make(chan RemoteAsset, params.Concurrency+1) + outputPath = path.Clean(params.OutputPath) + smdClient = client.NewClient( + client.WithSecureTLS[client.SmdClient](params.CaCertPath), ) ) - wg.Add(q.Concurrency) - for i := 0; i < q.Concurrency; i++ { + // set the client's host from the CLI param + smdClient.URI = params.URI + wg.Add(params.Concurrency) + for i := 0; i < params.Concurrency; i++ { go func() { for { - ps, ok := <-chanProbeState + sr, ok := <-chanAssets if !ok { wg.Done() return } - q.Host = ps.Host - q.Port = ps.Port // generate custom xnames for bmcs + // TODO: add xname customization via CLI node := xnames.Node{ Cabinet: 1000, Chassis: 1, @@ -105,107 +84,102 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err } offset += 1 - gofishClient, err := connectGofish(q) + // crawl BMC node to fetch inventory data via Redfish + systems, err := crawler.CrawlBMC(crawler.CrawlerConfig{ + URI: fmt.Sprintf("%s:%d", sr.Host, sr.Port), + Username: params.Username, + Password: params.Password, + Insecure: true, + }) if err != nil { - l.Log.Errorf("failed to connect to BMC (%v:%v): %v", q.Host, q.Port, err) + log.Error().Err(err).Msgf("failed to crawl BMC") } - defer gofishClient.Logout() // data to be sent to smd data := map[string]any{ - "ID": fmt.Sprintf("%v", node.String()[:len(node.String())-2]), - "Type": "", - "Name": "", - "FQDN": ps.Host, - "User": q.Username, - // "Password": q.Pass, + "ID": fmt.Sprintf("%v", node.String()[:len(node.String())-2]), + "Type": "", + "Name": "", + "FQDN": sr.Host, + "User": params.Username, "MACRequired": true, "RediscoverOnUpdate": false, + "Systems": systems, } - // chassis - if gofishClient != nil { - chassis, err := CollectChassis(gofishClient, q) - if err != nil { - l.Log.Errorf("failed to collect chassis: %v", err) - continue - } - data["Chassis"] = chassis - - // systems - systems, err := CollectSystems(gofishClient, q) - if err != nil { - l.Log.Errorf("failed to collect systems: %v", err) - } - data["Systems"] = systems - - // add other fields from systems - if len(systems) > 0 { - system := systems[0]["Data"].(*redfish.ComputerSystem) - if system == nil { - l.Log.Errorf("invalid system data (data is nil)") - } else { - data["Name"] = system.Name - } - } - } else { - l.Log.Errorf("invalid client (client is nil)") - continue - } - - headers := make(map[string]string) - headers["Content-Type"] = "application/json" - - // use access token in authorization header if we have it - if q.AccessToken != "" { - headers["Authorization"] = "Bearer " + q.AccessToken - } + // create and set headers for request + headers := client.HTTPHeader{} + headers.Authorization(params.AccessToken) + headers.ContentType("application/json") body, err := json.MarshalIndent(data, "", " ") if err != nil { - l.Log.Errorf("failed to marshal output to JSON: %v", err) + log.Error().Err(err).Msgf("failed to marshal output to JSON") } - if q.Verbose { + if params.Verbose { fmt.Printf("%v\n", string(body)) } - // write JSON data to file if output path is set + // write JSON data to file if output path is set using hive partitioning strategy if outputPath != "" { - err = os.WriteFile(path.Clean(outputPath+"/"+q.Host+".json"), body, os.ModePerm) - if err != nil { - l.Log.Errorf("failed to write data to file: %v", err) + // make directory if it does exists + exists, err := util.PathExists(outputPath) + if err == nil && !exists { + err = os.MkdirAll(outputPath, 0o644) + if err != nil { + log.Error().Err(err).Msg("failed to make directory for output") + } else { + // make the output directory to store files + outputPath, err := util.MakeOutputDirectory(outputPath, false) + if err != nil { + log.Error().Err(err).Msg("failed to make output directory") + } else { + // write the output to the final path + err = os.WriteFile(path.Clean(fmt.Sprintf("%s/%s/%d.json", params.URI, outputPath, time.Now().Unix())), body, os.ModePerm) + if err != nil { + log.Error().Err(err).Msgf("failed to write data to file") + } + } + } } } - // add all endpoints to smd - err = client.AddRedfishEndpoint(body, headers) - if err != nil { - l.Log.Error(err) - - // try updating instead - if q.ForceUpdate { - err = client.UpdateRedfishEndpoint(data["ID"].(string), body, headers) - if err != nil { - l.Log.Error(err) + // add all endpoints to SMD ONLY if a host is provided + if smdClient.URI != "" { + err = smdClient.Add(body, headers) + if err != nil { + log.Error().Err(err).Msgf("failed to add Redfish endpoint") + + // try updating instead + if params.ForceUpdate { + smdClient.Xname = data["ID"].(string) + err = smdClient.Update(body, headers) + if err != nil { + log.Error().Err(err).Msgf("failed to forcibly update Redfish endpoint") + } } } + } else { + if params.Verbose { + log.Warn().Msg("no request made (host argument is empty)") + } } // got host information, so add to list of already probed hosts - found = append(found, ps.Host) + found = append(found, sr.Host) } }() } // use the found results to query bmc information - for _, ps := range *probeStates { + for _, ps := range *assets { // skip if found info from host foundHost := slices.Index(found, ps.Host) if !ps.State || foundHost >= 0 { continue } - chanProbeState <- ps + chanAssets <- ps } // handle goroutine paths @@ -219,416 +193,9 @@ func CollectAll(probeStates *[]ScannedResult, l *log.Logger, q *QueryParams) err } }() - close(chanProbeState) + close(chanAssets) wg.Wait() close(done) return nil } - -// CollectEthernetInterfaces() collects all of the ethernet interfaces found -// from all systems from under the "/redfish/v1/Systems" endpoint. -// -// TODO: This function needs to be refactored entirely...if not deleted -// in favor of using crawler.CrawlBM() instead. -func CollectEthernetInterfaces(c *gofish.APIClient, q *QueryParams, systemID string) ([]byte, error) { - // TODO: add more endpoints to test for ethernet interfaces - // /redfish/v1/Chassis/{ChassisID}/NetworkAdapters/{NetworkAdapterId}/NetworkDeviceFunctions/{NetworkDeviceFunctionId}/EthernetInterfaces/{EthernetInterfaceId} - // /redfish/v1/Managers/{ManagerId}/EthernetInterfaces/{EthernetInterfaceId} - // /redfish/v1/Systems/{ComputerSystemId}/EthernetInterfaces/{EthernetInterfaceId} - // /redfish/v1/Systems/{ComputerSystemId}/OperatingSystem/Containers/EthernetInterfaces/{EthernetInterfaceId} - systems, err := c.Service.Systems() - if err != nil { - return nil, fmt.Errorf("failed to get systems: (%v:%v): %v", q.Host, q.Port, err) - } - - var ( - interfaces []*redfish.EthernetInterface - errList []error - ) - - // get all of the ethernet interfaces in our systems - for _, system := range systems { - eth, err := system.EthernetInterfaces() - if err != nil { - errList = append(errList, err) - } - - interfaces = append(interfaces, eth...) - } - - // print any report errors - err = util.FormatErrorList(errList) - if util.HasErrors(errList) { - return nil, fmt.Errorf("failed to get ethernet interfaces with %d error(s): \n%v", len(errList), err) - } - - data := map[string]any{"EthernetInterfaces": interfaces} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - -// CollectChassis() fetches all chassis related information from each node specified -// via the Redfish API. Like the other collect functions, this function uses the gofish -// library to make requests to each node. Additionally, all of the network adapters found -// are added to the output as well. -// -// Returns a map that represents a Chassis object with NetworkAdapters. -func CollectChassis(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) { - rfChassis, err := c.Service.Chassis() - if err != nil { - return nil, fmt.Errorf("failed to query chassis (%v:%v): %v", q.Host, q.Port, err) - } - - var chassis []map[string]any - for _, ch := range rfChassis { - networkAdapters, err := ch.NetworkAdapters() - if err != nil { - return nil, fmt.Errorf("failed to get network adapters: %v", err) - } - - chassis = append(chassis, map[string]any{ - "Data": ch, - "NetworkAdapters": networkAdapters, - }) - } - - return chassis, nil -} - -// TODO: DELETE ME!!! -func CollectStorage(c *gofish.APIClient, q *QueryParams) ([]byte, error) { - systems, err := c.Service.StorageSystems() - if err != nil { - return nil, fmt.Errorf("failed to query storage systems (%v:%v): %v", q.Host, q.Port, err) - } - - services, err := c.Service.StorageServices() - if err != nil { - return nil, fmt.Errorf("failed to query storage services (%v:%v): %v", q.Host, q.Port, err) - } - - data := map[string]any{ - "Storage": map[string]any{ - "Systems": systems, - "Services": services, - }, - } - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - -// CollectSystems pulls system information from each BMC node via Redfish using the -// `gofish` library. -// -// The process of collecting this info is as follows: -// 1. check if system has ethernet interfaces -// 1.a. if yes, create system data and ethernet interfaces JSON -// 1.b. if no, try to get data using manager instead -// 2. check if manager has "ManagerForServices" and "EthernetInterfaces" properties -// 2.a. if yes, query both properties to use in next step -// 2.b. for each service, query its data and add the ethernet interfaces -// 2.c. add the system to list of systems to marshal and return -func CollectSystems(c *gofish.APIClient, q *QueryParams) ([]map[string]any, error) { - rfSystems, err := c.Service.Systems() - if err != nil { - return nil, fmt.Errorf("failed to get systems (%v:%v): %v", q.Host, q.Port, err) - } - - var systems []map[string]any - - for _, system := range rfSystems { - eths, err := system.EthernetInterfaces() - if err != nil { - return nil, fmt.Errorf("failed to get system ethernet interfaces: %v", err) - } - - // try and get ethernet interfaces through manager if empty - if len(eths) <= 0 { - if q.Verbose { - fmt.Printf("no system ethernet interfaces found...trying to get from managers interface\n") - } - managedBy, err := system.ManagedBy() - if err != nil { - return nil, fmt.Errorf("failed to get system managers for '%s': %w", system.Name, err) - } - for _, manager := range managedBy { - // try getting ethernet interface from all managers until one is found - eths, err = manager.EthernetInterfaces() - if err != nil { - return nil, fmt.Errorf("failed to get system manager ethernet interfaces: %v", err) - } - if len(eths) > 0 { - break - } - } - } - - // add network interfaces to system - rfNetworkInterfaces, err := system.NetworkInterfaces() - if err != nil { - return nil, fmt.Errorf("failed to get system network interfaces: %v", err) - } - - // get the network adapter ID for each network interface - var networkInterfaces []map[string]any - for _, rfNetworkInterface := range rfNetworkInterfaces { - networkAdapter, err := rfNetworkInterface.NetworkAdapter() - if err != nil { - return nil, fmt.Errorf("failed to get network adapter: %v", err) - } - - networkInterfaces = append(networkInterfaces, map[string]any{ - "Data": rfNetworkInterface, - "NetworkAdapterId": networkAdapter.ID, - }) - } - - // add system to collection of systems - systems = append(systems, map[string]any{ - "Data": system, - "EthernetInterfaces": eths, - "NetworkInterfaces": networkInterfaces, - }) - } - - // do manual requests if systems is empty to only get necessary info as last resort - // /redfish/v1/Systems - - // /redfish/v1/Systems/Members - // /redfish/v1/Systems/ - // fmt.Printf("system count: %d\n", len(systems)) - // if len(systems) == 0 { - // url := baseRedfishUrl(q) + "/Systems" - // if q.Verbose { - // fmt.Printf("%s\n", url) - // } - // res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) - // if err != nil { - // return nil, fmt.Errorf("failed to make request: %v", err) - // } else if res.StatusCode != http.StatusOK { - // return nil, fmt.Errorf("request returned status code %d", res.StatusCode) - // } - - // // sweet syntatic sugar type aliases - // type System = map[string]any - // type Member = map[string]string - - // // get all the systems - // var ( - // tempSystems System - // interfaces []*redfish.EthernetInterface - // errList []error - // ) - // err = json.Unmarshal(body, &tempSystems) - // if err != nil { - // return nil, fmt.Errorf("failed to unmarshal systems: %v", err) - // } - - // // then, get all the members within a system - // members, ok := tempSystems["Members"] - // if ok { - // for _, member := range members.([]Member) { - // id, ok := member["@odata.id"] - // if ok { - // // /redfish/v1/Systems/Self (or whatever) - // // memberEndpoint := fmt.Sprintf("%s%s", url, id) - // // res, body, err := util.MakeRequest(nil, baseRedfishUrl(q)+memberEndpoint, http.MethodGet, nil, nil) - // // if err != nil { - // // continue - // // } else if res.StatusCode != http.StatusOK { - // // continue - // // } - // // TODO: extract EthernetInterfaces from Systems then query - - // // get all of the ethernet interfaces in our systems - // ethernetInterface, err := redfish.ListReferencedEthernetInterfaces(c, id+"/EthernetInterfaces/") - // if err != nil { - // errList = append(errList, err) - // continue - // } - // interfaces = append(interfaces, ethernetInterface...) - // } else { - // return nil, fmt.Errorf("no ID found for member") - // } - // if util.HasErrors(errList) { - // return nil, util.FormatErrorList(errList) - // } - // } - // i, err := json.Marshal(interfaces) - // if err != nil { - // return nil, fmt.Errorf("failed to unmarshal interface: %v", err) - // } - // temp = append(temp, map[string]any{ - // "Data": nil, - // "EthernetInterfaces": string(i), - // }) - // } else { - // return nil, fmt.Errorf("no members found in systems") - // } - - // } else { - // b, err := json.Marshal(systems) - // if err != nil { - // fmt.Printf("failed to marshal systems: %v", err) - // } - // fmt.Printf("systems: %v\n", string(b)) - - // // query the system's ethernet interfaces - // // var temp []map[string]any - // var errList []error - // for _, system := range systems { - // interfaces, err := CollectEthernetInterfaces(c, q, system.ID) - // if err != nil { - // errList = append(errList, fmt.Errorf("failed to collect ethernet interface: %v", err)) - // continue - // } - // var i map[string]any - // err = json.Unmarshal(interfaces, &i) - // if err != nil { - // return nil, fmt.Errorf("failed to unmarshal interface: %v", err) - // } - // temp = append(temp, map[string]any{ - // "Data": system, - // "EthernetInterfaces": i["EthernetInterfaces"], - // }) - // } - // if util.HasErrors(errList) { - // err = util.FormatErrorList(errList) - // if err != nil { - // return nil, fmt.Errorf("multiple errors occurred: %v", err) - // } - // } - // } - - return systems, nil -} - -// TODO: DELETE ME!!! -func CollectRegisteries(c *gofish.APIClient, q *QueryParams) ([]byte, error) { - registries, err := c.Service.Registries() - if err != nil { - return nil, fmt.Errorf("failed to query storage systems (%v:%v): %v", q.Host, q.Port, err) - } - - data := map[string]any{"Registries": registries} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - -// TODO: MAYBE DELETE??? -func CollectProcessors(q *QueryParams) ([]byte, error) { - url := baseRedfishUrl(q) + "/Systems" - res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) - if err != nil { - return nil, fmt.Errorf("something went wrong: %v", err) - } else if res == nil { - return nil, fmt.Errorf("no response returned (url: %s)", url) - } else if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("returned status code %d", res.StatusCode) - } - - // convert to not get base64 string - var procs map[string]json.RawMessage - var members []map[string]any - err = json.Unmarshal(body, &procs) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal processors: %v", err) - } - err = json.Unmarshal(procs["Members"], &members) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal processor members: %v", err) - } - - // request data about each processor member on node - for _, member := range members { - var oid = member["@odata.id"].(string) - var infoUrl = url + oid - res, _, err := util.MakeRequest(nil, infoUrl, "GET", nil, nil) - if err != nil { - return nil, fmt.Errorf("something went wrong: %v", err) - } else if res == nil { - return nil, fmt.Errorf("no response returned (url: %s)", url) - } else if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("returned status code %d", res.StatusCode) - } - } - - data := map[string]any{"Processors": procs} - b, err := json.MarshalIndent(data, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - - return b, nil -} - -func connectGofish(q *QueryParams) (*gofish.APIClient, error) { - config, err := makeGofishConfig(q) - if err != nil { - return nil, fmt.Errorf("failed to make gofish config: %v", err) - } - c, err := gofish.Connect(config) - if err != nil { - return nil, fmt.Errorf("failed to connect to redfish endpoint: %v", err) - } - if c != nil { - c.Service.ProtocolFeaturesSupported = gofish.ProtocolFeaturesSupported{ - ExpandQuery: gofish.Expand{ - ExpandAll: true, - Links: true, - }, - } - } - return c, err -} - -func makeGofishConfig(q *QueryParams) (gofish.ClientConfig, error) { - var ( - client = &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - url = baseRedfishUrl(q) - ) - return gofish.ClientConfig{ - Endpoint: url, - Username: q.Username, - Password: q.Password, - Insecure: true, - TLSHandshakeTimeout: q.Timeout, - HTTPClient: client, - // MaxConcurrentRequests: int64(q.Threads), // NOTE: this was added in latest version of gofish - }, nil -} - -func makeJson(object any) ([]byte, error) { - b, err := json.MarshalIndent(object, "", " ") - if err != nil { - return nil, fmt.Errorf("failed to marshal JSON: %v", err) - } - return []byte(b), nil -} - -func baseRedfishUrl(q *QueryParams) string { - url := fmt.Sprintf("%s://", q.Protocol) - if q.Username != "" && q.Password != "" { - url += fmt.Sprintf("%s:%s@", q.Username, q.Password) - } - return fmt.Sprintf("%s%s:%d", url, q.Host, q.Port) -} diff --git a/internal/db/sqlite/sqlite.go b/internal/db/sqlite/sqlite.go deleted file mode 100644 index 7eb75a8..0000000 --- a/internal/db/sqlite/sqlite.go +++ /dev/null @@ -1,95 +0,0 @@ -package sqlite - -import ( - "fmt" - - magellan "github.com/OpenCHAMI/magellan/internal" - - "github.com/jmoiron/sqlx" -) - -func CreateProbeResultsIfNotExists(path string) (*sqlx.DB, error) { - schema := ` - CREATE TABLE IF NOT EXISTS magellan_scanned_ports ( - host TEXT NOT NULL, - port INTEGER NOT NULL, - protocol TEXT, - state INTEGER, - timestamp TIMESTAMP, - PRIMARY KEY (host, port) - ); - ` - // TODO: it may help with debugging to check for file permissions here first - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return nil, fmt.Errorf("failed toopen database: %v", err) - } - db.MustExec(schema) - return db, nil -} - -func InsertProbeResults(path string, states *[]magellan.ScannedResult) error { - if states == nil { - return fmt.Errorf("states == nil") - } - - // create database if it doesn't already exist - db, err := CreateProbeResultsIfNotExists(path) - if err != nil { - return err - } - - // insert all probe states into db - tx := db.MustBegin() - for _, state := range *states { - sql := `INSERT OR REPLACE INTO magellan_scanned_ports (host, port, protocol, state, timestamp) - VALUES (:host, :port, :protocol, :state, :timestamp);` - _, err := tx.NamedExec(sql, &state) - if err != nil { - fmt.Printf("failed toexecute transaction: %v\n", err) - } - } - err = tx.Commit() - if err != nil { - return fmt.Errorf("failed tocommit transaction: %v", err) - } - return nil -} - -func DeleteProbeResults(path string, results *[]magellan.ScannedResult) error { - if results == nil { - return fmt.Errorf("no probe results found") - } - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return fmt.Errorf("failed toopen database: %v", err) - } - tx := db.MustBegin() - for _, state := range *results { - sql := `DELETE FROM magellan_scanned_ports WHERE host = :host, port = :port;` - _, err := tx.NamedExec(sql, &state) - if err != nil { - fmt.Printf("failed toexecute transaction: %v\n", err) - } - } - - err = tx.Commit() - if err != nil { - return fmt.Errorf("failed tocommit transaction: %v", err) - } - return nil -} - -func GetProbeResults(path string) ([]magellan.ScannedResult, error) { - db, err := sqlx.Open("sqlite3", path) - if err != nil { - return nil, fmt.Errorf("failed toopen database: %v", err) - } - - results := []magellan.ScannedResult{} - err = db.Select(&results, "SELECT * FROM magellan_scanned_ports ORDER BY host ASC, port ASC;") - if err != nil { - return nil, fmt.Errorf("failed toretrieve probes: %v", err) - } - return results, nil -} diff --git a/internal/log/logger.go b/internal/log/logger.go deleted file mode 100644 index 43e2db2..0000000 --- a/internal/log/logger.go +++ /dev/null @@ -1,22 +0,0 @@ -package log - -import ( - "github.com/sirupsen/logrus" -) - -type Logger struct { - Log *logrus.Logger - Path string -} - -func NewLogger(l *logrus.Logger, level logrus.Level) *Logger { - l.SetLevel(level) - return &Logger{ - Log: logrus.New(), - Path: "", - } -} - -func (l *Logger) WriteFile(path string) { - -} diff --git a/internal/scan.go b/internal/scan.go index 660aede..a88116d 100644 --- a/internal/scan.go +++ b/internal/scan.go @@ -5,13 +5,17 @@ import ( "math" "net" "net/http" + "net/url" + "strconv" "sync" "time" - "github.com/OpenCHAMI/magellan/internal/util" + urlx "github.com/OpenCHAMI/magellan/internal/url" + "github.com/OpenCHAMI/magellan/pkg/client" + "github.com/rs/zerolog/log" ) -type ScannedResult struct { +type RemoteAsset struct { Host string `json:"host"` Port int `json:"port"` Protocol string `json:"protocol"` @@ -19,9 +23,21 @@ type ScannedResult struct { Timestamp time.Time `json:"timestamp"` } +// ScanParams is a collection of commom parameters passed to the CLI +type ScanParams struct { + TargetHosts [][]string + Scheme string + Protocol string + Concurrency int + Timeout int + DisableProbing bool + Verbose bool + Debug bool +} + // ScanForAssets() performs a net scan on a network to find available services -// running. The function expects a list of hosts and ports to make requests. -// Note that each all ports will be used per host. +// running. The function expects a list of targets (as [][]string) to make requests. +// The 2D list is to permit one goroutine per BMC node when making each request. // // This function runs in a goroutine with the "concurrency" flag setting the // number of concurrent requests. Only one request is made to each BMC node @@ -34,54 +50,67 @@ type ScannedResult struct { // remove the service from being stored in the list of scanned results. // // Returns a list of scanned results to be stored in cache (but isn't doing here). -func ScanForAssets(hosts []string, ports []int, concurrency int, timeout int, disableProbing bool, verbose bool) []ScannedResult { +func ScanForAssets(params *ScanParams) []RemoteAsset { var ( - results = make([]ScannedResult, 0, len(hosts)) - done = make(chan struct{}, concurrency+1) - chanHost = make(chan string, concurrency+1) + results = make([]RemoteAsset, 0, len(params.TargetHosts)) + done = make(chan struct{}, params.Concurrency+1) + chanHosts = make(chan []string, params.Concurrency+1) ) + if params.Verbose { + log.Info().Any("args", params).Msg("starting scan...") + } + var wg sync.WaitGroup - wg.Add(concurrency) - for i := 0; i < concurrency; i++ { + wg.Add(params.Concurrency) + for i := 0; i < params.Concurrency; i++ { go func() { for { - host, ok := <-chanHost + hosts, ok := <-chanHosts if !ok { wg.Done() return } - scannedResults := rawConnect(host, ports, timeout, true) - if !disableProbing { - probeResults := []ScannedResult{} - for _, result := range scannedResults { - url := fmt.Sprintf("https://%s:%d/redfish/v1/", result.Host, result.Port) - res, _, err := util.MakeRequest(nil, url, "GET", nil, nil) - if err != nil || res == nil { - if verbose { - fmt.Printf("failed to make request: %v\n", err) - } - continue - } else if res.StatusCode != http.StatusOK { - if verbose { - fmt.Printf("request returned code: %v\n", res.StatusCode) + for _, host := range hosts { + foundAssets, err := rawConnect(host, params.Protocol, params.Timeout, true) + // if we failed to connect, exit from the function + if err != nil { + if params.Verbose { + log.Debug().Err(err).Msgf("failed to connect to host") + } + wg.Done() + return + } + if !params.DisableProbing { + assetsToAdd := []RemoteAsset{} + for _, foundAsset := range foundAssets { + url := fmt.Sprintf("%s:%d/redfish/v1/", foundAsset.Host, foundAsset.Port) + res, _, err := client.MakeRequest(nil, url, http.MethodGet, nil, nil) + if err != nil || res == nil { + if params.Verbose { + log.Printf("failed to make request: %v\n", err) + } + continue + } else if res.StatusCode != http.StatusOK { + if params.Verbose { + log.Printf("request returned code: %v\n", res.StatusCode) + } + continue + } else { + assetsToAdd = append(assetsToAdd, foundAsset) } - continue - } else { - probeResults = append(probeResults, result) } + results = append(results, assetsToAdd...) + } else { + results = append(results, foundAssets...) } - results = append(results, probeResults...) - } else { - results = append(results, scannedResults...) } - } }() } - for _, host := range hosts { - chanHost <- host + for _, hosts := range params.TargetHosts { + chanHosts <- hosts } go func() { select { @@ -92,97 +121,129 @@ func ScanForAssets(hosts []string, ports []int, concurrency int, timeout int, di time.Sleep(1000) } }() - close(chanHost) + close(chanHosts) wg.Wait() close(done) + + if params.Verbose { + log.Info().Msg("scan complete") + } return results } -// GenerateHosts() builds a list of hosts to scan using the "subnet" +// GenerateHostsWithSubnet() builds a list of hosts to scan using the "subnet" // and "subnetMask" arguments passed. The function is capable of -// distinguishing between IP formats: a subnet with just an IP address (172.16.0.0) and -// a subnet with IP address and CIDR (172.16.0.0/24). +// distinguishing between IP formats: a subnet with just an IP address (172.16.0.0) +// and a subnet with IP address and CIDR (172.16.0.0/24). // // NOTE: If a IP address is provided with CIDR, then the "subnetMask" // parameter will be ignored. If neither is provided, then the default // subnet mask will be used instead. -func GenerateHosts(subnet string, subnetMask *net.IP) []string { +func GenerateHostsWithSubnet(subnet string, subnetMask *net.IPMask, additionalPorts []int, defaultScheme string) [][]string { if subnet == "" || subnetMask == nil { return nil } - // convert subnets from string to net.IP + // convert subnets from string to net.IP to test if CIDR is included subnetIp := net.ParseIP(subnet) if subnetIp == nil { - // try parse CIDR instead + // not a valid IP so try again with CIDR ip, network, err := net.ParseCIDR(subnet) if err != nil { return nil } subnetIp = ip - if network != nil { - t := net.IP(network.Mask) - subnetMask = &t + if network == nil { + // use the default subnet mask if a valid one is not provided + network = &net.IPNet{ + IP: subnetIp, + Mask: net.IPv4Mask(255, 255, 255, 0), + } } + subnetMask = &network.Mask } - mask := net.IPMask(subnetMask.To4()) - - // if no subnet mask, use a default 24-bit mask (for now) - return generateHosts(&subnetIp, &mask) + // generate new IPs from subnet and format to full URL + subnetIps := generateIPsWithSubnet(&subnetIp, subnetMask) + return urlx.FormatIPs(subnetIps, additionalPorts, defaultScheme, false) } +// GetDefaultPorts() returns a list of default ports. The only reason to have +// this function is to add/remove ports without affecting usage. func GetDefaultPorts() []int { - return []int{HTTPS_PORT} + return []int{443} } -func rawConnect(host string, ports []int, timeout int, keepOpenOnly bool) []ScannedResult { - results := []ScannedResult{} - for _, p := range ports { - result := ScannedResult{ - Host: host, - Port: p, - Protocol: "tcp", +// rawConnect() tries to connect to the host using DialTimeout() and waits +// until a response is receive or if the timeout (in seconds) expires. This +// function expects a full URL such as https://my.bmc.host:443/ to make the +// connection. +func rawConnect(address string, protocol string, timeoutSeconds int, keepOpenOnly bool) ([]RemoteAsset, error) { + uri, err := url.ParseRequestURI(address) + if err != nil { + return nil, fmt.Errorf("failed to split host/port: %w", err) + } + + // convert port to its "proper" type + port, err := strconv.Atoi(uri.Port()) + if err != nil { + return nil, fmt.Errorf("failed to convert port to integer type: %w", err) + } + + var ( + timeoutDuration = time.Second * time.Duration(timeoutSeconds) + assets []RemoteAsset + asset = RemoteAsset{ + Host: fmt.Sprintf("%s://%s", uri.Scheme, uri.Hostname()), + Port: port, + Protocol: protocol, State: false, Timestamp: time.Now(), } - t := time.Second * time.Duration(timeout) - port := fmt.Sprint(p) - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), t) - if err != nil { - result.State = false - // fmt.Println("Connecting error:", err) - } - if conn != nil { - result.State = true - defer conn.Close() - // fmt.Println("Opened", net.JoinHostPort(host, port)) - } - if keepOpenOnly { - if result.State { - results = append(results, result) - } - } else { - results = append(results, result) + ) + + // try to conntect to host (expects host in format [10.0.0.0]:443) + target := fmt.Sprintf("%s:%s", uri.Hostname(), uri.Port()) + conn, err := net.DialTimeout(protocol, target, timeoutDuration) + if err != nil { + asset.State = false + return nil, fmt.Errorf("failed to dial host: %w", err) + } + if conn != nil { + asset.State = true + defer conn.Close() + } + if keepOpenOnly { + if asset.State { + assets = append(assets, asset) } + } else { + assets = append(assets, asset) } - return results + return assets, nil } -func generateHosts(ip *net.IP, mask *net.IPMask) []string { +// generateIPsWithSubnet() returns a collection of host IP strings with a +// provided subnet mask. +// +// TODO: add a way for filtering/exclude specific IPs and IP ranges. +func generateIPsWithSubnet(ip *net.IP, mask *net.IPMask) []string { + // check if subnet IP and mask are valid + if ip == nil || mask == nil { + log.Error().Msg("invalid subnet IP or mask (ip == nil or mask == nil)") + return nil + } // get all IP addresses in network - ones, _ := mask.Size() + ones, bits := mask.Size() hosts := []string{} - end := int(math.Pow(2, float64((32-ones)))) - 1 + end := int(math.Pow(2, float64((bits-ones)))) - 1 for i := 0; i < end; i++ { - // ip[3] = byte(i) - ip = util.GetNextIP(ip, 1) + ip = client.GetNextIP(ip, 1) if ip == nil { continue } - // host := fmt.Sprintf("%v.%v.%v.%v", (*ip)[0], (*ip)[1], (*ip)[2], (*ip)[3]) - // fmt.Printf("host: %v\n", ip.String()) + hosts = append(hosts, ip.String()) } return hosts diff --git a/internal/update.go b/internal/update.go index 252eba0..9191818 100644 --- a/internal/update.go +++ b/internal/update.go @@ -4,28 +4,32 @@ import ( "encoding/json" "fmt" "net/http" + "net/url" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/client" ) type UpdateParams struct { - QueryParams + CollectParams FirmwarePath string FirmwareVersion string Component string TransferProtocol string } -// UpdateFirmware() uses 'bmc-toolbox/bmclib' to update the firmware of a BMC node. +// UpdateFirmwareRemote() uses 'gofish' to update the firmware of a BMC node. // The function expects the firmware URL, firmware version, and component flags to be // set from the CLI to perform a firmware update. -// -// NOTE: Multipart HTTP updating may not work since older verions of OpenBMC, which bmclib -// uses underneath, did not support support multipart updates. This was changed with the -// inclusion of support for MultipartHttpPushUri in OpenBMC (https://gerrit.openbmc.org/c/openbmc/bmcweb/+/32174). -// Also, related to bmclib: https://github.com/bmc-toolbox/bmclib/issues/341 func UpdateFirmwareRemote(q *UpdateParams) error { - url := baseRedfishUrl(&q.QueryParams) + "/redfish/v1/UpdateService/Actions/SimpleUpdate" + // parse URI to set up full address + uri, err := url.ParseRequestURI(q.URI) + if err != nil { + return fmt.Errorf("failed to parse URI: %w", err) + } + uri.User = url.UserPassword(q.Username, q.Password) + + // set up other vars + updateUrl := fmt.Sprintf("%s/redfish/v1/UpdateService/Actions/SimpleUpdate", uri.String()) headers := map[string]string{ "Content-Type": "application/json", "cache-control": "no-cache", @@ -37,13 +41,13 @@ func UpdateFirmwareRemote(q *UpdateParams) error { } data, err := json.Marshal(b) if err != nil { - return fmt.Errorf("failed tomarshal data: %v", err) + return fmt.Errorf("failed to marshal data: %v", err) } - res, body, err := util.MakeRequest(nil, url, "POST", data, headers) + res, body, err := client.MakeRequest(nil, updateUrl, "POST", data, headers) if err != nil { return fmt.Errorf("something went wrong: %v", err) } else if res == nil { - return fmt.Errorf("no response returned (url: %s)", url) + return fmt.Errorf("no response returned (url: %s)", updateUrl) } if len(body) > 0 { fmt.Printf("%d: %v\n", res.StatusCode, string(body)) @@ -52,12 +56,18 @@ func UpdateFirmwareRemote(q *UpdateParams) error { } func GetUpdateStatus(q *UpdateParams) error { - url := baseRedfishUrl(&q.QueryParams) + "/redfish/v1/UpdateService" - res, body, err := util.MakeRequest(nil, url, "GET", nil, nil) + // parse URI to set up full address + uri, err := url.ParseRequestURI(q.URI) + if err != nil { + return fmt.Errorf("failed to parse URI: %w", err) + } + uri.User = url.UserPassword(q.Username, q.Password) + updateUrl := fmt.Sprintf("%s/redfish/v1/UpdateService", uri.String()) + res, body, err := client.MakeRequest(nil, updateUrl, "GET", nil, nil) if err != nil { return fmt.Errorf("something went wrong: %v", err) } else if res == nil { - return fmt.Errorf("no response returned (url: %s)", url) + return fmt.Errorf("no response returned (url: %s)", updateUrl) } else if res.StatusCode != http.StatusOK { return fmt.Errorf("returned status code %d", res.StatusCode) } diff --git a/internal/url/url.go b/internal/url/url.go new file mode 100644 index 0000000..38f0eed --- /dev/null +++ b/internal/url/url.go @@ -0,0 +1,116 @@ +package url + +import ( + "fmt" + "net/url" + "strings" + + "github.com/rs/zerolog/log" +) + +func Sanitize(uri string) (string, error) { + // URL sanitanization for host argument + parsedURI, err := url.ParseRequestURI(uri) + if err != nil { + return "", fmt.Errorf("failed to parse URI: %w", err) + } + // Remove any trailing slashes + parsedURI.Path = strings.TrimSuffix(parsedURI.Path, "/") + // Collapse any doubled slashes + parsedURI.Path = strings.ReplaceAll(parsedURI.Path, "//", "/") + return parsedURI.String(), nil +} + +// FormatHosts() takes a list of hosts and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatHosts(hosts []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, host := range hosts { + uri, err := url.ParseRequestURI(host) + if err != nil { + if verbose { + log.Warn().Msgf("invalid URI parsed: %s", host) + } + continue + } + + // check if scheme is set, if not set it with flag or default value ('https' if flag is not set) + if uri.Scheme == "" { + if scheme != "" { + uri.Scheme = scheme + } else { + // hardcoded assumption + uri.Scheme = "https" + } + } + + // tidy up slashes and update arg with new value + uri.Path = strings.TrimSuffix(uri.Path, "/") + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} + +// FormatIPs() takes a list of IP addresses and ports and builds full URLs in the +// form of scheme://host:port. If no scheme is provided, it will use "https" by +// default. +// +// Returns a 2D string slice where each slice contains URL host strings for each +// port. The intention is to have all of the URLs for a single host combined into +// a single slice to initiate one goroutine per host, but making request to multiple +// ports. +func FormatIPs(ips []string, ports []int, scheme string, verbose bool) [][]string { + // format each positional arg as a complete URL + var formattedHosts [][]string + for _, ip := range ips { + if scheme == "" { + scheme = "https" + } + // make an entirely new object since we're expecting just IPs + uri := &url.URL{ + Scheme: scheme, + Host: ip, + } + + // tidy up slashes and update arg with new value + uri.Path = strings.ReplaceAll(uri.Path, "//", "/") + uri.Path = strings.TrimSuffix(uri.Path, "/") + + // for hosts with unspecified ports, add ports to scan from flag + if uri.Port() == "" { + if len(ports) == 0 { + ports = append(ports, 443) + } + var tmp []string + for _, port := range ports { + uri.Host += fmt.Sprintf(":%d", port) + tmp = append(tmp, uri.String()) + } + formattedHosts = append(formattedHosts, tmp) + } else { + formattedHosts = append(formattedHosts, []string{uri.String()}) + } + + } + return formattedHosts +} diff --git a/internal/util/error.go b/internal/util/error.go new file mode 100644 index 0000000..0141ea7 --- /dev/null +++ b/internal/util/error.go @@ -0,0 +1,25 @@ +package util + +import "fmt" + +// FormatErrorList() is a wrapper function that unifies error list formatting +// and makes printing error lists consistent. +// +// NOTE: The error returned IS NOT an error in itself and may be a bit misleading. +// Instead, it is a single condensed error composed of all of the errors included +// in the errList argument. +func FormatErrorList(errList []error) error { + var err error + for i, e := range errList { + // NOTE: for multi-error formating, we want to include \n here + err = fmt.Errorf("\t[%d] %v\n", i, e) + } + return err +} + +// HasErrors() is a simple wrapper function to check if an error list contains +// errors. Having a function that clearly states its purpose helps to improve +// readibility although it may seem pointless. +func HasErrors(errList []error) bool { + return len(errList) > 0 +} diff --git a/internal/util/path.go b/internal/util/path.go new file mode 100644 index 0000000..c2e3e58 --- /dev/null +++ b/internal/util/path.go @@ -0,0 +1,69 @@ +package util + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// PathExists() is a wrapper function that simplifies checking +// if a file or directory already exists at the provided path. +// +// Returns whether the path exists and no error if successful, +// otherwise, it returns false with an error. +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +// SplitPathForViper() is an utility function to split a path into 3 parts: +// - directory +// - filename +// - extension +// The intent was to break a path into a format that's more easily consumable +// by spf13/viper's API. See the "LoadConfig()" function in internal/config.go +// for more details. +// +// TODO: Rename function to something more generalized. +func SplitPathForViper(path string) (string, string, string) { + filename := filepath.Base(path) + ext := filepath.Ext(filename) + return filepath.Dir(path), strings.TrimSuffix(filename, ext), strings.TrimPrefix(ext, ".") +} + +// MakeOutputDirectory() creates a new directory at the path argument if +// the path does not exist. +// +// Returns the final path that was created if no errors occurred. Otherwise, +// it returns an empty string with an error. +func MakeOutputDirectory(path string, overwrite bool) (string, error) { + // get the current data + time using Go's stupid formatting + t := time.Now() + dirname := t.Format("2006-01-01") + final := path + "/" + dirname + + // check if path is valid and directory + pathExists, err := PathExists(final) + if err != nil { + return "", fmt.Errorf("failed to check for existing path: %v", err) + } + if pathExists && !overwrite { + // make sure it is directory with 0o644 permissions + return "", fmt.Errorf("found existing path: %v", final) + } + + // create directory with data + time + err = os.MkdirAll(final, 0766) + if err != nil { + return "", fmt.Errorf("failed to make directory: %v", err) + } + return final, nil +} diff --git a/internal/util/util.go b/internal/util/util.go deleted file mode 100644 index 6817f4a..0000000 --- a/internal/util/util.go +++ /dev/null @@ -1,151 +0,0 @@ -package util - -import ( - "bytes" - "crypto/tls" - "fmt" - "io" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" -) - -// PathExists() is a wrapper function that simplifies checking -// if a file or directory already exists at the provided path. -// -// Returns whether the path exists and no error if successful, -// otherwise, it returns false with an error. -func PathExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return false, err -} - -// GetNextIP() returns the next IP address, but does not account -// for net masks. -func GetNextIP(ip *net.IP, inc uint) *net.IP { - if ip == nil { - return &net.IP{} - } - i := ip.To4() - v := uint(i[0])<<24 + uint(i[1])<<16 + uint(i[2])<<8 + uint(i[3]) - v += inc - v3 := byte(v & 0xFF) - v2 := byte((v >> 8) & 0xFF) - v1 := byte((v >> 16) & 0xFF) - v0 := byte((v >> 24) & 0xFF) - // return &net.IP{[]byte{v0, v1, v2, v3}} - r := net.IPv4(v0, v1, v2, v3) - return &r -} - -// MakeRequest() is a wrapper function that condenses simple HTTP -// requests done to a single call. It expects an optional HTTP client, -// URL, HTTP method, request body, and request headers. This function -// is useful when making many requests where only these few arguments -// are changing. -// -// Returns a HTTP response object, response body as byte array, and any -// error that may have occurred with making the request. -func MakeRequest(client *http.Client, url string, httpMethod string, body []byte, headers map[string]string) (*http.Response, []byte, error) { - // use defaults if no client provided - if client == nil { - client = http.DefaultClient - client.Transport = &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - } - req, err := http.NewRequest(httpMethod, url, bytes.NewBuffer(body)) - if err != nil { - return nil, nil, fmt.Errorf("failed to create new HTTP request: %v", err) - } - req.Header.Add("User-Agent", "magellan") - for k, v := range headers { - req.Header.Add(k, v) - } - res, err := client.Do(req) - if err != nil { - return nil, nil, fmt.Errorf("failed to make request: %v", err) - } - b, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - return nil, nil, fmt.Errorf("failed to read response body: %v", err) - } - return res, b, err -} - -// MakeOutputDirectory() creates a new directory at the path argument if -// the path does not exist -// -// TODO: Refactor this function for hive partitioning or possibly move into -// the logging package. -// TODO: Add an option to force overwriting the path. -func MakeOutputDirectory(path string) (string, error) { - // get the current data + time using Go's stupid formatting - t := time.Now() - dirname := t.Format("2006-01-01 15:04:05") - final := path + "/" + dirname - - // check if path is valid and directory - pathExists, err := PathExists(final) - if err != nil { - return final, fmt.Errorf("failed to check for existing path: %v", err) - } - if pathExists { - // make sure it is directory with 0o644 permissions - return final, fmt.Errorf("found existing path: %v", final) - } - - // create directory with data + time - err = os.MkdirAll(final, 0766) - if err != nil { - return final, fmt.Errorf("failed to make directory: %v", err) - } - return final, nil -} - -// SplitPathForViper() is an utility function to split a path into 3 parts: -// - directory -// - filename -// - extension -// The intent was to break a path into a format that's more easily consumable -// by spf13/viper's API. See the "LoadConfig()" function in internal/config.go -// for more details. -// -// TODO: Rename function to something more generalized. -func SplitPathForViper(path string) (string, string, string) { - filename := filepath.Base(path) - ext := filepath.Ext(filename) - return filepath.Dir(path), strings.TrimSuffix(filename, ext), strings.TrimPrefix(ext, ".") -} - -// FormatErrorList() is a wrapper function that unifies error list formatting -// and makes printing error lists consistent. -// -// NOTE: The error returned IS NOT an error in itself and may be a bit misleading. -// Instead, it is a single condensed error composed of all of the errors included -// in the errList argument. -func FormatErrorList(errList []error) error { - var err error - for i, e := range errList { - err = fmt.Errorf("\t[%d] %v\n", i, e) - i += 1 - } - return err -} - -// HasErrors() is a simple wrapper function to check if an error list contains -// errors. Having a function that clearly states its purpose helps to improve -// readibility although it may seem pointless. -func HasErrors(errList []error) bool { - return len(errList) > 0 -} diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go new file mode 100644 index 0000000..da5285d --- /dev/null +++ b/pkg/auth/auth.go @@ -0,0 +1,37 @@ +package auth + +import ( + "fmt" + "os" + + "github.com/spf13/viper" +) + +// LoadAccessToken() tries to load a JWT string from an environment +// variable, file, or config in that order. If loading the token +// fails with one options, it will fallback to the next option until +// all options are exhausted. +// +// Returns a token as a string with no error if successful. +// Alternatively, returns an empty string with an error if a token is +// not able to be loaded. +func LoadAccessToken(path string) (string, error) { + // try to load token from env var + testToken := os.Getenv("ACCESS_TOKEN") + if testToken != "" { + return testToken, nil + } + + // try reading access token from a file + b, err := os.ReadFile(path) + if err == nil { + return string(b), nil + } + + // TODO: try to load token from config + testToken = viper.GetString("access-token") + if testToken != "" { + return testToken, nil + } + return "", fmt.Errorf("failed to load token from environment variable, file, or config") +} diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..0005254 --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,79 @@ +package client + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "time" +) + +type Option[T Client] func(client T) + +// The 'Client' struct is a wrapper around the default http.Client +// that provides an extended API to work with functional options. +// It also provides functions that work with `collect` data. +type Client interface { + Name() string + GetClient() *http.Client + RootEndpoint(endpoint string) string + + // functions needed to make request + Add(data HTTPBody, headers HTTPHeader) error + Update(data HTTPBody, headers HTTPHeader) error +} + +// NewClient() creates a new client +func NewClient[T Client](opts ...func(T)) T { + client := new(T) + for _, opt := range opts { + opt(*client) + } + return *client +} + +func WithCertPool[T Client](certPool *x509.CertPool) func(T) { + if certPool == nil { + return func(client T) {} + } + return func(client T) { + client.GetClient().Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: true, + }, + DisableKeepAlives: true, + Dial: (&net.Dialer{ + Timeout: 120 * time.Second, + KeepAlive: 120 * time.Second, + }).Dial, + TLSHandshakeTimeout: 120 * time.Second, + ResponseHeaderTimeout: 120 * time.Second, + } + } +} + +func WithSecureTLS[T Client](certPath string) func(T) { + cacert, err := os.ReadFile(certPath) + if err != nil { + return func(client T) {} + } + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(cacert) + return WithCertPool[T](certPool) +} + +// Post() is a simplified wrapper function that packages all of the +// that marshals a mapper into a JSON-formatted byte array, and then performs +// a request to the specified URL. +func (c *MagellanClient) Post(url string, data map[string]any, header HTTPHeader) (*http.Response, HTTPBody, error) { + // serialize data into byte array + body, err := json.Marshal(data) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal data for request: %v", err) + } + return MakeRequest(c.Client, url, http.MethodPost, body, header) +} diff --git a/pkg/client/default.go b/pkg/client/default.go new file mode 100644 index 0000000..2830921 --- /dev/null +++ b/pkg/client/default.go @@ -0,0 +1,53 @@ +package client + +import ( + "fmt" + "net/http" +) + +type MagellanClient struct { + *http.Client +} + +func (c *MagellanClient) Name() string { + return "default" +} + +// Add() is the default function that is called with a client with no implementation. +// This function will simply make a HTTP request including all the data passed as +// the first argument with no data processing or manipulation. The function sends +// the data to a set callback URL (which may be changed to use a configurable value +// instead). +func (c *MagellanClient) Add(data HTTPBody, headers HTTPHeader) error { + if data == nil { + return fmt.Errorf("no data found") + } + + path := "/inventory/add" + res, body, err := MakeRequest(c.Client, path, http.MethodPost, data, headers) + if res != nil { + statusOk := res.StatusCode >= 200 && res.StatusCode < 300 + if !statusOk { + return fmt.Errorf("returned status code %d when POST'ing to endpoint", res.StatusCode) + } + fmt.Printf("%v (%v)\n%s\n", path, res.Status, string(body)) + } + return err +} + +func (c *MagellanClient) Update(data HTTPBody, headers HTTPHeader) error { + if data == nil { + return fmt.Errorf("no data found") + } + + path := "/inventory/update" + res, body, err := MakeRequest(c.Client, path, http.MethodPut, data, headers) + if res != nil { + statusOk := res.StatusCode >= 200 && res.StatusCode < 300 + if !statusOk { + return fmt.Errorf("returned status code %d when PUT'ing to endpoint", res.StatusCode) + } + fmt.Printf("%v (%v)\n%s\n", path, res.Status, string(body)) + } + return err +} diff --git a/pkg/client/net.go b/pkg/client/net.go new file mode 100644 index 0000000..9a41d77 --- /dev/null +++ b/pkg/client/net.go @@ -0,0 +1,80 @@ +package client + +import ( + "bytes" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" +) + +// HTTP aliases for readibility +type HTTPHeader map[string]string +type HTTPBody []byte + +func (h HTTPHeader) Authorization(accessToken string) HTTPHeader { + if accessToken != "" { + h["Authorization"] = fmt.Sprintf("Bearer %s", accessToken) + } + return h +} + +func (h HTTPHeader) ContentType(contentType string) HTTPHeader { + h["Content-Type"] = contentType + return h +} + +// GetNextIP() returns the next IP address, but does not account +// for net masks. +func GetNextIP(ip *net.IP, inc uint) *net.IP { + if ip == nil { + return &net.IP{} + } + i := ip.To4() + v := uint(i[0])<<24 + uint(i[1])<<16 + uint(i[2])<<8 + uint(i[3]) + v += inc + v3 := byte(v & 0xFF) + v2 := byte((v >> 8) & 0xFF) + v1 := byte((v >> 16) & 0xFF) + v0 := byte((v >> 24) & 0xFF) + // return &net.IP{[]byte{v0, v1, v2, v3}} + r := net.IPv4(v0, v1, v2, v3) + return &r +} + +// MakeRequest() is a wrapper function that condenses simple HTTP +// requests done to a single call. It expects an optional HTTP client, +// URL, HTTP method, request body, and request headers. This function +// is useful when making many requests where only these few arguments +// are changing. +// +// Returns a HTTP response object, response body as byte array, and any +// error that may have occurred with making the request. +func MakeRequest(client *http.Client, url string, httpMethod string, body HTTPBody, header HTTPHeader) (*http.Response, HTTPBody, error) { + // use defaults if no client provided + if client == nil { + client = http.DefaultClient + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + } + req, err := http.NewRequest(httpMethod, url, bytes.NewBuffer(body)) + if err != nil { + return nil, nil, fmt.Errorf("failed to create new HTTP request: %v", err) + } + req.Header.Add("User-Agent", "magellan") + for k, v := range header { + req.Header.Add(k, v) + } + res, err := client.Do(req) + if err != nil { + return nil, nil, fmt.Errorf("failed to make request: %v", err) + } + b, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, nil, fmt.Errorf("failed to read response body: %v", err) + } + return res, b, err +} diff --git a/pkg/client/smd.go b/pkg/client/smd.go new file mode 100644 index 0000000..69fbf17 --- /dev/null +++ b/pkg/client/smd.go @@ -0,0 +1,65 @@ +package client + +// See ref for API docs: +// https://github.com/OpenCHAMI/hms-smd/blob/master/docs/examples.adoc +// https://github.com/OpenCHAMI/hms-smd +import ( + "fmt" + "net/http" +) + +type SmdClient struct { + *http.Client + URI string + Xname string +} + +func (c SmdClient) Name() string { + return "smd" +} + +func (c SmdClient) RootEndpoint(endpoint string) string { + return fmt.Sprintf("%s/hsm/v2%s", c.URI, endpoint) +} + +func (c SmdClient) GetClient() *http.Client { + return c.Client +} + +// Add() has a similar function definition to that of the default implementation, +// but also allows further customization and data/header manipulation that would +// be specific and/or unique to SMD's API. +func (c SmdClient) Add(data HTTPBody, headers HTTPHeader) error { + if data == nil { + return fmt.Errorf("failed to add redfish endpoint: no data found") + } + + // Add redfish endpoint via POST `/hsm/v2/Inventory/RedfishEndpoints` endpoint + url := c.RootEndpoint("/Inventory/RedfishEndpoints") + res, body, err := MakeRequest(c.Client, url, http.MethodPost, data, headers) + if res != nil { + statusOk := res.StatusCode >= 200 && res.StatusCode < 300 + if !statusOk { + return fmt.Errorf("returned status code %d when adding endpoint", res.StatusCode) + } + fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) + } + return err +} + +func (c SmdClient) Update(data HTTPBody, headers HTTPHeader) error { + if data == nil { + return fmt.Errorf("failed to add redfish endpoint: no data found") + } + // Update redfish endpoint via PUT `/hsm/v2/Inventory/RedfishEndpoints` endpoint + url := c.RootEndpoint("/Inventory/RedfishEndpoints/" + c.Xname) + res, body, err := MakeRequest(c.Client, url, http.MethodPut, data, headers) + if res != nil { + statusOk := res.StatusCode >= 200 && res.StatusCode < 300 + if !statusOk { + return fmt.Errorf("failed to update redfish endpoint (returned %s)", res.Status) + } + fmt.Printf("%v (%v)\n%s\n", url, res.Status, string(body)) + } + return err +} diff --git a/tests/api_test.go b/tests/api_test.go index 558e688..c213451 100644 --- a/tests/api_test.go +++ b/tests/api_test.go @@ -11,43 +11,63 @@ import ( "testing" magellan "github.com/OpenCHAMI/magellan/internal" - "github.com/OpenCHAMI/magellan/internal/log" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog/log" +) + +var ( + scanParams = &magellan.ScanParams{ + TargetHosts: [][]string{ + []string{ + "http://127.0.0.1:443", + "http://127.0.0.1:5000", + }, + }, + Scheme: "https", + Protocol: "tcp", + Concurrency: 1, + Timeout: 30, + DisableProbing: false, + Verbose: false, + } ) func TestScanAndCollect(t *testing.T) { - var ( - hosts = []string{"http://127.0.0.1"} - ports = []int{5000} - l = log.NewLogger(logrus.New(), logrus.DebugLevel) - ) // do a scan on the emulator cluster with probing disabled and check results - results := magellan.ScanForAssets(hosts, ports, 1, 30, true, false) + results := magellan.ScanForAssets(scanParams) if len(results) <= 0 { t.Fatal("expected to find at least one BMC node, but found none") } // do a scan on the emulator cluster with probing enabled - results = magellan.ScanForAssets(hosts, ports, 1, 30, false, false) + results = magellan.ScanForAssets(scanParams) if len(results) <= 0 { t.Fatal("expected to find at least one BMC node, but found none") } // do a collect on the emulator cluster to collect Redfish info - magellan.CollectAll(results) + err := magellan.CollectInventory(&results, &magellan.CollectParams{}) + if err != nil { + log.Error().Err(err).Msg("failed to collect inventory") + } } func TestCrawlCommand(t *testing.T) { - + // TODO: add test to check the crawl command's behavior } func TestListCommand(t *testing.T) { - + // TODO: add test to check the list command's output } func TestUpdateCommand(t *testing.T) { - + // TODO: add test that does a Redfish simple update checking it success and + // failure points } func TestGofishFunctions(t *testing.T) { + // TODO: add test that checks certain gofish function output to make sure + // gofish's output isn't changing spontaneously and remains predictable +} +func TestGenerateHosts(t *testing.T) { + // TODO: add test to generate hosts using a collection of subnets/masks } diff --git a/tests/compatibility_test.go b/tests/compatibility_test.go index 5517bd2..86e3a4f 100644 --- a/tests/compatibility_test.go +++ b/tests/compatibility_test.go @@ -13,8 +13,9 @@ import ( "net/http" "testing" - "github.com/OpenCHAMI/magellan/internal/util" + "github.com/OpenCHAMI/magellan/pkg/client" "github.com/OpenCHAMI/magellan/pkg/crawler" + "github.com/rs/zerolog/log" ) var ( @@ -26,11 +27,11 @@ var ( // Simple test to fetch the base Redfish URL and assert a 200 OK response. func TestRedfishV1Availability(t *testing.T) { var ( - url = fmt.Sprintf("%s/redfish/v1", host) + url = fmt.Sprintf("%s/redfish/v1", *host) body = []byte{} headers = map[string]string{} ) - res, b, err := util.MakeRequest(nil, url, http.MethodGet, body, headers) + res, b, err := client.MakeRequest(nil, url, http.MethodGet, body, headers) if err != nil { t.Fatalf("failed to make request to BMC: %v", err) } @@ -55,12 +56,16 @@ func TestRedfishV1Availability(t *testing.T) { // Simple test to ensure an expected Redfish version minimum requirement. func TestRedfishVersion(t *testing.T) { var ( - url = fmt.Sprintf("%s/redfish/v1", host) - body = []byte{} - headers = map[string]string{} + url string = fmt.Sprintf("%s/redfish/v1", *host) + body client.HTTPBody = []byte{} + headers client.HTTPHeader = map[string]string{} + err error ) - util.MakeRequest(nil, url, http.MethodGet, body, headers) + _, _, err = client.MakeRequest(nil, url, http.MethodGet, body, headers) + if err != nil { + log.Error().Err(err).Msg("failed to make request") + } } // Crawls a BMC node and checks that we're able to query certain properties