diff --git a/README.md b/README.md index e6755fd2..dec26e30 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ The `magellan` CLI tool is a Redfish-based, board management controller (BMC) di - [Checking for Redfish](#checking-for-redfish) - [BMC ID Mapping](#bmc-id-mapping) - [Running the Tool](#running-the-tool) + - [Modular Workflows](#modular-workflows) - [PDU Inventory Collection](#pdu-inventory-collection) - [Starting the Emulator](#starting-the-emulator) - [Updating Firmware](#updating-firmware) @@ -213,8 +214,9 @@ If you are using `magellan` in an application that is not OpenCHAMI and have a n There are three main commands to use with the tool: `scan`, `list`, and `collect`. To see all of the available commands, run `magellan` with the `help` subcommand which will print this output: -``` -Redfish-based BMC discovery tool +```bash +magellan help +Redfish-based BMC discovery tool with dynamic discovery features. Usage: magellan [flags] @@ -227,21 +229,25 @@ Available Commands: help Help about any command list List information stored in cache from a scan login Log in with identity provider for access token + power Get and set node power states scan Scan to discover BMC nodes on a network + secrets Manage credentials for BMC nodes + send Send collected node information to specified host. update Update BMC node firmware version Print version info and exit Flags: --access-token string Set the access token --cache string Set the scanning result cache path (default "/tmp/allend/magellan/assets.db") - --concurrency int Set the number of concurrent processes (default -1) + -j, --concurrency int Set the number of concurrent processes (default -1) -c, --config string Set the config file path - -d, --debug Set to enable/disable debug messages -h, --help help for magellan - --timeout int Set the timeout for requests (default 5) - -v, --verbose Set to enable/disable verbose output + --log-file string Set the path to store a log file + -l, --log-level LogLevel Set the logger log-level (debug|info|warn|error|trace|disabled) (default info) + -t, --timeout int Set the timeout for requests in seconds (default 5) Use "magellan [command] --help" for more information about a command. + ``` To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe the common Redfish port 443 by default: @@ -294,7 +300,7 @@ magellan send -F yaml -d @nodes.yaml https://example.openchami.cluster:8443 This allows for modification of the data before making the request. However, be cautious as there is no data validation done before the request is made. -Alternatively, we can pass the output of `collect` into `send` using pipes. The `--verbose` flag is currently required to do this. +Alternatively, we can pass the output of `collect` into `send` using pipes. See the ["Modular Workflows"](#modular-workflows) section for more details. ```bash # collect and send data in YAML format @@ -303,7 +309,7 @@ magellan collect -u $USERNAME -p $PASSWORD -v -F yaml | magellan send -F yaml ht # collect and send data using default JSON format and secret store (see below) export MASTER_KEY=mysecret magellan secrets store default $USERNAME:$PASSWORD -magellan collect -v | magellan send https://example.openchami.cluster:8443 +magellan collect | magellan send https://example.openchami.cluster:8443 ``` This maintains the original behavior of passing the `--host` flag to `collect` with the added flexibility of having the intermediate step. @@ -311,7 +317,6 @@ This maintains the original behavior of passing the `--host` flag to `collect` w > [!TIP] > If the `cache` flag is not set, `magellan` will use `/tmp/$USER/magellan.db` by default. - > [!TIP] > The output of `collect` can be saved in separate directories using the `-O/--output-dir` flag. The output will be organized similar to below for the following command in YAML format: > @@ -324,6 +329,29 @@ This maintains the original behavior of passing the `--host` flag to `collect` w > └── 1747550498.yaml > ``` +#### Modular Workflows + +The `magellan` CLI commands can be ran in a single command or broken up to run different parts of the workflow without needing to write to the filesystem. + +For example, the `scan`, `collect`, and `send` can be done in a single command. + +```bash +# scan -> collect -> send +magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | magellan collect -f json --show-output -i | magellan send https://smd.example.com +``` + +Alternatively, we can run `scan -> collect` and `collect -> send` parts separately if we're only interested in performing one part of the process. + +```bash +# scan -> collect +magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | magellan collect -f json --show-output -i + +# collect -> send +magellan collect pdu x3000m0 x3000m1 -u admin -p initial0 | magellan send https://smd.example.com +``` + +> [!NOTE] See `magellan-send(1)` and `magellan-collect(1)` documentation for more info and examples. + ### PDU Inventory Collection In addition to collecting Redfish inventory from BMCs, `magellan` can also collect inventory from Power Distribution Units (PDUs) that expose a JAWS-style API. The `collect` command has a `pdu` subcommand for this purpose. diff --git a/cmd/collect.go b/cmd/collect.go index 6fd1bbd5..c6010b4d 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -9,16 +9,20 @@ import ( "github.com/OpenCHAMI/magellan/internal/cache/sqlite" "github.com/OpenCHAMI/magellan/internal/format" magellan "github.com/OpenCHAMI/magellan/pkg" - "github.com/OpenCHAMI/magellan/pkg/auth" "github.com/OpenCHAMI/magellan/pkg/bmc" "github.com/OpenCHAMI/magellan/pkg/secrets" "github.com/cznic/mathutil" "github.com/rs/zerolog/log" "github.com/spf13/cobra" "github.com/spf13/viper" + _ "modernc.org/sqlite" ) -var collectOutputFormat format.DataFormat = format.FORMAT_JSON +var ( + collectInputFormat format.DataFormat = format.FORMAT_JSON + collectOutputFormat format.DataFormat = format.FORMAT_JSON + collectDataArgs []string +) // The `collect` command fetches data from a collection of BMC nodes. // This command should be ran after the `scan` to find available hosts @@ -35,21 +39,62 @@ var CollectCmd = &cobra.Command{ // run a collect using secrets from the secrets manager export MASTER_KEY=$(magellan secrets generatekey) magellan secrets store $node_creds_json -f nodes.json - magellan collect -o nodes.yaml`, + magellan collect -o nodes.yaml + + // Take the output of 'scan' and input directly into 'collect' + magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | ./magellan collect -f json --show-output -i + + // Complete flow combined as a single line + magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | ./magellan collect -f json --show-output -i | magellan send https://smd.example.com + `, Short: "Collect system information by interrogating BMC node", Long: "Send request(s) to a collection of hosts running Redfish services found stored from the 'scan' in cache.\nSee the 'scan' command on how to perform a scan.", Run: func(cmd *cobra.Command, args []string) { // get probe states stored in db from scan - scannedResults, err := sqlite.GetScannedAssets(cachePath) - if err != nil { - log.Error().Err(err).Msgf("failed to get scanned results from cache") - } + var ( + scannedResults []magellan.RemoteAsset + err error + ) + if cachePath != "" { + scannedResults, err = sqlite.GetScannedAssets(cachePath) + if err != nil { + log.Error().Err(err).Msgf("failed to get scanned results from cache") + } + } else { + // try to get the data from standard input or the -d/--data flag + for _, arg := range args { + var asset magellan.RemoteAsset + err = format.UnmarshalData([]byte(arg), &asset, collectInputFormat) + if err != nil { + log.Warn().Err(err).Msg("failed to unmarshal data from standard input") + continue + } + } + + var inputData []map[string]any + temp := append(handleArgs(args), processDataArgs(sendDataArgs)...) + for _, data := range temp { + if data != nil { + inputData = append(inputData, data) + } + } + if len(inputData) == 0 { + log.Error().Msg("data required with standard input or -d/--data flag") + os.Exit(1) + } + + // show the data that was just loaded as input + // inputRaw, _ := json.MarshalIndent(inputData, "", " ") + log.Debug().Int("endpoint_count", len(inputData)).Send() + + // build and append target hosts from input data + // for _, dataObject := range inputData { + // // assert that we have certain values in data object + // var asset magellan.RemoteAsset + // format.UnmarshalData() + // host := dataObject["host"].(string) - // try to load access token either from env var, file, or config if var not set - if accessToken == "" { - var err error - accessToken, err = auth.LoadAccessToken(tokenPath) - log.Warn().Err(err).Msgf("could not load access token") + // } } // set the minimum/maximum number of concurrent processes @@ -153,13 +198,18 @@ func init() { CollectCmd.Flags().StringVarP(&outputDir, "output-dir", "O", "", "Set the path to store collection data using HIVE partitioning") CollectCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Skip TLS certificate verification during probe") CollectCmd.Flags().BoolVar(&showOutput, "show", false, "Show the output of a collect run") - CollectCmd.Flags().VarP(&collectOutputFormat, "format", "F", "Set the default output data format (json|yaml; can be overridden by file extensions)") + CollectCmd.Flags().BoolVar(&showOutput, "show-output", false, "Show the output of a collect run") + CollectCmd.Flags().VarP(&collectInputFormat, "input-format", "f", "Set the default input data format (json|yaml)") + CollectCmd.Flags().VarP(&collectOutputFormat, "output-format", "F", "Set the default output data format (json|yaml; can be overridden by file extensions)") CollectCmd.Flags().StringVarP(&idMap, "bmc-id-map", "m", "", "Set the BMC ID mapping from raw json data or use @ to specify a file path (json or yaml input)") + CollectCmd.Flags().StringArrayVarP(&collectDataArgs, "data", "d", []string{}, "Set the data as input for collect (prepend @ for files)") + // set mutually exclusive flags CollectCmd.MarkFlagsMutuallyExclusive("output-file", "output-dir") // register completion flag functions - checkRegisterFlagCompletionError(CollectCmd.RegisterFlagCompletionFunc("format", completionFormatData)) + checkRegisterFlagCompletionError(CollectCmd.RegisterFlagCompletionFunc("input-format", completionFormatData)) + checkRegisterFlagCompletionError(CollectCmd.RegisterFlagCompletionFunc("output-format", completionFormatData)) // bind flags to config properties checkBindFlagError(viper.BindPFlag("collect.protocol", CollectCmd.Flags().Lookup("protocol"))) diff --git a/cmd/crawl.go b/cmd/crawl.go index 01358e01..59501b86 100644 --- a/cmd/crawl.go +++ b/cmd/crawl.go @@ -150,7 +150,8 @@ func init() { CrawlCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Ignore SSL errors") CrawlCmd.Flags().StringVarP(&secretsFile, "secrets-file", "f", "secrets.json", "Set path to the node secrets file") CrawlCmd.Flags().BoolVar(&showOutput, "show", false, "Show the output of a crawl") - CrawlCmd.Flags().VarP(&crawlOutputFormat, "format", "F", "Set the output format (json|yaml)") + CrawlCmd.Flags().BoolVar(&showOutput, "show-output", false, "Show the output of a collect run") + CrawlCmd.Flags().VarP(&crawlOutputFormat, "output-format", "F", "Set the output format (json|yaml)") checkRegisterFlagCompletionError(CrawlCmd.RegisterFlagCompletionFunc("format", completionFormatData)) diff --git a/cmd/scan.go b/cmd/scan.go index 01efdf95..6b8b8823 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -36,8 +36,7 @@ var ( // related to the implementation. var ScanCmd = &cobra.Command{ Use: "scan urls...", - Example: ` - // assumes host https://10.0.0.101:443 + Example: ` // assumes host https://10.0.0.101:443 magellan scan 10.0.0.101 --insecure // assumes subnet using HTTPS and port 443 except for specified host @@ -52,6 +51,9 @@ var ScanCmd = &cobra.Command{ // assumes subnet using HTTPS and port 443 with specified CIDR magellan scan --subnet 10.0.0.0/16 -i + // same as above example but output is in JSON without caching + magellan scan --subnet 10.0.0.0/16 -i -f json --disable-cache + // assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16 magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0 @@ -166,10 +168,14 @@ var ScanCmd = &cobra.Command{ } else { fmt.Println(string(output)) } + // stop here so we don't write to cache if using JSON or YAML + return default: log.Error().Msgf("unknown format specified: %s. Please use 'db', 'json', or 'yaml'.", scanFormat) } } + + // write to a cache file if not disabled at specified path if !disableCache && cachePath != "" { err := os.MkdirAll(path.Dir(cachePath), 0755) if err != nil { diff --git a/go.mod b/go.mod index aaeb9977..6a381578 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/OpenCHAMI/magellan -go 1.23 +go 1.24.0 toolchain go1.24.5 @@ -9,12 +9,11 @@ require ( github.com/go-chi/chi/v5 v5.1.0 github.com/jmoiron/sqlx v1.4.0 github.com/lestrrat-go/jwx v1.2.29 - github.com/mattn/go-sqlite3 v1.14.22 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 github.com/stmcginnis/gofish v0.19.0 - golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 ) require ( @@ -23,10 +22,10 @@ require ( github.com/stretchr/testify v1.9.0 golang.org/x/crypto v0.32.0 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.46.1 ) require ( - github.com/google/go-cmp v0.6.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect ) @@ -34,8 +33,10 @@ require ( require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect @@ -45,6 +46,7 @@ require ( github.com/lestrrat-go/option v1.0.1 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -58,8 +60,11 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/sys v0.29.0 // indirect + golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.21.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect + modernc.org/libc v1.67.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index edef927f..66f7f602 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8Yc github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -26,6 +28,13 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -66,6 +75,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= @@ -125,10 +136,12 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= -golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -138,6 +151,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -151,8 +166,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -171,6 +186,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -180,3 +197,31 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= +modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc= +modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE= +modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= +modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/cache/sqlite/sqlite.go b/internal/cache/sqlite/sqlite.go index e653aa03..5e429016 100644 --- a/internal/cache/sqlite/sqlite.go +++ b/internal/cache/sqlite/sqlite.go @@ -7,9 +7,13 @@ import ( magellan "github.com/OpenCHAMI/magellan/pkg" "github.com/jmoiron/sqlx" + _ "modernc.org/sqlite" ) -const TABLE_NAME = "magellan_scanned_assets" +const ( + SQLITE_DRIVER = "sqlite" + TABLE_NAME = "magellan_scanned_assets" +) func CreateScannedAssetIfNotExists(path string) (*sqlx.DB, error) { schema := fmt.Sprintf(` @@ -23,7 +27,7 @@ func CreateScannedAssetIfNotExists(path string) (*sqlx.DB, error) { ); `, TABLE_NAME) // TODO: it may help with debugging to check for file permissions here first - db, err := sqlx.Open("sqlite3", path) + db, err := sqlx.Open(SQLITE_DRIVER, path) if err != nil { return nil, fmt.Errorf("failed to open database: %v", err) } @@ -66,7 +70,7 @@ func DeleteScannedAssets(path string, results ...magellan.RemoteAsset) error { if results == nil { return fmt.Errorf("no assets found") } - db, err := sqlx.Open("sqlite3", path) + db, err := sqlx.Open(SQLITE_DRIVER, path) if err != nil { return fmt.Errorf("failed to open database: %v", err) } @@ -94,7 +98,7 @@ func GetScannedAssets(path string) ([]magellan.RemoteAsset, error) { } // now check if the file is the SQLite database - db, err := sqlx.Open("sqlite3", path) + db, err := sqlx.Open(SQLITE_DRIVER, path) if err != nil { return nil, fmt.Errorf("failed to open database: %v", err) } diff --git a/man/magellan-collect.1.sc b/man/magellan-collect.1.sc index 66ac3c63..4dbe5071 100644 --- a/man/magellan-collect.1.sc +++ b/man/magellan-collect.1.sc @@ -29,6 +29,12 @@ magellan collect pdu x3000m0 --username admin --password initial0 // Collect from multiple PDUs and send to SMD++ magellan collect pdu x3000m0 x3000m1 -u admin -p initial0 | magellan send https://smd.example.com +// Take the output of 'scan' and input directly into 'collect' +magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | ./magellan collect -f json --show-output -i + +// Complete flow combined as a single line +magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | ./magellan collect -f json --show-output -i | magellan send https://smd.example.com + # FLAGS *-m, --bmc-id-map* (_data_ | @_path_) @@ -65,7 +71,26 @@ magellan collect pdu x3000m0 x3000m1 -u admin -p initial0 | magellan send https: remove and then re-create the objects whenever a 409 is received from the initial response. -*-F, --format* _format_ +*-f, --input-format* _format_ + Set the data format used for STDIN for the collection input. + + Supported values _format_ are: + + - _json_ (default) + - _yaml_ + + STDIN expects the following values to be specified. These values are usually + found from running a scan. + + *host* - Host of the remote asset. + *port* - Port of the remote asset. + *protocol* - Protocol TCP or UDP. (tcp|udp) + *state* - State of the remote asset. (true|false) + *timestamp* - Last time the remote asset was scanned or pinged. + *service_type* - Type of service. (Redfish|PDU) + + +*-F, --output-format* _format_ Set the data format used for the collection output. This value is overridden whenever a file is specified with *--output-file* with a file extension @@ -125,6 +150,9 @@ magellan collect pdu x3000m0 x3000m1 -u admin -p initial0 | magellan send https: *--show* Show the output of a collect run. +*--show-output* + Alias for '--show'. + *-u, --username* _value_ Set the username to _value_ used for basic authentication to the BMC node. When this flag is set, the value overrides all of the values loaded from the diff --git a/man/magellan-scan.1.sc b/man/magellan-scan.1.sc index 9b8e8118..619a1732 100644 --- a/man/magellan-scan.1.sc +++ b/man/magellan-scan.1.sc @@ -25,17 +25,20 @@ magellan scan --subnet 10.0.0.0 -i // assumes subnet using HTTPS and port 443 with specified CIDR++ magellan scan --subnet 10.0.0.0/16 -i +// same as above example but output is in JSON without caching +magellan scan --subnet 10.0.0.0/16 -i -f json --disable-cache + // assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16++ magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0 -// assumes subnet without CIDR has a subnet-mask of 255.255.0.0++ +// assumes subnet without CIDR has a subnet-mask of 255.255.0.0 magellan scan --subnet 10.0.0.0 --subnet 172.16.0.0 --subnet-mask 255.255.0.0 --cache ./assets.db # FLAGS *--disable-cache* Disable saving found remote assets that respond to a Redfish request to a - cache database specified with *--cache* flag. By default, the cache is saved + cache database specifiead with *--cache* flag. By default, the cache is saved at */tmp/$USER/magellan/assets.db* as a SQLite3 file with a table named *magellan_scanned_assets*. It is set to _false_ by default. diff --git a/pkg/collect.go b/pkg/collect.go index 3ccb8a5d..a2a04c8e 100644 --- a/pkg/collect.go +++ b/pkg/collect.go @@ -20,7 +20,6 @@ import ( "github.com/rs/zerolog/log" - _ "github.com/mattn/go-sqlite3" "github.com/stmcginnis/gofish" "github.com/stmcginnis/gofish/redfish" "golang.org/x/exp/slices" diff --git a/pkg/crawler/main.go b/pkg/crawler/main.go index 28d519ef..dedbe967 100644 --- a/pkg/crawler/main.go +++ b/pkg/crawler/main.go @@ -492,6 +492,7 @@ func walkManagers(rf_managers []*redfish.Manager, baseURI string) ([]Manager, er for _, shell_type := range rf_manager.CommandShell.ConnectTypesSupported { supported_command_shell = append(supported_command_shell, string(shell_type)) } + managers = append(managers, Manager{ URI: baseURI + "/redfish/v1/Managers/" + rf_manager.ID, UUID: rf_manager.UUID, @@ -508,43 +509,6 @@ func walkManagers(rf_managers []*redfish.Manager, baseURI string) ([]Manager, er return managers, nil } -// func getPowerInfo(serviceroot *gofish.Service) ([]Power, error) { -// // get the power control related information (Actions, URL, PowerControl, Links, etc.) - -// // get the SupportedResetTypes from /redfish/v1/Systems -// // get the Power/PowerControl from /redfish/v1/Chassis -// rf_chassis, err := serviceroot.Chassis() -// if err != nil { - -// } - -// power := []Power{} -// for _, chassis := range rf_chassis { -// rf_power, err := chassis.Power() -// if err != nil { - -// } -// rf_computersystems, err := chassis.ComputerSystems() -// if err != nil { - -// } - -// for _, computersystem := range rf_computersystems { -// computersystem.SupportedResetTypes -// } - -// power = append(power, Power{ -// URL: "", -// Control: PowerControl{ -// MemberID: "", -// ResetTypes: rf_computersystem.SupportedResetTypes, -// RelatedItems: []string{}, -// }, -// }) -// } - -// } - func loadBMCCreds(config CrawlerConfig) (bmc.BMCCredentials, error) { // NOTE: it is possible for the SecretStore to be nil, so we need a check if config.CredentialStore == nil { diff --git a/tests/api_test.go b/tests/api_test.go index 999f142d..bc650e2b 100644 --- a/tests/api_test.go +++ b/tests/api_test.go @@ -45,6 +45,9 @@ func TestScanAndCollect(t *testing.T) { buferr bytes.Buffer ) + // say what test we're starting + fmt.Printf("[%s] Starting test...", t.Name()) + // set up the emulator to run before test err = waitUntilEmulatorIsReady() if err != nil { @@ -69,12 +72,18 @@ func TestScanAndCollect(t *testing.T) { if err != nil { t.Fatalf("failed to get absolute path: %v", err) } - command = strings.Split("scan https://127.0.0.1 --port 5000 --verbose", " ") + command = strings.Split("scan https://127.0.0.1 --port 5000 --log-level debug", " ") cmd = exec.Command(path, command...) cmd.Stdout = &bufout cmd.Stderr = &buferr err = cmd.Run() - fmt.Printf("out:\n%s\nerr:\n%s\n", bufout.String(), buferr.String()) + + // show output and error of test + fmt.Printf("[%s INFO] %s\n [%s ERR] %s\n", + t.Name(), bufout.String(), + t.Name(), buferr.String(), + ) + if err != nil { t.Fatalf("failed to run 'scan' command: %v", err) } @@ -86,12 +95,18 @@ func TestScanAndCollect(t *testing.T) { // try and run a "collect" with the emulator - command = strings.Split("collect --username root --password root_password --verbose", " ") + command = strings.Split("collect --username root --password root_password --log-level debug --insecure --show-output", " ") cmd = exec.Command(path, command...) cmd.Stdout = &bufout cmd.Stderr = &buferr err = cmd.Run() - fmt.Printf("out:\n%s\nerr:\n%s\n", bufout.String(), buferr.String()) + + // show output and error of test + fmt.Printf("[%s INFO] %s\n [%s ERR] %s\n", + t.Name(), bufout.String(), + t.Name(), buferr.String(), + ) + if err != nil { t.Fatalf("failed to run 'collect' command: %v", err) } @@ -101,6 +116,9 @@ func TestScanAndCollect(t *testing.T) { t.Fatalf("expected the 'collect' output to not be empty") } + // say what test we're completing + fmt.Printf("[%s] Test complete.", t.Name()) + // TODO: check for at least one System/EthernetInterface that we know should exist } @@ -114,6 +132,9 @@ func TestCrawlCommand(t *testing.T) { path string ) + // say what test we're starting + fmt.Printf("[%s] Starting test...", t.Name()) + // set up the emulator to run before test path, err = filepath.Abs(*exePath) if err != nil { @@ -126,26 +147,29 @@ func TestCrawlCommand(t *testing.T) { } // try and run a "collect" with the emulator - command = strings.Split("crawl --username root --password root_password -i https://127.0.0.1:5000", " ") + command = strings.Split("crawl --username root --password root_password --insecure https://127.0.0.1:5000 --show-output", " ") cmd = exec.Command(path, command...) cmd.Stdout = &bufout cmd.Stderr = &buferr err = cmd.Run() - fmt.Printf("out:\n%s\nerr:\n%s\n", bufout.String(), buferr.String()) + + // show output and error of test + fmt.Printf("[%s INFO] %s\n [%s ERR] %s\n", + t.Name(), bufout.String(), + t.Name(), buferr.String(), + ) + if err != nil { t.Fatalf("failed to run 'crawl' command: %v", err) } - // err = cmd.Wait() - // if err != nil { - // t.Fatalf("failed to call 'wait' for crawl: %v", err) - // } - // make sure that the output is not empty if len(bufout.Bytes()) <= 0 { t.Fatalf("expected the 'crawl' output to not be empty") } + // say what test we're completing + fmt.Printf("[%s] Test complete.", t.Name()) } func TestListCommand(t *testing.T) { @@ -154,6 +178,9 @@ func TestListCommand(t *testing.T) { cmd *exec.Cmd ) + // say what test we're starting + fmt.Printf("[%s] Starting test...", t.Name()) + // set up the emulator to run before test err = waitUntilEmulatorIsReady() if err != nil { @@ -168,6 +195,8 @@ func TestListCommand(t *testing.T) { } // NOTE: the output of `list` can be empty if no scan has been performed + // say what test we're completing + fmt.Printf("[%s] Test complete.", t.Name()) } func TestUpdateCommand(t *testing.T) { @@ -178,6 +207,9 @@ func TestUpdateCommand(t *testing.T) { err error ) + // say what test we're starting + fmt.Printf("[%s] Starting test...", t.Name()) + // set up the emulator to run before test err = waitUntilEmulatorIsReady() if err != nil { @@ -191,6 +223,8 @@ func TestUpdateCommand(t *testing.T) { t.Fatalf("failed to run 'update' command: %v", err) } + // say what test we're completing + fmt.Printf("[%s] Test complete.", t.Name()) } func TestGofishFunctions(t *testing.T) { @@ -208,6 +242,10 @@ func TestGenerateHosts(t *testing.T) { scheme = "https" hosts = [][]string{} ) + + // say what test we're starting + fmt.Printf("[%s] Starting test...", t.Name()) + t.Run("generate-hosts", func(t *testing.T) { hosts = magellan.GenerateHostsWithSubnet(subnet, subnetMask, ports, scheme) @@ -237,6 +275,8 @@ func TestGenerateHosts(t *testing.T) { } }) + // say what test we're completing + fmt.Printf("[%s] Test complete.", t.Name()) } func startEmulatorInBackground(path string) (int, error) {