Update collect to take input with specified format

README.md

@@ -19,6 +19,7 @@ The `magellan` CLI tool is a Redfish-based, board management controller (BMC) di
 		- [Checking for Redfish](#checking-for-redfish)
 		- [BMC ID Mapping](#bmc-id-mapping)
 		- [Running the Tool](#running-the-tool)
+			- [Modular Workflows](#modular-workflows)
 		- [PDU Inventory Collection](#pdu-inventory-collection)
 		- [Starting the Emulator](#starting-the-emulator)
 		- [Updating Firmware](#updating-firmware)
@@ -213,8 +214,9 @@ If you are using `magellan` in an application that is not OpenCHAMI and have a n
 
 There are three main commands to use with the tool: `scan`, `list`, and `collect`. To see all of the available commands, run `magellan` with the `help` subcommand which will print this output:
 
-```
-Redfish-based BMC discovery tool
+```bash
+magellan help     
+Redfish-based BMC discovery tool with dynamic discovery features.
 
 Usage:
   magellan [flags]
@@ -227,21 +229,25 @@ Available Commands:
   help        Help about any command
   list        List information stored in cache from a scan
   login       Log in with identity provider for access token
+  power       Get and set node power states
   scan        Scan to discover BMC nodes on a network
+  secrets     Manage credentials for BMC nodes
+  send        Send collected node information to specified host.
   update      Update BMC node firmware
   version     Print version info and exit
 
 Flags:
       --access-token string   Set the access token
       --cache string          Set the scanning result cache path (default "/tmp/allend/magellan/assets.db")
-      --concurrency int       Set the number of concurrent processes (default -1)
+  -j, --concurrency int       Set the number of concurrent processes (default -1)
   -c, --config string         Set the config file path
-  -d, --debug                 Set to enable/disable debug messages
   -h, --help                  help for magellan
-      --timeout int           Set the timeout for requests (default 5)
-  -v, --verbose               Set to enable/disable verbose output
+      --log-file string       Set the path to store a log file
+  -l, --log-level LogLevel    Set the logger log-level (debug|info|warn|error|trace|disabled) (default info)
+  -t, --timeout int           Set the timeout for requests in seconds (default 5)
 
 Use "magellan [command] --help" for more information about a command.
+
 ```
 
 To start a network scan for BMC nodes, use the `scan` command. If the port is not specified, `magellan` will probe the common Redfish port 443 by default:
@@ -294,7 +300,7 @@ magellan send -F yaml -d @nodes.yaml https://example.openchami.cluster:8443
 
 This allows for modification of the data before making the request. However, be cautious as there is no data validation done before the request is made.
 
-Alternatively, we can pass the output of `collect` into `send` using pipes. The `--verbose` flag is currently required to do this.
+Alternatively, we can pass the output of `collect` into `send` using pipes. See the ["Modular Workflows"](#modular-workflows) section for more details.
 
 ```bash
 # collect and send data in YAML format
@@ -303,15 +309,14 @@ magellan collect -u $USERNAME -p $PASSWORD -v -F yaml | magellan send -F yaml ht
 # collect and send data using default JSON format and secret store (see below)
 export MASTER_KEY=mysecret
 magellan secrets store default $USERNAME:$PASSWORD
-magellan collect -v | magellan send https://example.openchami.cluster:8443
+magellan collect | magellan send https://example.openchami.cluster:8443
 ```
 
 This maintains the original behavior of passing the `--host` flag to `collect` with the added flexibility of having the intermediate step.
 
 > [!TIP]
 > If the `cache` flag is not set, `magellan` will use `/tmp/$USER/magellan.db` by default.
 
-
 > [!TIP]
 > The output of `collect` can be saved in separate directories using the `-O/--output-dir` flag. The output will be organized similar to below for the following command in YAML format:
 >
@@ -324,6 +329,29 @@ This maintains the original behavior of passing the `--host` flag to `collect` w
 >     └── 1747550498.yaml
 > ```
 
+#### Modular Workflows
+
+The `magellan` CLI commands can be ran in a single command or broken up to run different parts of the workflow without needing to write to the filesystem. 
+
+For example, the `scan`, `collect`, and `send` can be done in a single command.
+
+```bash
+# scan -> collect -> send
+magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | magellan collect -f json --show-output -i | magellan send https://smd.example.com
+```
+
+Alternatively, we can run `scan -> collect` and `collect -> send` parts separately if we're only interested in performing one part of the process.
+
+```bash
+# scan -> collect
+magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | magellan collect -f json --show-output -i
+
+# collect -> send
+magellan collect pdu x3000m0 x3000m1 -u admin -p initial0 | magellan send https://smd.example.com
+```
+
+> [!NOTE] See `magellan-send(1)` and `magellan-collect(1)` documentation for more info and examples.
+
 ### PDU Inventory Collection
 
 In addition to collecting Redfish inventory from BMCs, `magellan` can also collect inventory from Power Distribution Units (PDUs) that expose a JAWS-style API. The `collect` command has a `pdu` subcommand for this purpose.

cmd/collect.go

@@ -9,16 +9,20 @@ import (
 	"github.com/OpenCHAMI/magellan/internal/cache/sqlite"
 	"github.com/OpenCHAMI/magellan/internal/format"
 	magellan "github.com/OpenCHAMI/magellan/pkg"
-	"github.com/OpenCHAMI/magellan/pkg/auth"
 	"github.com/OpenCHAMI/magellan/pkg/bmc"
 	"github.com/OpenCHAMI/magellan/pkg/secrets"
 	"github.com/cznic/mathutil"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	_ "modernc.org/sqlite"
 )
 
-var collectOutputFormat format.DataFormat = format.FORMAT_JSON
+var (
+	collectInputFormat  format.DataFormat = format.FORMAT_JSON
+	collectOutputFormat format.DataFormat = format.FORMAT_JSON
+	collectDataArgs     []string
+)
 
 // The `collect` command fetches data from a collection of BMC nodes.
 // This command should be ran after the `scan` to find available hosts
@@ -35,21 +39,72 @@ var CollectCmd = &cobra.Command{
   // run a collect using secrets from the secrets manager
   export MASTER_KEY=$(magellan secrets generatekey)
   magellan secrets store $node_creds_json -f nodes.json
-  magellan collect -o nodes.yaml`,
+  magellan collect -o nodes.yaml
+
+  // Take the output of 'scan' and input directly into 'collect'
+  magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | ./magellan collect -f json --show-output -i
+
+  // Complete flow combined as a single line
+  magellan scan --subnet 172.18.0.0/24 --port 5000 -l info -i -F json | ./magellan collect -f json --show-output -i | magellan send https://smd.example.com
+  `,
 	Short: "Collect system information by interrogating BMC node",
 	Long:  "Send request(s) to a collection of hosts running Redfish services found stored from the 'scan' in cache.\nSee the 'scan' command on how to perform a scan.",
 	Run: func(cmd *cobra.Command, args []string) {
 		// get probe states stored in db from scan
-		scannedResults, err := sqlite.GetScannedAssets(cachePath)
-		if err != nil {
-			log.Error().Err(err).Msgf("failed to get scanned results from cache")
-		}
+		var (
+			scannedResults []magellan.RemoteAsset
+			err            error
+		)
+		if cachePath != "" {
+			scannedResults, err = sqlite.GetScannedAssets(cachePath)
+			if err != nil {
+				log.Error().Err(err).Msgf("failed to get scanned results from cache")
+			}
+		} else {
+			// unmarshal directly from standard input
+			for _, arg := range args {
+				var asset magellan.RemoteAsset
+				err = format.UnmarshalData([]byte(arg), &asset, collectInputFormat)
+				if err != nil {
+					log.Warn().Err(err).Msg("failed to unmarshal data from standard input")
+					continue
+				}
+				scannedResults = append(scannedResults, asset)
+			}
 
-		// try to load access token either from env var, file, or config if var not set
-		if accessToken == "" {
-			var err error
-			accessToken, err = auth.LoadAccessToken(tokenPath)
-			log.Warn().Err(err).Msgf("could not load access token")
+			// process input provided from the -d/--data flag
+			var inputData []map[string]any
+			temp := append(handleArgs(args), processDataArgs(sendDataArgs)...)
+			for _, data := range temp {
+				if data != nil {
+					inputData = append(inputData, data)
+				}
+			}
+			if len(inputData) == 0 {
+				log.Error().Msg("data required with standard input or -d/--data flag")
+				os.Exit(1)
+			}
+
+			// show the data that was just loaded as input
+			log.Debug().Int("endpoint_count", len(inputData)).Send()
+
+			// build and append target hosts from input data
+			for _, dataObject := range inputData {
+				// assert that we have certain values in data object
+				var (
+					asset    magellan.RemoteAsset
+					inputRaw []byte
+				)
+				inputRaw, err = format.MarshalData(dataObject, collectInputFormat)
+				if err != nil {
+					log.Error().Err(err).Msg("failed to marshal input data")
+				}
+				err = format.UnmarshalData(inputRaw, &asset, collectInputFormat)
+				if err != nil {
+					log.Error().Err(err).Msg("failed to unmarshal input data")
+				}
+				scannedResults = append(scannedResults, asset)
+			}
 		}
 
 		// set the minimum/maximum number of concurrent processes
@@ -115,14 +170,15 @@ var CollectCmd = &cobra.Command{
 
 		// set the collect parameters from CLI params
 		params := &magellan.CollectParams{
-			Timeout:     timeout,
-			Concurrency: concurrency,
-			OutputPath:  outputPath,
-			OutputDir:   outputDir,
-			Insecure:    insecure,
-			Format:      collectOutputFormat,
-			SecretStore: store,
-			BMCIDMap:    idMap,
+			Timeout:      timeout,
+			Concurrency:  concurrency,
+			OutputPath:   outputPath,
+			OutputDir:    outputDir,
+			Insecure:     insecure,
+			OutputFormat: collectOutputFormat,
+			InputFormat:  collectInputFormat,
+			SecretStore:  store,
+			BMCIDMap:     idMap,
 		}
 
 		// show all of the 'collect' parameters being set from CLI if verbose
@@ -153,13 +209,18 @@ func init() {
 	CollectCmd.Flags().StringVarP(&outputDir, "output-dir", "O", "", "Set the path to store collection data using HIVE partitioning")
 	CollectCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Skip TLS certificate verification during probe")
 	CollectCmd.Flags().BoolVar(&showOutput, "show", false, "Show the output of a collect run")
-	CollectCmd.Flags().VarP(&collectOutputFormat, "format", "F", "Set the default output data format (json|yaml; can be overridden by file extensions)")
+	CollectCmd.Flags().BoolVar(&showOutput, "show-output", false, "Show the output of a collect run")
+	CollectCmd.Flags().VarP(&collectInputFormat, "input-format", "f", "Set the default input data format (json|yaml)")
+	CollectCmd.Flags().VarP(&collectOutputFormat, "output-format", "F", "Set the default output data format (json|yaml; can be overridden by file extensions)")
 	CollectCmd.Flags().StringVarP(&idMap, "bmc-id-map", "m", "", "Set the BMC ID mapping from raw json data or use @<path> to specify a file path (json or yaml input)")
+	CollectCmd.Flags().StringArrayVarP(&collectDataArgs, "data", "d", []string{}, "Set the data as input for collect (prepend @ for files)")
 
+	// set mutually exclusive flags
 	CollectCmd.MarkFlagsMutuallyExclusive("output-file", "output-dir")
 
 	// register completion flag functions
-	checkRegisterFlagCompletionError(CollectCmd.RegisterFlagCompletionFunc("format", completionFormatData))
+	checkRegisterFlagCompletionError(CollectCmd.RegisterFlagCompletionFunc("input-format", completionFormatData))
+	checkRegisterFlagCompletionError(CollectCmd.RegisterFlagCompletionFunc("output-format", completionFormatData))
 
 	// bind flags to config properties
 	checkBindFlagError(viper.BindPFlag("collect.protocol", CollectCmd.Flags().Lookup("protocol")))

cmd/crawl.go

@@ -150,9 +150,10 @@ func init() {
 	CrawlCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Ignore SSL errors")
 	CrawlCmd.Flags().StringVarP(&secretsFile, "secrets-file", "f", "secrets.json", "Set path to the node secrets file")
 	CrawlCmd.Flags().BoolVar(&showOutput, "show", false, "Show the output of a crawl")
-	CrawlCmd.Flags().VarP(&crawlOutputFormat, "format", "F", "Set the output format (json|yaml)")
+	CrawlCmd.Flags().BoolVar(&showOutput, "show-output", false, "Show the output of a collect run")
+	CrawlCmd.Flags().VarP(&crawlOutputFormat, "output-format", "F", "Set the output format (json|yaml)")
 
-	checkRegisterFlagCompletionError(CrawlCmd.RegisterFlagCompletionFunc("format", completionFormatData))
+	checkRegisterFlagCompletionError(CrawlCmd.RegisterFlagCompletionFunc("output-format", completionFormatData))
 
 	checkBindFlagError(viper.BindPFlag("crawl.insecure", CrawlCmd.Flags().Lookup("insecure")))
 

cmd/list.go

@@ -67,10 +67,10 @@ var ListCmd = &cobra.Command{
 }
 
 func init() {
-	ListCmd.Flags().VarP(&listOutputFormat, "format", "F", "Set the output format (list|json|yaml)")
+	ListCmd.Flags().VarP(&listOutputFormat, "output-format", "F", "Set the output format (list|json|yaml)")
 	ListCmd.Flags().BoolVar(&showCache, "cache-info", false, "Show cache information and exit")
 
-	checkRegisterFlagCompletionError(ListCmd.RegisterFlagCompletionFunc("format", completionFormatData))
+	checkRegisterFlagCompletionError(ListCmd.RegisterFlagCompletionFunc("output-format", completionFormatData))
 
 	rootCmd.AddCommand(ListCmd)
 }

cmd/power.go

@@ -248,13 +248,13 @@ func init() {
 	PowerCmd.Flags().String("secrets-file", "", "Set path to the node secrets file")
 	PowerCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Ignore SSL errors")
 	PowerCmd.Flags().String("cacert", "", "Set the path to CA cert file (defaults to system CAs when blank)")
-	PowerCmd.Flags().VarP(&powerFormat, "format", "F", "Set the output format (json|yaml)")
+	PowerCmd.Flags().VarP(&powerFormat, "output-format", "F", "Set the output format (json|yaml)")
 
-	checkRegisterFlagCompletionError(PowerCmd.RegisterFlagCompletionFunc("format", completionFormatData))
+	checkRegisterFlagCompletionError(PowerCmd.RegisterFlagCompletionFunc("output-format", completionFormatData))
 
 	// Bind flags to config properties
 	checkBindFlagError(viper.BindPFlag("power.cacert", PowerCmd.Flags().Lookup("cacert")))
-	checkBindFlagError(viper.BindPFlag("power.format", PowerCmd.Flags().Lookup("format")))
+	checkBindFlagError(viper.BindPFlag("power.output-format", PowerCmd.Flags().Lookup("output-format")))
 	checkBindFlagError(viper.BindPFlags(PowerCmd.Flags()))
 
 	rootCmd.AddCommand(PowerCmd)

cmd/scan.go

@@ -36,8 +36,7 @@ var (
 // related to the implementation.
 var ScanCmd = &cobra.Command{
 	Use: "scan urls...",
-	Example: `
-  // assumes host https://10.0.0.101:443
+	Example: `  // assumes host https://10.0.0.101:443
   magellan scan 10.0.0.101 --insecure
 
   // assumes subnet using HTTPS and port 443 except for specified host
@@ -52,6 +51,9 @@ var ScanCmd = &cobra.Command{
   // assumes subnet using HTTPS and port 443 with specified CIDR
   magellan scan --subnet 10.0.0.0/16 -i
 
+  // same as above example but output is in JSON without caching
+  magellan scan --subnet 10.0.0.0/16 -i -f json --disable-cache
+
   // assumes subnet using HTTP and port 5000 similar to 192.168.0.0/16
   magellan scan --subnet 192.168.0.0 --protocol tcp --scheme https --port 5000 --subnet-mask 255.255.0.0
 
@@ -166,10 +168,14 @@ var ScanCmd = &cobra.Command{
 				} else {
 					fmt.Println(string(output))
 				}
+				// stop here so we don't write to cache if using JSON or YAML
+				return
 			default:
 				log.Error().Msgf("unknown format specified: %s. Please use 'db', 'json', or 'yaml'.", scanFormat)
 			}
 		}
+
+		// write to a cache file if not disabled at specified path
 		if !disableCache && cachePath != "" {
 			err := os.MkdirAll(path.Dir(cachePath), 0755)
 			if err != nil {
@@ -193,10 +199,14 @@ func init() {
 	ScanCmd.Flags().BoolVar(&disableProbing, "disable-probing", false, "Disable probing found assets for Redfish service(s) running on BMC nodes")
 	ScanCmd.Flags().BoolVar(&disableCache, "disable-cache", false, "Disable saving found assets to a cache database specified with 'cache' flag")
 	ScanCmd.Flags().BoolVarP(&insecure, "insecure", "i", false, "Skip TLS certificate verification during probe")
-	ScanCmd.Flags().VarP(&scanFormat, "format", "F", "Output format (json, yaml)")
+	ScanCmd.Flags().VarP(&scanFormat, "output-format", "F", "Output format (json, yaml)")
 	ScanCmd.Flags().StringVarP(&outputPath, "output", "o", "", "Output file path (for json/yaml formats)")
 	ScanCmd.Flags().StringSliceVar(&include, "include", []string{"bmcs"}, "Asset types to scan for (bmcs, pdus)")
 
+	// register completion flag functions
+	checkRegisterFlagCompletionError(ScanCmd.RegisterFlagCompletionFunc("output-format", completionFormatData))
+
+	// bind flags to config properties
 	checkBindFlagError(viper.BindPFlag("scan.ports", ScanCmd.Flags().Lookup("port")))
 	checkBindFlagError(viper.BindPFlag("scan.scheme", ScanCmd.Flags().Lookup("scheme")))
 	checkBindFlagError(viper.BindPFlag("scan.protocol", ScanCmd.Flags().Lookup("protocol")))

cmd/send.go

@@ -141,11 +141,11 @@ var sendCmd = &cobra.Command{
 
 func init() {
 	sendCmd.Flags().StringArrayVarP(&sendDataArgs, "data", "d", []string{}, "Set the data to send to specified host (prepend @ for files)")
-	sendCmd.Flags().VarP(&sendInputFormat, "format", "F", "Set the default data input format (json|yaml) can be overridden by file extension")
-	sendCmd.Flags().BoolVarP(&forceUpdate, "force-update", "f", false, "Set flag to force update data sent to SMD")
+	sendCmd.Flags().VarP(&sendInputFormat, "input-format", "f", "Set the default data input format (json|yaml) can be overridden by file extension")
+	sendCmd.Flags().BoolVar(&forceUpdate, "force-update", false, "Set flag to force update data sent to SMD")
 	sendCmd.Flags().StringVar(&cacertPath, "cacert", "", "Set the path to CA cert file (defaults to system CAs when blank)")
 
-	checkRegisterFlagCompletionError(sendCmd.RegisterFlagCompletionFunc("format", completionFormatData))
+	checkRegisterFlagCompletionError(sendCmd.RegisterFlagCompletionFunc("input-format", completionFormatData))
 	rootCmd.AddCommand(sendCmd)
 }