diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9d9e1584c..1aab8c33e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,6 +28,23 @@ updates: patterns: - "*" + - package-ecosystem: "pip" + directory: "/doc" + ignore: + # Pinned intentionally: myst-parser v5.0.0 causes version conflicts. + - dependency-name: "myst-parser" + versions: [">=5.0.0"] + labels: [] + schedule: + interval: "weekly" + target-branch: "main" + cooldown: + default-days: 7 + groups: + pip: + patterns: + - "*" + - package-ecosystem: "github-actions" directories: - "/" @@ -55,3 +72,20 @@ updates: gomod: patterns: - "*" + + - package-ecosystem: "pip" + directory: "/doc" + ignore: + # Pinned intentionally: myst-parser v5.0.0 causes version conflicts. + - dependency-name: "myst-parser" + versions: [">=5.0.0"] + labels: [] + schedule: + interval: "weekly" + target-branch: "v2-edge" + cooldown: + default-days: 7 + groups: + pip: + patterns: + - "*" diff --git a/api/status.go b/api/status.go index d486ad2da..6fc119ff1 100644 --- a/api/status.go +++ b/api/status.go @@ -50,7 +50,7 @@ func statusGet(sh *service.Handler) endpointHandler { err = cluster.Query(r.Context(), true, func(ctx context.Context, c *microClient.Client) error { memberStatuses, err := client.GetStatus(ctx, c) if err != nil { - logger.Error("Failed to get status for cluster member", logger.Ctx{"error": err, "address": c.URL()}) + logger.Error("Failed to get status for cluster member", logger.Ctx{"err": err, "address": c.URL()}) return nil } @@ -90,7 +90,7 @@ func statusGet(sh *service.Handler) endpointHandler { case types.LXD: clusterMembers, err := lxdStatus(r.Context(), s) if err != nil { - logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name}) + logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name, "err": err}) } statusMu.Lock() @@ -99,7 +99,7 @@ func statusGet(sh *service.Handler) endpointHandler { case types.MicroCeph: clusterMembers, osds, cephServices, err := cephStatus(r.Context(), s) if err != nil { - logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name}) + logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name, "err": err}) } status.OSDs = osds @@ -111,7 +111,7 @@ func statusGet(sh *service.Handler) endpointHandler { case types.MicroOVN: clusterMembers, ovnServices, err := ovnStatus(r.Context(), s) if err != nil { - logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name}) + logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name, "err": err}) } status.OVNServices = ovnServices @@ -127,7 +127,7 @@ func statusGet(sh *service.Handler) endpointHandler { clusterMembers, err := microStatus(r.Context(), microClient, s) if err != nil { - logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name}) + logger.Error("Failed to get service status", logger.Ctx{"type": s.Type(), "name": sh.Name, "err": err}) } statusMu.Lock() diff --git a/cmd/microcloud/main.go b/cmd/microcloud/main.go index 8d53f2bb5..fa4853d43 100644 --- a/cmd/microcloud/main.go +++ b/cmd/microcloud/main.go @@ -37,7 +37,7 @@ func main() { asker, err := setupAsker(ctx) if err != nil { - fmt.Println(err.Error()) + fmt.Fprintf(os.Stderr, "Failed setting up asker: %v\n", err) os.Exit(1) } @@ -62,6 +62,14 @@ func main() { app.SetVersionTemplate("{{.Version}}\n") + // Don't display the --state-dir flag in the help output. + // It is used by the snaps "microcloud" wrapper command but never by the user directly. + err = app.PersistentFlags().MarkHidden("state-dir") + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot hide --state-dir flag: %v\n", err) + os.Exit(1) + } + var cmdInit = cmdInit{common: &commonCmd} app.AddCommand(cmdInit.command()) diff --git a/cmd/microcloud/main_init.go b/cmd/microcloud/main_init.go index fc4b87588..c576cac31 100644 --- a/cmd/microcloud/main_init.go +++ b/cmd/microcloud/main_init.go @@ -885,8 +885,18 @@ func (c *initConfig) setupCluster(s *service.Handler) error { fmt.Println("Configuring cluster-wide devices ...") - var ovnConfig string - if s.Services[types.MicroOVN] != nil { + // Update LXD's global config. + server, _, err := lxdClient.GetServer() + if err != nil { + return err + } + + config := make(map[string]string) + + // LXD can dynamically determine the OVN northbound DB connection string from MicroOVN's `ovn.env` file. + // This feature was added with the ovn_dynamic_northbound_connection API extension. + // Only set the connection string in case of an older LXD. + if s.Services[types.MicroOVN] != nil && !lxdClient.HasExtension("ovn_dynamic_northbound_connection") { serviceOVN := s.Services[types.MicroOVN].(*service.OVNService) services, err := serviceOVN.GetServices(context.Background()) @@ -911,14 +921,7 @@ func (c *initConfig) setupCluster(s *service.Handler) error { } } - ovnConfig = strings.Join(conns, ",") - } - - config := map[string]string{"network.ovn.northbound_connection": ovnConfig} - // Update LXD's global config. - server, _, err := lxdClient.GetServer() - if err != nil { - return err + config["network.ovn.northbound_connection"] = strings.Join(conns, ",") } newServer := server.Writable() diff --git a/cmd/microcloud/preseed.go b/cmd/microcloud/preseed.go index 920df5034..4fc4397ac 100644 --- a/cmd/microcloud/preseed.go +++ b/cmd/microcloud/preseed.go @@ -918,13 +918,6 @@ func (p *Preseed) Parse(s *service.Handler, c *initConfig, installedServices map directLocal = sys.Storage.Local directCeph = sys.Storage.Ceph } - - for _, disk := range directCeph { - _, err := os.Stat(disk.Path) - if err != nil { - return nil, fmt.Errorf("Failed to find specified disk path: %w", err) - } - } } // Setup directly specified disks for ZFS pool. diff --git a/doc/how-to/initialize.md b/doc/how-to/initialize.md index 9c9076ef7..9dc1881c7 100644 --- a/doc/how-to/initialize.md +++ b/doc/how-to/initialize.md @@ -182,13 +182,7 @@ If you want to automate the initialization process, you can provide a preseed co cat | microcloud preseed Make sure to distribute and run the same preseed configuration on all systems that should be part of the MicroCloud. - -The preseed YAML file must use the following syntax: - -```{literalinclude} preseed.yaml -:language: YAML -:emphasize-lines: 1-4,7-10,13-14,17-19,22,25-27,30-35,63-66,72,79-87 -``` +See the {ref}`full reference ` for possible configuration options or the minimal example below. ### Minimal preseed using multicast discovery @@ -239,6 +233,40 @@ ovn: If you initialized MicroCloud without local storage _and_ with CephFS storage, continue to the section below to complete your initialization. ``` +### Use storage disk filters + +You may not know the exact disk paths used for local and remote storage when crafting the preseed file. +In such cases, you can add disk filters for local and remote storage configuration. +By using those filters, you can narrow down the list of available disks to the ones eligible based on the given rules. + +For example you might want to use all disks for local storage which are of type `nvme` and use a model description ``: + +```yaml +storage: + local: + - find: type == nvme && model == "" +``` + +As another example, you can filter for remote (Ceph) storage disks with size greater than 1TiB and model description ``, and ensure there are at least six disks (maximum eight) selected across all members: + +```yaml +storage: + ceph: + - find: size > 1TiB && model == "" + find_min: 6 + find_max: 8 +``` + +See the {ref}`list of filters ` for a full reference. + +```{admonition} Finding the right filters +:class: note +If you want to see the actual filter values for your system(s) for further refinement of the preseed file, you can run `lxc query /1.0/resources | jq .storage.disks` on any of the MicroCloud members prior to initialization. + +The response will show all the disks available to MicroCloud on this member. +Repeat the command on every member for a full list of disks across the cluster. +``` + (howto-initialize-images-backups)= ## Configure `backups_volume` and `images_volume` diff --git a/doc/how-to/member_add.md b/doc/how-to/member_add.md index 50532ed36..56b8c2225 100644 --- a/doc/how-to/member_add.md +++ b/doc/how-to/member_add.md @@ -31,13 +31,7 @@ In the list of systems, include only the new machine and set either `initiator` that is already part of the MicroCloud. Distribute and run the same preseed configuration on both the machine being added, and the cluster member used for the `initiator` or `initiator_address`. - -The preseed YAML file must use the following syntax: - -```{literalinclude} preseed.yaml -:language: YAML -:emphasize-lines: 1-4,7-10,13-14,17-19,22,25-27,30-35,63-66,72,79-88 -``` +See the {ref}`full reference ` for possible configuration options or the minimal example below. ### Minimal preseed using multicast discovery diff --git a/doc/reference/index.md b/doc/reference/index.md index cff299af6..69a92196d 100644 --- a/doc/reference/index.md +++ b/doc/reference/index.md @@ -19,6 +19,16 @@ Consult this command reference to work with MicroCloud through the CLI. /reference/commands ``` +## Preseed + +Consult this preseed reference for a detailed explanation of the various configuration options. + +```{toctree} +:maxdepth: 1 + +/reference/preseed +``` + ## Requirements and releases ```{toctree} diff --git a/doc/reference/preseed.md b/doc/reference/preseed.md new file mode 100644 index 000000000..78a35d9b5 --- /dev/null +++ b/doc/reference/preseed.md @@ -0,0 +1,100 @@ +(ref-preseed)= +# Preseed configuration options + +MicroCloud preseed allows the unattended (non-interactive) deployment of a cluster using a pre-configured file. +See below for detailed descriptions of the main building blocks of the file, followed by a {ref}`full configuration example ` file. + +(ref-preseed-filters)= +## Storage disk filters + +Explicitly setting the storage disks per system under `systems.[*].storage` is optional. +Use filters if the exact disk paths are unknown when crafting the preseed file. +This also makes the preseed file generic enough to be usable across various MicroCloud deployments. + +Filters allow MicroCloud to make a selection from a list of all disks available on the systems. +These filters correspond to the YAML field names of the disk resources returned from LXD's `/1.0/resources` endpoint. + +The following table lists all of the available filters: + +```{table} +:align: left + +| Filter | Example | +| ------------------ | ------------------------------------------- | +| `id` | `nvme0n1` | +| `device` | `259:0` | +| `model` | `` | +| `type` | `nvme` | +| `read_only` | `false` | +| `mounted` | `false` | +| `size` | `1024209543168` (size of the disk in bytes) | +| `removable` | `false` | +| `wwn` | `eui.00xxxxxxxxxxxxxx` | +| `numa_node` | `0` | +| `device_path` | `pci-0000:04:00.0-nvme-1` | +| `block_size` | `512` | +| `firmware_version` | `4L2XXXXX` | +| `rpm` | `0` | +| `serial` | `S7XKXXXXXXXXXX` | +| `device_id` | `nvme-eui.00xxxxxxxxxxxxxx` | +| `pci_address` | `0000:04:00.0` | +| `used_by` | `bcache` | +``` + +When using the `size` filter, its value can be compared against a user-defined number using byte suffixes in either units of 1000 or 1024: + +`B`, `kB`, `MB`, `GB`, `TB`, `EB`, `KiB`, `MiB`, `GiB`, `TiB`, `PiB`, `EiB` + +### Filter operands + +All filters can use the following operands to compare against defined values: + +`&&`, `||`, `<`, `>`, `<=`, `>=`, `==`, `!=`, `!` + +Furthermore the following restrictions apply: + +* Filters are checked in order of appearance +* String values must not be in quotes unless the string contains a space +* Single quotes are fine, but double quotes must be escaped + + +Multiple filters can be added to a single section: + +```yaml +storage: + ceph: + - find: + - find: +``` + +### Limit filtered disks + +In addition to finding disks by filter, the minimum and maximum number of disks can also be specified. +For this, the `find_min` and `find_max` settings can be added to the relevant section: + +```yaml +storage: + ceph: + - find: + find_min: 1 + find_max: 2 +``` + +The example above will make sure that the filters select at least one, but not more than two, disks for remote (Ceph) storage. + +```{note} +For local storage there can only ever be one disk per system. +If the filters return more than one disk, only one of them will be used. + +For remote storage the filters apply for all disks across all systems. +``` + +(ref-preseed-full-configuration-example)= +## Full configuration example + +The preseed YAML file must use the following syntax: + +```{literalinclude} preseed.yaml +:language: YAML +:emphasize-lines: 1-4,7-10,13-14,17-19,22,25-27,30-35,63-66,72,79-88 +``` diff --git a/doc/how-to/preseed.yaml b/doc/reference/preseed.yaml similarity index 100% rename from doc/how-to/preseed.yaml rename to doc/reference/preseed.yaml diff --git a/test/e2e/README.md b/test/e2e/README.md index 8973f5527..28b007fa2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -105,3 +105,11 @@ EVACUATION_COUNTS=0 ./run mc # do multuple rolling reboots/evacuation tests: EVACUATION_COUNTS=5 ./run mc ``` + +### HTML report + +Passing `--report` to the `run` script will generate a brief HTML report upon completion (whether the tests pass or fail). The report file is created in the current directory. + +```sh +./run --report mc +``` diff --git a/test/e2e/run b/test/e2e/run index e6ee35d24..4f12e2f04 100755 --- a/test/e2e/run +++ b/test/e2e/run @@ -202,6 +202,11 @@ instanceList() { lxc list --project e2e-testing -f csv -c n "${REMOTE}:" } +instanceListByMember() { + local member="${1}" + lxc list --project e2e-testing -f csv -c n "${REMOTE}:" location="${member}" +} + clusterMembers() { local state="${1}" [ "${state}" = "ALL" ] && state=".*" @@ -284,14 +289,23 @@ evacuation() { fi lxc cluster restore --project e2e-testing --force "${REMOTE}:${member}" - done - echo -n "Allow time for stopped instances to start back again " - for _ in $(seq 20); do - echo -n "." - sleep 1 + echo -n "Allow time for stopped instances on ${member} to start back again and have an IPv4 address" + for instance in $(instanceListByMember "${member}"); do + attempts=0 + while [ -z "$(getIPv4 "${instance}")" ]; do + echo -n "." + sleep 1 + attempts=$((attempts + 1)) + if [ "${attempts}" -ge 30 ]; then + echo " FAILED" + exit 1 + fi + done + done + + echo " DONE" done - echo " DONE" } diff --git a/test/includes/microcloud.sh b/test/includes/microcloud.sh index 401f8c595..99ac9cd59 100644 --- a/test/includes/microcloud.sh +++ b/test/includes/microcloud.sh @@ -550,16 +550,19 @@ validate_system_lxd_ovn() { echo " ${name} Validating OVN network" - num_conns=3 - if [ "${num_peers}" -lt "${num_conns}" ]; then - num_conns="${num_peers}" - fi + # Check if the connection string is set correctly. + if ! check_api_extension ovn_dynamic_northbound_connection "${name}"; then + num_conns=3 + if [ "${num_peers}" -lt "${num_conns}" ]; then + num_conns="${num_peers}" + fi - [ "$(lxc config get network.ovn.northbound_connection --target "${name}" | sed -e 's/,/\n/g' | wc -l)" = "${num_conns}" ] + [ "$(lxc config get network.ovn.northbound_connection --target "${name}" | sed -e 's/,/\n/g' | wc -l)" = "${num_conns}" ] - # Make sure there's no empty addresses. - ! lxc config get network.ovn.northbound_connection --target "${name}" | sed -e 's/,/\n/g' | grep -q '^ssl:$' || false - ! lxc config get network.ovn.northbound_connection --target "${name}" | sed -e 's/,/\n/g' | grep -q '^ssl::' || false + # Make sure there's no empty addresses. + ! lxc config get network.ovn.northbound_connection --target "${name}" | sed -e 's/,/\n/g' | grep -q '^ssl:$' || false + ! lxc config get network.ovn.northbound_connection --target "${name}" | sed -e 's/,/\n/g' | grep -q '^ssl::' || false + fi # Check that the created UPLINK network has the right DNS servers. if [ -n "${dns_namesersers}" ] ; then