Feat/scaleset label change

charms/garm/src/charm.py

@@ -34,7 +34,7 @@
     CredentialSpec,
     GithubReconciler,
 )
-from scaleset_reconciler import ScalesetReconciler, ScalesetSpec
+from scaleset_reconciler import ScalesetProgress, ScalesetReconciler, ScalesetSpec
 
 logger = logging.getLogger(__name__)
 
@@ -48,6 +48,13 @@
 GARM_PORT: typing.Final[int] = 8080
 GARM_LISTEN_ADDRESS: typing.Final[str] = "0.0.0.0"
 _DB_PASSPHRASE_LENGTH: typing.Final[int] = 32
+# Juju truncates long statuses; name a couple of scalesets and count the rest.
+_MAX_DRAINING_IN_STATUS: typing.Final[int] = 2
+# The phases a scaleset replacement passes through, in order. Named here rather than
+# spelled inline so the status vocabulary is greppable; not an enum, because the
+# draining phase carries the runner count and so is formatted, not selected.
+_PHASE_CREATING: typing.Final[str] = "creating replacement"
+_PHASE_AWAITING_DELETION: typing.Final[str] = "awaiting deletion"
 
 GARM_CONFIG_VERSION: typing.Final[str] = "1"
 
@@ -116,6 +123,49 @@ def _parse_pre_install_scripts(raw: str) -> dict[str, str]:
     return {}
 
 
+def _scaleset_replacement_phase(progress: ScalesetProgress) -> str:
+    """Describe where one scaleset replacement has got to.
+
+    Args:
+        progress: A replacement still in flight.
+
+    Returns:
+        The phase, in the operator's terms. The runner count is only reported once
+        the labels have been handed over, since before that nothing is draining yet;
+        with the count at zero the scaleset has drained and GARM has yet to accept
+        its deletion, which is a distinct thing to be waiting on.
+    """
+    if not progress.handed_over:
+        return _PHASE_CREATING
+    runners = progress.remaining_runners
+    if not runners:
+        return _PHASE_AWAITING_DELETION
+    return f"draining {runners} runner{'' if runners == 1 else 's'}"
+
+
+def _scaleset_replacement_status(replacing: list[ScalesetProgress]) -> str:
+    """Summarise in-progress scaleset replacements for the unit status.
+
+    Args:
+        replacing: Scaleset replacements still in flight.
+
+    Returns:
+        A status message naming at most two scalesets, so it stays readable when
+        several are replaced at once. Each is named by the logical name the operator
+        configured — the live names carry a label hash the operator never chose — and
+        the live names are logged in full by the reconciler.
+    """
+    shown = [
+        f"{progress.logical_name} -> {progress.replacement_name}"
+        f" ({_scaleset_replacement_phase(progress)})"
+        for progress in replacing[:_MAX_DRAINING_IN_STATUS]
+    ]
+    remainder = len(replacing) - len(shown)
+    if remainder > 0:
+        shown.append(f"+{remainder} more")
+    return f"Replacing scaleset {', '.join(shown)}"
+
+
 class GarmCharm(paas_charm.go.Charm):
     """GARM charm — manages the GARM service via Pebble."""
 
@@ -459,8 +509,7 @@ def _get_configurator_provider_configs(
             )
 
         logger.info(
-            "GARM configurator provider data: relation_unit_count=%d "
-            "configured_provider_count=%d",
+            "GARM configurator provider data: relation_unit_count=%d configured_provider_count=%d",
             len(relation.units),
             len(configs),
         )
@@ -694,8 +743,21 @@ def _reconcile_runners(self) -> None:
             GithubReconciler(auth_client).reconcile(self._build_desired_credentials())
             EntityReconciler(auth_client).reconcile(charm_state.desired_entities)
             template_id = _apply_garm_template(auth_client, charm_state.ssh_debug_connections)
-            ScalesetReconciler(auth_client).reconcile(self._build_desired_scalesets(template_id))
-            self.update_app_and_unit_status(ops.ActiveStatus())
+            replacing = ScalesetReconciler(auth_client).reconcile(
+                self._build_desired_scalesets(template_id)
+            )
+            # A label change recreates the scaleset and drains the old one, which
+            # outlives this hook: report progress and let update-status converge it.
+            # Active, not maintenance: every label is served throughout the drain, by
+            # the replacement or the predecessor, so nothing is degraded. A drain can
+            # run to DRAIN_DEADLINE, and blocking `juju wait-for` on hours of healthy
+            # background convergence would be wrong.
+            if replacing:
+                self.update_app_and_unit_status(
+                    ops.ActiveStatus(_scaleset_replacement_status(replacing))
+                )
+            else:
+                self.update_app_and_unit_status(ops.ActiveStatus())
         except CharmedTemplateError as exc:
             logger.warning("GARM charmed template error during reconcile: %s", exc)
             self.update_app_and_unit_status(ops.WaitingStatus(str(exc)))

charms/garm/src/garm_api.py

@@ -14,6 +14,7 @@
 from garm_client.api.controller_info_api import ControllerInfoApi
 from garm_client.api.credentials_api import CredentialsApi
 from garm_client.api.first_run_api import FirstRunApi
+from garm_client.api.instances_api import InstancesApi
 from garm_client.api.login_api import LoginApi
 from garm_client.api.organizations_api import OrganizationsApi
 from garm_client.api.providers_api import ProvidersApi
@@ -29,6 +30,7 @@
 from garm_client.models.create_scale_set_params import CreateScaleSetParams
 from garm_client.models.create_template_params import CreateTemplateParams
 from garm_client.models.forge_credentials import ForgeCredentials
+from garm_client.models.instance import Instance
 from garm_client.models.new_user_params import NewUserParams
 from garm_client.models.organization import Organization
 from garm_client.models.password_login_params import PasswordLoginParams
@@ -529,6 +531,35 @@ def list_scalesets(self) -> list[ScaleSet]:
             except urllib3.exceptions.HTTPError as exc:
                 raise GarmConnectionError(f"GARM connection error: {exc}") from exc
 
+    def list_scaleset_instances(self, scaleset_id: int) -> list[Instance]:
+        """List the runner instances currently backing a scaleset.
+
+        Args:
+            scaleset_id: GARM scaleset id.
+
+        Returns:
+            List of Instance model objects, empty when the scaleset has no runners.
+
+        Raises:
+            GarmApiError: On API error.
+        """
+        with self._api_client() as client:
+            try:
+                return (
+                    InstancesApi(api_client=client).list_scale_set_instances(
+                        scaleset_id=str(scaleset_id),
+                        _request_timeout=_REQUEST_TIMEOUT,
+                    )
+                    or []
+                )
+            except ApiException as exc:
+                raise GarmApiError(
+                    f"Failed to list instances for scaleset {scaleset_id} "
+                    f"({exc.status}): {exc.body}"
+                ) from exc
+            except urllib3.exceptions.HTTPError as exc:
+                raise GarmConnectionError(f"GARM connection error: {exc}") from exc
+
     def find_org_id(self, org_name: str) -> str | None:
         """Find a GARM organization's UUID by name.
 

charms/garm/src/garm_client_README.md

@@ -29,24 +29,21 @@ In your own code, to use this library to connect and interact with garm-client,
 you can run the following:
 
 ```python
-
 import garm_client
 from garm_client.rest import ApiException
 from pprint import pprint
 
 # Defining the host is optional and defaults to /api/v1
 # See configuration.py for a list of all supported configuration parameters.
-configuration = garm_client.Configuration(
-    host = "/api/v1"
-)
+configuration = garm_client.Configuration(host="/api/v1")
 
 # The client must configure the authentication and authorization parameters
 # in accordance with the API server security policy.
 # Examples for each auth method are provided below, use the example that
 # satisfies your auth use case.
 
 # Configure API key authorization: Bearer
-configuration.api_key['Bearer'] = os.environ["API_KEY"]
+configuration.api_key["Bearer"] = os.environ["API_KEY"]
 
 # Uncomment below to setup prefix (e.g. Bearer) for API key, if needed
 # configuration.api_key_prefix['Bearer'] = 'Bearer'
@@ -64,7 +61,6 @@ with garm_client.ApiClient(configuration) as api_client:
         pprint(api_response)
     except ApiException as e:
         print("Exception when calling ControllerApi->force_tools_sync: %s\n" % e)
-
 ```
 
 ## Documentation for API Endpoints