diff --git a/charms/garm/src/charm.py b/charms/garm/src/charm.py index 855c3b58..07b83b99 100755 --- a/charms/garm/src/charm.py +++ b/charms/garm/src/charm.py @@ -34,7 +34,7 @@ CredentialSpec, GithubReconciler, ) -from scaleset_reconciler import ScalesetReconciler, ScalesetSpec +from scaleset_reconciler import ScalesetProgress, ScalesetReconciler, ScalesetSpec logger = logging.getLogger(__name__) @@ -48,6 +48,13 @@ GARM_PORT: typing.Final[int] = 8080 GARM_LISTEN_ADDRESS: typing.Final[str] = "0.0.0.0" _DB_PASSPHRASE_LENGTH: typing.Final[int] = 32 +# Juju truncates long statuses; name a couple of scalesets and count the rest. +_MAX_DRAINING_IN_STATUS: typing.Final[int] = 2 +# The phases a scaleset replacement passes through, in order. Named here rather than +# spelled inline so the status vocabulary is greppable; not an enum, because the +# draining phase carries the runner count and so is formatted, not selected. +_PHASE_CREATING: typing.Final[str] = "creating replacement" +_PHASE_AWAITING_DELETION: typing.Final[str] = "awaiting deletion" GARM_CONFIG_VERSION: typing.Final[str] = "1" @@ -116,6 +123,49 @@ def _parse_pre_install_scripts(raw: str) -> dict[str, str]: return {} +def _scaleset_replacement_phase(progress: ScalesetProgress) -> str: + """Describe where one scaleset replacement has got to. + + Args: + progress: A replacement still in flight. + + Returns: + The phase, in the operator's terms. The runner count is only reported once + the labels have been handed over, since before that nothing is draining yet; + with the count at zero the scaleset has drained and GARM has yet to accept + its deletion, which is a distinct thing to be waiting on. + """ + if not progress.handed_over: + return _PHASE_CREATING + runners = progress.remaining_runners + if not runners: + return _PHASE_AWAITING_DELETION + return f"draining {runners} runner{'' if runners == 1 else 's'}" + + +def _scaleset_replacement_status(replacing: list[ScalesetProgress]) -> str: + """Summarise in-progress scaleset replacements for the unit status. + + Args: + replacing: Scaleset replacements still in flight. + + Returns: + A status message naming at most two scalesets, so it stays readable when + several are replaced at once. Each is named by the logical name the operator + configured — the live names carry a label hash the operator never chose — and + the live names are logged in full by the reconciler. + """ + shown = [ + f"{progress.logical_name} -> {progress.replacement_name}" + f" ({_scaleset_replacement_phase(progress)})" + for progress in replacing[:_MAX_DRAINING_IN_STATUS] + ] + remainder = len(replacing) - len(shown) + if remainder > 0: + shown.append(f"+{remainder} more") + return f"Replacing scaleset {', '.join(shown)}" + + class GarmCharm(paas_charm.go.Charm): """GARM charm — manages the GARM service via Pebble.""" @@ -459,8 +509,7 @@ def _get_configurator_provider_configs( ) logger.info( - "GARM configurator provider data: relation_unit_count=%d " - "configured_provider_count=%d", + "GARM configurator provider data: relation_unit_count=%d configured_provider_count=%d", len(relation.units), len(configs), ) @@ -694,8 +743,21 @@ def _reconcile_runners(self) -> None: GithubReconciler(auth_client).reconcile(self._build_desired_credentials()) EntityReconciler(auth_client).reconcile(charm_state.desired_entities) template_id = _apply_garm_template(auth_client, charm_state.ssh_debug_connections) - ScalesetReconciler(auth_client).reconcile(self._build_desired_scalesets(template_id)) - self.update_app_and_unit_status(ops.ActiveStatus()) + replacing = ScalesetReconciler(auth_client).reconcile( + self._build_desired_scalesets(template_id) + ) + # A label change recreates the scaleset and drains the old one, which + # outlives this hook: report progress and let update-status converge it. + # Active, not maintenance: every label is served throughout the drain, by + # the replacement or the predecessor, so nothing is degraded. A drain can + # run to DRAIN_DEADLINE, and blocking `juju wait-for` on hours of healthy + # background convergence would be wrong. + if replacing: + self.update_app_and_unit_status( + ops.ActiveStatus(_scaleset_replacement_status(replacing)) + ) + else: + self.update_app_and_unit_status(ops.ActiveStatus()) except CharmedTemplateError as exc: logger.warning("GARM charmed template error during reconcile: %s", exc) self.update_app_and_unit_status(ops.WaitingStatus(str(exc))) diff --git a/charms/garm/src/garm_api.py b/charms/garm/src/garm_api.py index d321864b..2dd6c7e5 100644 --- a/charms/garm/src/garm_api.py +++ b/charms/garm/src/garm_api.py @@ -14,6 +14,7 @@ from garm_client.api.controller_info_api import ControllerInfoApi from garm_client.api.credentials_api import CredentialsApi from garm_client.api.first_run_api import FirstRunApi +from garm_client.api.instances_api import InstancesApi from garm_client.api.login_api import LoginApi from garm_client.api.organizations_api import OrganizationsApi from garm_client.api.providers_api import ProvidersApi @@ -29,6 +30,7 @@ from garm_client.models.create_scale_set_params import CreateScaleSetParams from garm_client.models.create_template_params import CreateTemplateParams from garm_client.models.forge_credentials import ForgeCredentials +from garm_client.models.instance import Instance from garm_client.models.new_user_params import NewUserParams from garm_client.models.organization import Organization from garm_client.models.password_login_params import PasswordLoginParams @@ -529,6 +531,35 @@ def list_scalesets(self) -> list[ScaleSet]: except urllib3.exceptions.HTTPError as exc: raise GarmConnectionError(f"GARM connection error: {exc}") from exc + def list_scaleset_instances(self, scaleset_id: int) -> list[Instance]: + """List the runner instances currently backing a scaleset. + + Args: + scaleset_id: GARM scaleset id. + + Returns: + List of Instance model objects, empty when the scaleset has no runners. + + Raises: + GarmApiError: On API error. + """ + with self._api_client() as client: + try: + return ( + InstancesApi(api_client=client).list_scale_set_instances( + scaleset_id=str(scaleset_id), + _request_timeout=_REQUEST_TIMEOUT, + ) + or [] + ) + except ApiException as exc: + raise GarmApiError( + f"Failed to list instances for scaleset {scaleset_id} " + f"({exc.status}): {exc.body}" + ) from exc + except urllib3.exceptions.HTTPError as exc: + raise GarmConnectionError(f"GARM connection error: {exc}") from exc + def find_org_id(self, org_name: str) -> str | None: """Find a GARM organization's UUID by name. diff --git a/charms/garm/src/garm_client_README.md b/charms/garm/src/garm_client_README.md index d6156ebd..d570d381 100644 --- a/charms/garm/src/garm_client_README.md +++ b/charms/garm/src/garm_client_README.md @@ -29,16 +29,13 @@ In your own code, to use this library to connect and interact with garm-client, you can run the following: ```python - import garm_client from garm_client.rest import ApiException from pprint import pprint # Defining the host is optional and defaults to /api/v1 # See configuration.py for a list of all supported configuration parameters. -configuration = garm_client.Configuration( - host = "/api/v1" -) +configuration = garm_client.Configuration(host="/api/v1") # The client must configure the authentication and authorization parameters # in accordance with the API server security policy. @@ -46,7 +43,7 @@ configuration = garm_client.Configuration( # satisfies your auth use case. # Configure API key authorization: Bearer -configuration.api_key['Bearer'] = os.environ["API_KEY"] +configuration.api_key["Bearer"] = os.environ["API_KEY"] # Uncomment below to setup prefix (e.g. Bearer) for API key, if needed # configuration.api_key_prefix['Bearer'] = 'Bearer' @@ -64,7 +61,6 @@ with garm_client.ApiClient(configuration) as api_client: pprint(api_response) except ApiException as e: print("Exception when calling ControllerApi->force_tools_sync: %s\n" % e) - ``` ## Documentation for API Endpoints diff --git a/charms/garm/src/scaleset_reconciler.py b/charms/garm/src/scaleset_reconciler.py index 2f616d60..d484d95b 100644 --- a/charms/garm/src/scaleset_reconciler.py +++ b/charms/garm/src/scaleset_reconciler.py @@ -5,11 +5,14 @@ """Scaleset reconciler: diffs desired vs observed GARM scalesets and applies changes.""" import base64 +import datetime +import hashlib import logging +import re from dataclasses import dataclass, field from charm_state import RunnerConfig -from garm_api import GarmApiError, GarmAuthenticatedClient +from garm_api import GarmApiError, GarmAuthenticatedClient, GarmConnectionError from garm_client.models.create_scale_set_params import CreateScaleSetParams from garm_client.models.scale_set import ScaleSet from garm_client.models.template import Template @@ -27,6 +30,27 @@ # operator-supplied script runs. APROXY_SCRIPT_NAME = "00-aproxy" +LABEL_HASH_LENGTH = 8 + +# GitHub rejects over-long scale set names; keep the generated name inside a +# conservative bound by truncating the operator-supplied part, never the hash. +MAX_SCALESET_NAME_LENGTH = 64 + +# Past GitHub's 6h job cap a remaining runner is stuck, not busy: stop gating on the +# count and retry the delete, which GARM rejects while runners are active. +DRAIN_DEADLINE = datetime.timedelta(hours=7) + + +@dataclass(frozen=True) +class ScalesetProgress: + """A scaleset replacement still in flight: the generation it replaced is not gone yet.""" + + logical_name: str + retiring_name: str + replacement_name: str + remaining_runners: int + handed_over: bool = True + @dataclass class ScalesetSpec: @@ -49,6 +73,107 @@ class ScalesetSpec: runner_config: RunnerConfig = field(default_factory=RunnerConfig) +def _name_base(logical_name: str) -> str: + """Return the part of a live scaleset name that precedes the label hash. + + Args: + logical_name: The scaleset name the operator configured. + + Returns: + The name itself when it fits, else a truncation ending in a hash of the full + name — two long names sharing a prefix would otherwise collapse onto one live + scaleset and fight over it on every reconcile. + """ + limit = MAX_SCALESET_NAME_LENGTH - LABEL_HASH_LENGTH - 1 + if len(logical_name) <= limit: + return logical_name + digest = hashlib.sha256(logical_name.encode("utf-8")).hexdigest()[:LABEL_HASH_LENGTH] + return f"{logical_name[: limit - LABEL_HASH_LENGTH - 1]}-{digest}" + + +def target_scaleset_name(logical_name: str, labels: list[str]) -> str: + """Return the live GARM name a spec's scaleset should have. + + Args: + logical_name: The scaleset name the operator configured. + labels: The desired labels. + + Returns: + ``-