diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c96ce56 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,131 @@ +# AGENTS.md + +## Project Overview + +Tilebox IaC - reusable Pulumi components for auto-scaling Tilebox runner clusters on AWS, Azure, and GCP. + +Clusters use the public `ghcr.io/tilebox/runner:latest` image. This repository consumes that image but does not build or publish runner images. Custom images are outside this library's scope. + +## Commands + +- **Install dependencies**: `uv sync` +- **Lint**: `ruff check .` +- **Format**: `ruff format .` +- **Type check**: `pyright` +- **Pulumi preview**: `pulumi preview` +- **Pulumi deploy**: `pulumi up` + +## Architecture + +This is a Pulumi Python library providing reusable infrastructure components, organized into cloud-specific submodules: + +### Package Structure + +``` +tilebox_iac/ +├── __init__.py # Exports aws, azure, and gcp submodules +├── release_runner.py # Official GHCR runner image default +├── aws/ # Amazon Web Services components +│ ├── __init__.py +│ ├── auto_scaling_cluster.py +│ ├── cloud-init.yaml +│ ├── iam_role.py +│ ├── network.py +│ └── secrets.py +├── azure/ # Microsoft Azure components +│ ├── __init__.py +│ ├── auto_scaling_cluster.py +│ ├── cloud-init.yaml +│ ├── identity.py +│ ├── network.py +│ └── secrets.py +└── gcp/ # Google Cloud Platform components + ├── __init__.py + ├── auto_scaling_cluster.py + ├── cloud-init.yaml + ├── network.py + ├── secrets.py + └── service_account.py +``` + +### GCP Components (`tilebox_iac.gcp`) + +- `AutoScalingCluster` - Managed Instance Group with Spot VMs and CPU-based autoscaling +- `Network` - VPC with Private Google Access and optional NAT Gateway +- `ServiceAccount` - GCP IAM service accounts with role bindings +- `Secret` - GCP Secret Manager wrapper + +### AWS Components (`tilebox_iac.aws`) + +- `AutoScalingCluster` - Auto Scaling Group with Spot instances and a Launch Template +- `Network` - VPC with private subnets, optional NAT Gateway, and S3 VPC Gateway Endpoint +- `IAMRole` - IAM roles with instance profiles, bucket policies, secrets access +- `Secret` - AWS Secrets Manager wrapper + +### Azure Components (`tilebox_iac.azure`) + +- `AutoScalingCluster` - Virtual Machine Scale Set with CPU-based autoscaling and optional Spot instances +- `Network` - VNet with a private subnet and optional NAT Gateway +- `ManagedIdentity` - Azure managed identity with role assignments +- `Secret` - Azure Key Vault secret wrapper + +## Usage + +```python +from tilebox_iac import gcp + +runner_environment = { + "TILEBOX_API_KEY": tilebox_api_key_secret, +} + +cluster = gcp.AutoScalingCluster( + "my-cluster", + environment_variables=runner_environment, + ..., +) +``` + +AWS, Azure, and GCP use the same runner image and environment variable contract. `TILEBOX_API_KEY` is required; `TILEBOX_CLUSTER` is optional and defaults to the account's default cluster. In the upstream Tilebox repository, keep `ghcr.io/tilebox/runner:latest` as the built-in image and do not add image configuration, image builders, publishing, or private registry authentication. Forks may customize the image behavior for their own deployments. + +## Code Style + +- Use `ComponentResource` pattern for Pulumi components +- Use `TypedDict` for configuration dictionaries +- Use Jinja2 for cloud-init templates +- Follow ruff ALL rules (see pyproject.toml for ignored rules) +- Use type hints everywhere +- Import order: stdlib, blank line, external, blank line, internal (enforced by ruff isort) +- Format with ruff format + +## Key Patterns + +### ComponentResource Structure +```python +class MyComponent(ComponentResource): + def __init__(self, name: str, ..., opts: ResourceOptions | None = None) -> None: + super().__init__("tilebox:cloud:ComponentName", name, opts=opts) + # Create resources with ResourceOptions(parent=self) + self.register_outputs({...}) +``` + +### TypedDict for Configs +```python +class MyConfigDict(TypedDict): + required_field: str + optional_field: NotRequired[str] +``` + +### Cloud-init Templates +- GCP: `gcp/cloud-init.yaml` - Uses GCP metadata server for secrets +- AWS: `aws/cloud-init.yaml` - Uses AWS CLI/IMDS for secrets +- Azure: `azure/cloud-init.yaml` - Uses a managed identity to fetch Key Vault secrets +- All templates anonymously pull the official GHCR runner image +- AWS self-reports persistent runner failures; GCP and Azure use provider health probes + +## Dependencies + +- `pulumi` - Core Pulumi SDK +- `pulumi-gcp` - GCP provider +- `pulumi-aws` - AWS provider +- `pulumi-azure-native` - Azure provider +- `jinja2` - Cloud-init templating diff --git a/README.md b/README.md index 411d4ff..f2e1edc 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,50 @@ -# IaC for a auto-scaling cluster utilizing GCP Spot instances +# Tilebox infrastructure components -This library provides the following [Pulumi](https://www.pulumi.com/) resources: +This library provides reusable [Pulumi](https://www.pulumi.com/) components for running auto-scaling Tilebox runners on AWS, Azure, and GCP. -- A `LocalBuildTrigger` ressource that runs a local command to build a Docker image on code changes and pushes it to a Google Artifact Registry repository -- A `Secret` resource to manage GCP secrets -- A `AutoScalingGCPCluster` resource to manage the auto-scaling cluster -- A `GCPNetwork` resource to manage networking within the Cluster, optionally enabling Private Google Access (PGA) and a router for outbound internet access +## Tilebox runner + +A [Tilebox runner](https://docs.tilebox.com/workflows/concepts/runners) watches one cluster and executes Python workflow releases deployed to it. It downloads release artifacts, starts their Python runtimes with `uv`, and updates its task registrations when deployments change. Workflow releases can therefore be deployed or rolled back without rebuilding the runner image or restarting the runner fleet. + +Tilebox publishes the runner image for Linux AMD64 and ARM64 at [`ghcr.io/tilebox/runner`](https://github.com/tilebox/runner/pkgs/container/runner). It includes `uv`, Python 3.12–3.14, Git, Git LFS, SSH support, and the matching Tilebox CLI release. Every cluster runs `ghcr.io/tilebox/runner:latest`; instances pull the current image when they start. The image starts the runner with: + +```bash +tilebox runner start +``` + +The default runner requires no image configuration: + +```python +from tilebox_iac import gcp + +cluster = gcp.AutoScalingCluster( + "tilebox-release-runners", + environment_variables={ + "TILEBOX_API_KEY": tilebox_api_key_secret, + "TILEBOX_CLUSTER": tilebox_cluster_slug, + }, + # Cloud, network, machine, and scaling configuration omitted. +) +``` + +AWS, Azure, and GCP use the same official image and environment variable contract. This library consumes the Tilebox image but does not build or publish runner images. + +## Deployment requirements + +Each release runner deployment needs: + +- A `TILEBOX_API_KEY`, injected through the cloud provider's secret manager rather than baked into the image. +- Optionally, a Tilebox cluster slug created with `tilebox cluster create`, passed as `TILEBOX_CLUSTER`. The [default cluster](https://docs.tilebox.com/workflows/concepts/clusters#default-cluster) is used when it is omitted. +- Optionally, a non-default Tilebox API endpoint passed as `TILEBOX_API_URL`. +- Outbound network access for the Tilebox API, workflow release artifacts, Python package indexes, and any data sources used by the workflows. +- Cloud credentials, network access, system libraries, disk space, and CPU or GPU hardware required by the deployed workflows. +- At least one workflow release deployed to the same cluster with `tilebox workflow deploy-release` before the runner can execute its tasks. + +[Release runners](https://docs.tilebox.com/guides/workflows/deploy-to-your-compute) currently execute Python workflow projects. The runner image is shared across workflows; workflow code and Python dependencies belong in each immutable workflow release. + +## Components + +The `tilebox_iac.aws`, `tilebox_iac.azure`, and `tilebox_iac.gcp` modules provide cloud-specific auto-scaling clusters, networking, identities, and secret integrations. ## Dev setup diff --git a/pyproject.toml b/pyproject.toml index 68e70e0..089a45e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,17 @@ [project] name = "tilebox_iac" version = "0.1.0" -description = "IaC for auto-scaling clusters utilizing GCP and AWS Spot instances." +description = "Pulumi components for auto-scaling Tilebox release runner clusters on AWS, Azure, and GCP." readme = "README.md" authors = [{ name = "Tilebox, Inc.", email = "support@tilebox.com" }] requires-python = ">=3.10" dependencies = [ - "dirhash>=0.5.0", "jinja2>=3.1.6", "pulumi>=3.186.0", "pulumi-aws>=6.60.0", - "pulumi-command>=1.1.0", + "pulumi-azure-native>=3.0.0", "pulumi-gcp>=8.39.0", + "typing-extensions>=4.0", ] [dependency-groups] diff --git a/tilebox_iac/__init__.py b/tilebox_iac/__init__.py index 3ba59ae..55d186d 100644 --- a/tilebox_iac/__init__.py +++ b/tilebox_iac/__init__.py @@ -1,6 +1,7 @@ -from tilebox_iac import aws, gcp +from tilebox_iac import aws, azure, gcp __all__ = [ "aws", + "azure", "gcp", ] diff --git a/tilebox_iac/aws/__init__.py b/tilebox_iac/aws/__init__.py index 6c9d7d5..0ae3d85 100644 --- a/tilebox_iac/aws/__init__.py +++ b/tilebox_iac/aws/__init__.py @@ -1,13 +1,11 @@ from tilebox_iac.aws.auto_scaling_cluster import AutoScalingCluster from tilebox_iac.aws.iam_role import IAMRole -from tilebox_iac.aws.image_builder import LocalBuildTrigger from tilebox_iac.aws.network import Network from tilebox_iac.aws.secrets import Secret __all__ = [ "AutoScalingCluster", "IAMRole", - "LocalBuildTrigger", "Network", "Secret", ] diff --git a/tilebox_iac/aws/auto_scaling_cluster.py b/tilebox_iac/aws/auto_scaling_cluster.py index 9ebea55..100073e 100644 --- a/tilebox_iac/aws/auto_scaling_cluster.py +++ b/tilebox_iac/aws/auto_scaling_cluster.py @@ -1,16 +1,19 @@ import base64 +import json from collections.abc import Sequence from pathlib import Path -from typing import Any, TypedDict +from typing import Any +import pulumi_aws as aws from jinja2 import Environment, FileSystemLoader from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_aws import autoscaling as aws_autoscaling from pulumi_aws import ec2 as aws_ec2 -from typing_extensions import NotRequired +from pulumi_aws import iam as aws_iam from tilebox_iac.aws.iam_role import IAMRole, IAMRoleConfigDict from tilebox_iac.aws.secrets import Secret +from tilebox_iac.release_runner import RUNNER_IMAGE env = Environment(loader=FileSystemLoader(Path(__file__).parent), autoescape=True) template = env.get_template("cloud-init.yaml") @@ -18,31 +21,22 @@ def _get_cloud_init(kwargs: dict[str, Any]) -> str: """Render the cloud-init config for the AWS VMs.""" - image: str = kwargs["image"] - tag: str = kwargs["tag"] or "latest" # Default empty string to "latest" environment_variables: dict[str, str] = kwargs["environment_variables"] secrets: dict[str, str] = kwargs["secrets"] secret_versions: dict[str, str] = kwargs["secret_versions"] return template.render( - CONTAINER_IMAGE=f"{image}:{tag}", - REGISTRY_HOSTNAME=image.split("/", maxsplit=1)[0], + CONTAINER_IMAGE=RUNNER_IMAGE, SECRETS=secrets, SECRET_VERSIONS=secret_versions, ENVIRONMENT_VARS=environment_variables, ) -class ContainerConfig(TypedDict): - image: Input[str] - tag: NotRequired[Input[str]] - - class AutoScalingCluster(ComponentResource): def __init__( # noqa: PLR0913 self, name: str, - container: ContainerConfig, instance_type: str, cpu_target: float, cluster_enabled: bool, @@ -59,7 +53,6 @@ def __init__( # noqa: PLR0913 Args: name: Name of the cluster. - container: Container image to run (ECR image URL). instance_type: EC2 instance type to use. cpu_target: CPU target for autoscaling (0.0 to 1.0). cluster_enabled: Whether the cluster is enabled. @@ -90,13 +83,6 @@ def __init__( # noqa: PLR0913 # Copy to avoid mutating caller's config (could cause side effects if reused) iam_config_copy: IAMRoleConfigDict = dict(iam_config) if iam_config else {} # type: ignore[assignment] - # ECR read access is required for cloud-init to docker pull the container image - managed_policies = list(iam_config_copy.get("managed_policies", [])) - ecr_policy = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" - if ecr_policy not in managed_policies: - managed_policies.append(ecr_policy) - iam_config_copy["managed_policies"] = managed_policies # type: ignore[typeddict-item] - secrets_access = list(iam_config_copy.get("secrets_access", [])) secrets_access.extend( {"secret_slug": secret.resource_name, "secret_arn": secret.arn} for secret in used_secrets.values() @@ -111,6 +97,36 @@ def __init__( # noqa: PLR0913 opts=ResourceOptions(depends_on=[*list(used_secrets.values())], parent=self), ) + asg_arn = Output.all( + partition=aws.get_partition_output().partition, + region=aws.get_region_output().name, + account_id=aws.get_caller_identity_output().account_id, + ).apply( + lambda values: ( + f"arn:{values['partition']}:autoscaling:{values['region']}:{values['account_id']}:" + f"autoScalingGroup:*:autoScalingGroupName/{name}-*" + ) + ) + health_policy = aws_iam.RolePolicy( + f"{name}-set-instance-health", + role=iam_role.role.name, + policy=asg_arn.apply( + lambda arn: json.dumps( + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "autoscaling:SetInstanceHealth", + "Resource": arn, + } + ], + } + ) + ), + opts=ResourceOptions(parent=self), + ) + secrets: dict[str, Input[str]] = {} # Include version IDs so secret value changes trigger Launch Template updates secret_versions: dict[str, Input[str]] = {} @@ -119,8 +135,6 @@ def __init__( # noqa: PLR0913 secret_versions[secret_env_var] = secret.latest_version cloud_init_config = Output.all( - image=container["image"], - tag=container.get("tag", "latest"), environment_variables=envs, secrets=secrets, secret_versions=secret_versions, @@ -170,7 +184,7 @@ def __init__( # noqa: PLR0913 tags={"Name": f"{name}-instance"}, ), ], - opts=ResourceOptions(depends_on=[iam_role], parent=self), + opts=ResourceOptions(depends_on=[iam_role, health_policy], parent=self), ) if cluster_enabled: diff --git a/tilebox_iac/aws/cloud-init.yaml b/tilebox_iac/aws/cloud-init.yaml index 3429481..61d9405 100644 --- a/tilebox_iac/aws/cloud-init.yaml +++ b/tilebox_iac/aws/cloud-init.yaml @@ -14,6 +14,7 @@ write_files: # Fetch AWS secrets from Secrets Manager using IMDSv2 credentials set -euo pipefail OUTPUT_FILE="${1:-/tmp/secrets.env}" + : > "${OUTPUT_FILE}" TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") REGION=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region) {% for secret_var, secret_arn in SECRETS.items() %} @@ -22,6 +23,42 @@ write_files: {% endfor %} chmod 600 "${OUTPUT_FILE}" +- path: /usr/local/bin/cloudservice-health + permissions: '0755' + owner: root + content: | + #!/bin/bash + set -euo pipefail + + FAILURE_FILE=/run/cloudservice-health-failures + if [ "$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' cloudservice 2>/dev/null || true)" = "true" ]; then + rm -f "${FAILURE_FILE}" + exit 0 + fi + + failures=$(cat "${FAILURE_FILE}" 2>/dev/null || echo 0) + failures=$((failures + 1)) + printf '%s\n' "${failures}" > "${FAILURE_FILE}" + if [ "${failures}" -lt 10 ]; then + exit 0 + fi + + # Use IMDSv2 to identify this instance before asking its ASG to replace it. + TOKEN=$(curl --fail --silent --show-error -X PUT \ + "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60") + INSTANCE_ID=$(curl --fail --silent --show-error \ + -H "X-aws-ec2-metadata-token: ${TOKEN}" \ + http://169.254.169.254/latest/meta-data/instance-id) + REGION=$(curl --fail --silent --show-error \ + -H "X-aws-ec2-metadata-token: ${TOKEN}" \ + http://169.254.169.254/latest/meta-data/placement/region) + aws autoscaling set-instance-health \ + --region "${REGION}" \ + --instance-id "${INSTANCE_ID}" \ + --health-status Unhealthy \ + --should-respect-grace-period + - path: /etc/systemd/system/cloudservice.service permissions: '0644' owner: root @@ -36,7 +73,6 @@ write_files: Restart=always RestartSec=5 ExecStartPre=/tmp/fetch-secrets.sh /tmp/secrets.env - ExecStartPre=/bin/bash -c 'TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") && REGION=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region) && aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin {{ REGISTRY_HOSTNAME }}' ExecStartPre=/usr/bin/docker pull {{ CONTAINER_IMAGE }} ExecStart=/usr/bin/docker run --rm --name cloudservice --env-file /tmp/secrets.env {% for key, value in ENVIRONMENT_VARS.items() %}-e {{ key }}={{ value }} {% endfor %}{{ CONTAINER_IMAGE }} ExecStop=/usr/bin/docker stop cloudservice @@ -44,9 +80,37 @@ write_files: [Install] WantedBy=multi-user.target +- path: /etc/systemd/system/cloudservice-health.service + permissions: '0644' + owner: root + content: | + [Unit] + Description=Check Tilebox Workflow Runner Health + After=cloudservice.service + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/cloudservice-health + +- path: /etc/systemd/system/cloudservice-health.timer + permissions: '0644' + owner: root + content: | + [Unit] + Description=Periodically Check Tilebox Workflow Runner Health + + [Timer] + OnBootSec=5min + OnUnitActiveSec=1min + + [Install] + WantedBy=timers.target + runcmd: - systemctl daemon-reload - systemctl enable docker.service - systemctl start docker.service - systemctl enable cloudservice.service - systemctl start cloudservice.service + - systemctl enable cloudservice-health.timer + - systemctl start cloudservice-health.timer diff --git a/tilebox_iac/aws/image_builder.py b/tilebox_iac/aws/image_builder.py deleted file mode 100644 index 9b9dc0b..0000000 --- a/tilebox_iac/aws/image_builder.py +++ /dev/null @@ -1,76 +0,0 @@ -from pathlib import Path - -from dirhash import dirhash -from pulumi import ComponentResource, Input, Output, ResourceOptions -from pulumi_command.local import Command - - -class LocalBuildTrigger(ComponentResource): - def __init__( # noqa: PLR0913 - self, - name: str, - aws_region: str, - aws_account_id: Input[str], - repository_name: Input[str], - source_dir: Path, - additional_ignore_patterns: list[str] | None = None, - platform: str = "linux/amd64", - opts: ResourceOptions | None = None, - ) -> None: - """A local build trigger that builds a Docker image on code changes and pushes it to an AWS ECR repository. - - Args: - name: Name of the image. - aws_region: AWS region. - aws_account_id: AWS account ID. - repository_name: ECR repository name. - source_dir: Path to the source directory. - additional_ignore_patterns: Additional ignore patterns for excluding files or directories when determining - if the source code has changed, and therefore if the image needs to be rebuilt. - platform: Docker platform to build for (e.g., "linux/amd64" or "linux/arm64"). - opts: Pulumi resource options. - """ - super().__init__("tilebox:aws:ImageBuilder", name, opts=opts) - - # Hash source files to detect changes; tag is used as image tag and Pulumi trigger - ignore = [".venv/*"] + (additional_ignore_patterns or []) - # Include all files so extensionless runtime binaries (for example `dynamic_runner/tilebox`) - # trigger rebuilds reliably. - self.tag = dirhash( - source_dir, - "sha256", - match=["*", "**/*"], - ignore=ignore, - ) - - def build_command(args: list[str]) -> str: - account_id, repo_name = args[0], args[1] - hostname = f"{account_id}.dkr.ecr.{aws_region}.amazonaws.com" - image_uri = f"{hostname}/{repo_name}" - - return ( - f"aws ecr get-login-password --region {aws_region} | docker login --username AWS --password-stdin {hostname} && " - f"docker build --platform {platform} --load -t {image_uri}:{self.tag} {source_dir} && " - f"docker tag {image_uri}:{self.tag} {image_uri}:latest && " - f"docker push {image_uri}:{self.tag} && " - f"docker push {image_uri}:latest" - ) - - self.docker_build = Command( - f"{name}-docker-build-image", - create=Output.all(aws_account_id, repository_name).apply(build_command), - # Only rebuild when source hash changes (content-addressable builds) - triggers=[self.tag], - opts=ResourceOptions(parent=self), - ) - - self.container_image = Output.concat( - aws_account_id, ".dkr.ecr.", aws_region, ".amazonaws.com/", repository_name - ) - - self.register_outputs( - { - "container_image": self.container_image, - "code_hash": self.tag, - } - ) diff --git a/tilebox_iac/azure/__init__.py b/tilebox_iac/azure/__init__.py new file mode 100644 index 0000000..5c0ea49 --- /dev/null +++ b/tilebox_iac/azure/__init__.py @@ -0,0 +1,11 @@ +from tilebox_iac.azure.auto_scaling_cluster import AutoScalingCluster +from tilebox_iac.azure.identity import ManagedIdentity +from tilebox_iac.azure.network import Network +from tilebox_iac.azure.secrets import Secret + +__all__ = [ + "AutoScalingCluster", + "ManagedIdentity", + "Network", + "Secret", +] diff --git a/tilebox_iac/azure/auto_scaling_cluster.py b/tilebox_iac/azure/auto_scaling_cluster.py new file mode 100644 index 0000000..68089a1 --- /dev/null +++ b/tilebox_iac/azure/auto_scaling_cluster.py @@ -0,0 +1,241 @@ +import base64 +from pathlib import Path +from typing import Any + +from jinja2 import Environment, FileSystemLoader +from pulumi import ComponentResource, Input, Output, ResourceOptions +from pulumi_azure_native import compute, monitor + +from tilebox_iac.azure.identity import ManagedIdentity, ManagedIdentityConfigDict +from tilebox_iac.azure.secrets import Secret +from tilebox_iac.release_runner import RUNNER_IMAGE + +env = Environment(loader=FileSystemLoader(Path(__file__).parent), autoescape=True) +template = env.get_template("cloud-init.yaml") + + +def _get_cloud_init(kwargs: dict[str, Any]) -> str: + environment_variables: dict[str, str] = kwargs["environment_variables"] + secrets: dict[str, str] = kwargs["secrets"] + client_id: str = kwargs["client_id"] + + return template.render( + CONTAINER_IMAGE=RUNNER_IMAGE, + SECRETS=secrets, + ENVIRONMENT_VARS=environment_variables, + CLIENT_ID=client_id, + ) + + +class AutoScalingCluster(ComponentResource): + def __init__( # noqa: PLR0913 + self, + name: str, + resource_group_name: Input[str], + location: Input[str], + vm_size: str, + cpu_target: float, + cluster_enabled: bool, + min_replicas_config: int, + max_replicas_config: int, + subnet_id: Input[str], + admin_ssh_public_key: Input[str], + admin_username: str = "cloudservice", + use_spot: bool = True, + environment_variables: dict[str, Input[str] | Secret] | None = None, + identity_config: ManagedIdentityConfigDict | None = None, + source_image_reference: compute.ImageReferenceArgs | None = None, + os_disk_size_gb: int = 30, + opts: ResourceOptions | None = None, + ) -> None: + """An auto-scaling cluster of Azure VMSS instances running a Docker container.""" + super().__init__("tilebox:azure:AutoScalingCluster", name, opts=opts) + + used_secrets: dict[str, Secret] = {} + envs: dict[str, Input[str]] = {} + if environment_variables is not None: + for key in sorted(environment_variables): + value = environment_variables[key] + if isinstance(value, Secret): + used_secrets[key] = value + else: + envs[key] = value + + managed_identity = ManagedIdentity.from_config( + name, + resource_group_name, + location, + identity_config, + opts=ResourceOptions(depends_on=[*list(used_secrets.values())], parent=self), + ) + + secrets: dict[str, Input[str]] = {} + for secret_env_var, secret in used_secrets.items(): + secrets[secret_env_var] = secret.secret_uri + + cloud_init_config = Output.all( + environment_variables=envs, + secrets=secrets, + client_id=managed_identity.client_id, + ).apply(_get_cloud_init) + custom_data = cloud_init_config.apply(lambda c: base64.b64encode(c.encode()).decode()) + + if source_image_reference is None: + source_image_reference = compute.ImageReferenceArgs( + publisher="Canonical", + offer="0001-com-ubuntu-server-jammy", + sku="22_04-lts-gen2", + version="latest", + ) + + capacity = min_replicas_config if cluster_enabled else 0 + max_capacity = max_replicas_config if cluster_enabled else 0 + ignore_changes = ["sku.capacity"] if cluster_enabled else [] + + self.vmss = compute.VirtualMachineScaleSet( + f"{name}-vmss", + resource_group_name=resource_group_name, + vm_scale_set_name=f"{name}-vmss", + location=location, + sku={"name": vm_size, "tier": "Standard", "capacity": capacity}, + overprovision=False, + upgrade_policy=compute.UpgradePolicyArgs( + mode=compute.UpgradeMode.ROLLING, + rolling_upgrade_policy=compute.RollingUpgradePolicyArgs( + # New VMs rerun cloud-init; quota fallback does not. + max_surge=True, + ), + ), + identity=compute.VirtualMachineScaleSetIdentityArgs( + type=compute.ResourceIdentityType.USER_ASSIGNED, + user_assigned_identities=[managed_identity.id], + ), + virtual_machine_profile=compute.VirtualMachineScaleSetVMProfileArgs( + priority="Spot" if use_spot else None, + eviction_policy="Delete" if use_spot else None, + billing_profile=compute.BillingProfileArgs(max_price=-1) if use_spot else None, + extension_profile=compute.VirtualMachineScaleSetExtensionProfileArgs( + extensions=[ + compute.VirtualMachineScaleSetExtensionArgs( + name="runner-health", + publisher="Microsoft.ManagedServices", + type="ApplicationHealthLinux", + type_handler_version="2.0", + auto_upgrade_minor_version=True, + settings={ + "protocol": "http", + "port": 8080, + "requestPath": "/health", + "gracePeriod": 900, + }, + ) + ] + ), + storage_profile=compute.VirtualMachineScaleSetStorageProfileArgs( + image_reference=source_image_reference, + os_disk=compute.VirtualMachineScaleSetOSDiskArgs( + create_option="FromImage", + caching=compute.CachingTypes.READ_WRITE, + managed_disk=compute.VirtualMachineScaleSetManagedDiskParametersArgs( + storage_account_type="Standard_LRS" + ), + disk_size_gb=os_disk_size_gb, + ), + ), + os_profile=compute.VirtualMachineScaleSetOSProfileArgs( + computer_name_prefix=name[:9], + admin_username=admin_username, + custom_data=custom_data, + linux_configuration=compute.LinuxConfigurationArgs( + disable_password_authentication=True, + ssh=compute.SshConfigurationArgs( + public_keys=[ + compute.SshPublicKeyArgs( + path=f"/home/{admin_username}/.ssh/authorized_keys", + key_data=admin_ssh_public_key, + ) + ] + ), + ), + ), + network_profile=compute.VirtualMachineScaleSetNetworkProfileArgs( + network_interface_configurations=[ + compute.VirtualMachineScaleSetNetworkConfigurationArgs( + name=f"{name}-nic", + primary=True, + ip_configurations=[ + compute.VirtualMachineScaleSetIPConfigurationArgs( + name=f"{name}-ipconfig", + primary=True, + subnet=compute.ApiEntityReferenceArgs(id=subnet_id), + ) + ], + ) + ] + ), + ), + opts=ResourceOptions(depends_on=[managed_identity], parent=self, ignore_changes=ignore_changes), + ) + + if cluster_enabled: + self.autoscale_setting = monitor.AutoscaleSetting( + f"{name}-autoscale", + resource_group_name=resource_group_name, + autoscale_setting_name=f"{name}-autoscale", + location=location, + enabled=True, + target_resource_uri=self.vmss.id, + profiles=[ + monitor.AutoscaleProfileArgs( + name="cpu-autoscale", + capacity=monitor.ScaleCapacityArgs( + minimum=str(min_replicas_config), + maximum=str(max_capacity), + default=str(capacity), + ), + rules=[ + monitor.ScaleRuleArgs( + metric_trigger=monitor.MetricTriggerArgs( + metric_name="Percentage CPU", + metric_namespace="microsoft.compute/virtualmachinescalesets", + metric_resource_uri=self.vmss.id, + time_grain="PT1M", + statistic=monitor.MetricStatisticType.AVERAGE, + time_window="PT5M", + time_aggregation=monitor.TimeAggregationType.AVERAGE, + operator=monitor.ComparisonOperationType.GREATER_THAN, + threshold=cpu_target * 100, + ), + scale_action=monitor.ScaleActionArgs( + direction=monitor.ScaleDirection.INCREASE, + type=monitor.ScaleType.CHANGE_COUNT, + value="1", + cooldown="PT1M", + ), + ), + monitor.ScaleRuleArgs( + metric_trigger=monitor.MetricTriggerArgs( + metric_name="Percentage CPU", + metric_namespace="microsoft.compute/virtualmachinescalesets", + metric_resource_uri=self.vmss.id, + time_grain="PT1M", + statistic=monitor.MetricStatisticType.AVERAGE, + time_window="PT5M", + time_aggregation=monitor.TimeAggregationType.AVERAGE, + operator=monitor.ComparisonOperationType.LESS_THAN, + threshold=max(cpu_target * 50, 1), + ), + scale_action=monitor.ScaleActionArgs( + direction=monitor.ScaleDirection.DECREASE, + type=monitor.ScaleType.CHANGE_COUNT, + value="1", + cooldown="PT1M", + ), + ), + ], + ) + ], + opts=ResourceOptions(depends_on=[self.vmss], parent=self), + ) + + self.register_outputs({"vmss_id": self.vmss.id, "identity_id": managed_identity.id}) diff --git a/tilebox_iac/azure/cloud-init.yaml b/tilebox_iac/azure/cloud-init.yaml new file mode 100644 index 0000000..db536af --- /dev/null +++ b/tilebox_iac/azure/cloud-init.yaml @@ -0,0 +1,135 @@ +#cloud-config + +packages: + - docker.io + - jq + +write_files: +- path: /tmp/fetch-secrets.sh + permissions: '0755' + owner: root + content: | + #!/bin/bash + # Fetch Azure Key Vault secrets using the VM's managed identity. + set -euo pipefail + + OUTPUT_FILE="${1:-/tmp/secrets.env}" + : > "${OUTPUT_FILE}" + + get_access_token() { + curl -s -H "Metadata: true" \ + "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https%3A%2F%2Fvault.azure.net{% if CLIENT_ID %}&client_id={{ CLIENT_ID }}{% endif %}" \ + | jq -r ".access_token" + } + + fetch_secret() { + local secret_url="$1" + local access_token="$2" + + curl -s -H "Authorization: Bearer ${access_token}" \ + "${secret_url}?api-version=7.4" \ + | jq -r ".value" + } + + {% if SECRETS %} + ACCESS_TOKEN=$(get_access_token) + {% for secret_var, secret_url in SECRETS.items() %} + printf '%s=%s\n' "{{ secret_var }}" "$(fetch_secret "{{ secret_url }}" "${ACCESS_TOKEN}")" >> "${OUTPUT_FILE}" + {% endfor %} + {% endif %} + + chmod 600 "${OUTPUT_FILE}" + +- path: /usr/local/bin/cloudservice-health + permissions: '0755' + owner: root + content: | + #!/usr/bin/env python3 + import json + import subprocess + import time + from http.server import BaseHTTPRequestHandler, HTTPServer + + def runner_is_running(): + try: + result = subprocess.run( + ["docker", "inspect", "--format", "{% raw %}{{.State.Running}}{% endraw %}", "cloudservice"], + check=False, + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return result.returncode == 0 and result.stdout.strip() == "true" + + class HealthHandler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path != "/health": + self.send_error(404) + return + + state = "Healthy" if runner_is_running() else "Unhealthy" + response = json.dumps({"ApplicationHealthState": state}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(response))) + self.end_headers() + self.wfile.write(response) + + def log_message(self, format, *args): + pass + + while not runner_is_running(): + time.sleep(5) + + HTTPServer(("127.0.0.1", 8080), HealthHandler).serve_forever() + +- path: /etc/systemd/system/cloudservice.service + permissions: '0644' + owner: root + content: | + [Unit] + Description=Tilebox Workflow Runner + After=docker.service network-online.target + Wants=network-online.target + Requires=docker.service + + [Service] + Type=simple + Restart=always + RestartSec=5 + ExecStartPre=/bin/bash /tmp/fetch-secrets.sh /tmp/secrets.env + ExecStartPre=/usr/bin/docker pull {{ CONTAINER_IMAGE }} + ExecStart=/usr/bin/docker run --rm --name cloudservice {% if ENVIRONMENT_VARS.get('TILEBOX_ENABLE_NVIDIA_GPU') == 'true' %}--gpus all {% endif %}--env-file /tmp/secrets.env {% for key, value in ENVIRONMENT_VARS.items() %}-e {{ key }}="{{ value }}" {% endfor %}{{ CONTAINER_IMAGE }} + ExecStop=-/usr/bin/docker stop cloudservice + + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/cloudservice-health.service + permissions: '0644' + owner: root + content: | + [Unit] + Description=Tilebox Workflow Runner Health Endpoint + After=docker.service + Requires=docker.service + + [Service] + Type=simple + Restart=always + RestartSec=5 + ExecStart=/usr/local/bin/cloudservice-health + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl daemon-reload + - systemctl enable docker.service + - systemctl start docker.service + - systemctl enable cloudservice.service + - systemctl start cloudservice.service + - systemctl enable cloudservice-health.service + - systemctl start cloudservice-health.service diff --git a/tilebox_iac/azure/identity.py b/tilebox_iac/azure/identity.py new file mode 100644 index 0000000..e38536f --- /dev/null +++ b/tilebox_iac/azure/identity.py @@ -0,0 +1,84 @@ +import uuid +from collections.abc import Sequence +from typing import TypedDict + +from pulumi import ComponentResource, Input, Output, ResourceOptions +from pulumi_azure_native import authorization, managedidentity +from typing_extensions import NotRequired + + +class ScopeRoleDict(TypedDict): + scope_slug: str + scope: Input[str] + role_definition_id: Input[str] + + +class ManagedIdentityConfigDict(TypedDict): + scope_roles: NotRequired[Sequence[ScopeRoleDict]] + + +class ManagedIdentity(ComponentResource): + def __init__( + self, + name: str, + resource_group_name: Input[str], + location: Input[str], + scope_roles: Sequence[ScopeRoleDict] | None = None, + opts: ResourceOptions | None = None, + ) -> None: + """Create a user-assigned managed identity and optional Azure role assignments.""" + super().__init__("tilebox:azure:ManagedIdentity", name, opts=opts) + + self.identity = managedidentity.UserAssignedIdentity( + f"{name}-identity", + resource_group_name=resource_group_name, + resource_name_=f"{name}-identity", + location=location, + opts=ResourceOptions(parent=self), + ) + + self.role_assignments = [] + for role in scope_roles or []: + assignment_name = str(uuid.uuid5(uuid.NAMESPACE_URL, f"{name}:{role['scope_slug']}")) + self.role_assignments.append( + authorization.RoleAssignment( + f"{name}-role-{role['scope_slug']}", + principal_id=self.identity.principal_id, + principal_type="ServicePrincipal", + role_assignment_name=assignment_name, + role_definition_id=role["role_definition_id"], + scope=role["scope"], + opts=ResourceOptions(depends_on=[self.identity], parent=self), + ) + ) + + self.id: Output[str] = self.identity.id + self.client_id: Output[str] = self.identity.client_id + self.principal_id: Output[str] = self.identity.principal_id + self.register_outputs( + { + "id": self.id, + "client_id": self.client_id, + "principal_id": self.principal_id, + } + ) + + @classmethod + def from_config( + cls, + name: str, + resource_group_name: Input[str], + location: Input[str], + config: ManagedIdentityConfigDict | None, + opts: ResourceOptions | None = None, + ) -> "ManagedIdentity": + if config is None: + return cls(name, resource_group_name, location, opts=opts) + + return cls( + name, + resource_group_name=resource_group_name, + location=location, + scope_roles=config.get("scope_roles"), + opts=opts, + ) diff --git a/tilebox_iac/azure/network.py b/tilebox_iac/azure/network.py new file mode 100644 index 0000000..112cf2b --- /dev/null +++ b/tilebox_iac/azure/network.py @@ -0,0 +1,69 @@ +import ipaddress + +from pulumi import ComponentResource, ResourceOptions +from pulumi_azure_native import network + + +class Network(ComponentResource): + def __init__( # noqa: PLR0913 + self, + name: str, + resource_group_name: str, + location: str, + enable_internet_access: bool = True, + cidr_block: str = "10.10.0.0/16", + opts: ResourceOptions | None = None, + ) -> None: + """An Azure VNet with a private subnet and optional NAT Gateway for outbound internet access.""" + super().__init__("tilebox:azure:Network", name, opts=opts) + + vnet_network = ipaddress.ip_network(cidr_block) + if vnet_network.prefixlen > 24: + msg = f"CIDR block {cidr_block} is too small. Must be /24 or larger to accommodate a subnet." + raise ValueError(msg) + subnet_cidr = str(next(vnet_network.subnets(new_prefix=24))) + + self.virtual_network = network.VirtualNetwork( + f"{name}-vnet", + resource_group_name=resource_group_name, + virtual_network_name=f"{name}-vnet", + location=location, + address_space={"address_prefixes": [cidr_block]}, + opts=ResourceOptions(parent=self), + ) + + nat_gateway_id = None + if enable_internet_access: + public_ip = network.PublicIPAddress( + f"{name}-nat-ip", + resource_group_name=resource_group_name, + public_ip_address_name=f"{name}-nat-ip", + location=location, + public_ip_allocation_method="Static", + sku={"name": "Standard"}, + opts=ResourceOptions(depends_on=[self.virtual_network], parent=self), + ) + nat_gateway = network.NatGateway( + f"{name}-nat", + resource_group_name=resource_group_name, + nat_gateway_name=f"{name}-nat", + location=location, + sku={"name": "Standard"}, + public_ip_addresses=[{"id": public_ip.id}], + opts=ResourceOptions(depends_on=[public_ip], parent=self), + ) + nat_gateway_id = nat_gateway.id + + self.subnet = network.Subnet( + f"{name}-subnet", + resource_group_name=resource_group_name, + virtual_network_name=self.virtual_network.name, + subnet_name=f"{name}-subnet", + address_prefix=subnet_cidr, + nat_gateway={"id": nat_gateway_id} if nat_gateway_id is not None else None, + opts=ResourceOptions(depends_on=[self.virtual_network], parent=self), + ) + + self.id = self.virtual_network.id + self.subnet_id = self.subnet.id + self.register_outputs({"id": self.id, "subnet_id": self.subnet_id}) diff --git a/tilebox_iac/azure/secrets.py b/tilebox_iac/azure/secrets.py new file mode 100644 index 0000000..21e8100 --- /dev/null +++ b/tilebox_iac/azure/secrets.py @@ -0,0 +1,36 @@ +from pulumi import ComponentResource, Input, ResourceOptions +from pulumi_azure_native import keyvault + + +class Secret(ComponentResource): + def __init__( + self, + name: str, + resource_group_name: Input[str], + vault_name: Input[str], + secret_data: Input[str] | None = None, + opts: ResourceOptions | None = None, + ) -> None: + """A secret stored in Azure Key Vault.""" + super().__init__("tilebox:azure:Secret", name, opts=opts) + + self.resource_name = name + self.secret = keyvault.Secret( + name, + resource_group_name=resource_group_name, + vault_name=vault_name, + secret_name=name, + properties=keyvault.SecretPropertiesArgs(value=secret_data), + opts=ResourceOptions(parent=self), + ) + + self.id = self.secret.id + self.name = self.secret.name + self.secret_uri = self.secret.properties.apply(lambda properties: properties.secret_uri) + self.register_outputs( + { + "id": self.id, + "name": self.name, + "secret_uri": self.secret_uri, + } + ) diff --git a/tilebox_iac/gcp/__init__.py b/tilebox_iac/gcp/__init__.py index 0d667a1..ba3da47 100644 --- a/tilebox_iac/gcp/__init__.py +++ b/tilebox_iac/gcp/__init__.py @@ -1,12 +1,10 @@ from tilebox_iac.gcp.auto_scaling_cluster import AutoScalingCluster -from tilebox_iac.gcp.image_builder import LocalBuildTrigger from tilebox_iac.gcp.network import Network from tilebox_iac.gcp.secrets import Secret from tilebox_iac.gcp.service_account import ServiceAccount __all__ = [ "AutoScalingCluster", - "LocalBuildTrigger", "Network", "Secret", "ServiceAccount", diff --git a/tilebox_iac/gcp/auto_scaling_cluster.py b/tilebox_iac/gcp/auto_scaling_cluster.py index 30e1d2b..9c78abb 100644 --- a/tilebox_iac/gcp/auto_scaling_cluster.py +++ b/tilebox_iac/gcp/auto_scaling_cluster.py @@ -1,20 +1,22 @@ from collections.abc import Sequence from pathlib import Path -from typing import Any, TypedDict +from typing import Any from jinja2 import Environment, FileSystemLoader from pulumi import Alias, ComponentResource, Input, Output, ResourceOptions from pulumi_gcp.compute import ( + Firewall, InstanceTemplate, InstanceTemplateNetworkInterfaceArgs, InstanceTemplateNetworkInterfaceArgsDict, RegionAutoscaler, + RegionHealthCheck, RegionInstanceGroupManager, ) -from typing_extensions import NotRequired from tilebox_iac.gcp.secrets import Secret from tilebox_iac.gcp.service_account import ServiceAccount, ServiceAccountConfigDict +from tilebox_iac.release_runner import RUNNER_IMAGE env = Environment(loader=FileSystemLoader(Path(__file__).parent), autoescape=True) template = env.get_template("cloud-init.yaml") @@ -22,29 +24,36 @@ def _get_cloud_init(kwargs: dict[str, Any]) -> str: """Render the cloud-init config for the GCP VMs.""" - image: str = kwargs["image"] - tag: str = kwargs["tag"] environment_variables: dict[str, str] = kwargs["environment_variables"] secrets: dict[str, str] = kwargs["secrets"] return template.render( - CONTAINER_IMAGE=f"{image}:{tag}", - REGISTRY_HOSTNAME=image.split("/")[0], + CONTAINER_IMAGE=RUNNER_IMAGE, SECRETS=secrets, ENVIRONMENT_VARS=environment_variables, ) -class ContainerConfig(TypedDict): - image: Input[str] - tag: NotRequired[Input[str]] +def _get_health_check_network(network_interfaces: Any) -> str: + if not network_interfaces: + return "default" + + first_interface = network_interfaces[0] + network = ( + first_interface.get("network") + if isinstance(first_interface, dict) + else getattr(first_interface, "network", None) + ) + if not network: + msg = "network_interfaces[0].network is required for GCP runner health checks" + raise ValueError(msg) + return str(network) class AutoScalingCluster(ComponentResource): def __init__( # noqa: PLR0913 self, name: str, - container: ContainerConfig, gcp_project: str, gcp_region: str, machine_type: str, @@ -64,7 +73,6 @@ def __init__( # noqa: PLR0913 Args: name: Name of the cluster. - container: Container image to run. gcp_project: GCP project ID to deploy the cluster in. gcp_region: Region to deploy the cluster in. machine_type: Machine type to use for the VMs. @@ -74,17 +82,13 @@ def __init__( # noqa: PLR0913 max_replicas_config: Maximum number of replicas. environment_variables: Environment variables to pass to the container. roles: Roles to assign to the service account. - network_interfaces: List of network interfaces to attach to the VMs. + network_interfaces: List of network interfaces to attach to the VMs. The first interface must include its + network so the runner health-check firewall can target it. opts: Pulumi resource options. """ opts = ResourceOptions.merge(opts, ResourceOptions(aliases=[Alias(type_="tilebox:AutoScalingGCPCluster")])) super().__init__("tilebox:gcp:AutoScalingCluster", name, opts=opts) - if container.get("tag") == "": - raise ValueError( - "Container tag cannot be empty. Leave unset or manually set to `latest` to use the latest tag." - ) - required_roles = { "roles/monitoring.metricWriter", } @@ -122,13 +126,40 @@ def __init__( # noqa: PLR0913 name, gcp_project, roles, opts=ResourceOptions(depends_on=[*list(used_secrets.values())], parent=self) ) + health_check = RegionHealthCheck( + f"{name}-health-check", + name=f"{name}-health-check", + project=gcp_project, + region=gcp_region, + check_interval_sec=30, + timeout_sec=5, + healthy_threshold=1, + unhealthy_threshold=3, + http_health_check={ + "port": 8080, + "request_path": "/health", + }, + opts=ResourceOptions(parent=self), + ) + health_check_network = Output.from_input(network_interfaces).apply(_get_health_check_network) + health_check_firewall = Firewall( + f"{name}-health-check", + name=f"{name}-health-check", + project=gcp_project, + network=health_check_network, + direction="INGRESS", + # Google Cloud health-check probe ranges. + source_ranges=["130.211.0.0/22", "35.191.0.0/16"], + target_service_accounts=[service_account.email], + allows=[{"protocol": "tcp", "ports": ["8080"]}], + opts=ResourceOptions(depends_on=[service_account], parent=self), + ) + secrets = {} for secret_env_var, secret in used_secrets.items(): secrets[secret_env_var] = secret.secret.id cloud_init_config = Output.all( - image=container["image"], - tag=container.get("tag", "latest"), environment_variables=envs, secrets=secrets, ).apply(_get_cloud_init) @@ -180,7 +211,14 @@ def __init__( # noqa: PLR0913 "max_surge_fixed": 10, "max_unavailable_fixed": 0, }, - opts=ResourceOptions(depends_on=[instance_template], parent=self), + auto_healing_policies={ + "health_check": health_check.id, + "initial_delay_sec": 900, + }, + opts=ResourceOptions( + depends_on=[instance_template, health_check, health_check_firewall], + parent=self, + ), ) if cluster_enabled: diff --git a/tilebox_iac/gcp/cloud-init.yaml b/tilebox_iac/gcp/cloud-init.yaml index 634172f..d5ed799 100644 --- a/tilebox_iac/gcp/cloud-init.yaml +++ b/tilebox_iac/gcp/cloud-init.yaml @@ -42,6 +42,17 @@ write_files: chmod 600 "${OUTPUT_FILE}" +- path: /etc/cloudservice-health + permissions: 0755 + owner: root + content: | + #!/bin/bash + if [ "$(docker inspect --format '{% raw %}{{.State.Running}}{% endraw %}' cloudservice.service 2>/dev/null || true)" = "true" ]; then + printf 'HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n' + else + printf 'HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n' + fi + - path: /etc/systemd/system/cloudservice.service permissions: 0644 owner: root @@ -56,7 +67,6 @@ write_files: Restart=always RestartSec=5 Environment="HOME=/home/cloudservice" - ExecStartPre=/usr/bin/docker-credential-gcr configure-docker --registries {{ REGISTRY_HOSTNAME }} ExecStartPre=/bin/bash /tmp/fetch-secrets.sh ExecStartPre=/usr/bin/docker pull {{ CONTAINER_IMAGE }} @@ -68,6 +78,29 @@ write_files: ExecStop=-/usr/bin/docker stop %n ExecStopPost=-/usr/bin/docker rm %n + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/cloudservice-health.service + permissions: 0644 + owner: root + content: | + [Unit] + Description=Tilebox Workflow Runner Health Endpoint + After=docker.service + + [Service] + Type=simple + Restart=always + RestartSec=5 + ExecStart=/usr/bin/socat TCP-LISTEN:8080,reuseaddr,fork EXEC:/etc/cloudservice-health + + [Install] + WantedBy=multi-user.target + runcmd: - systemctl daemon-reload +- systemctl enable cloudservice.service - systemctl start cloudservice.service +- systemctl enable cloudservice-health.service +- systemctl start cloudservice-health.service diff --git a/tilebox_iac/gcp/image_builder.py b/tilebox_iac/gcp/image_builder.py deleted file mode 100644 index c72db99..0000000 --- a/tilebox_iac/gcp/image_builder.py +++ /dev/null @@ -1,109 +0,0 @@ -import json -from pathlib import Path - -from dirhash import dirhash -from pulumi import Alias, ComponentResource, Input, Output, ResourceOptions -from pulumi_command.local import Command - - -class LocalBuildTrigger(ComponentResource): - def __init__( # noqa: PLR0913 - self, - name: str, - gcp_region: str, - gcp_project: str, - repository_id: Input[str], - source_dir: Path, - additional_ignore_patterns: list[str] | None = None, - platform: str = "linux/amd64", - opts: ResourceOptions | None = None, - ) -> None: - """A local build trigger that builds a Docker image on code changes and pushes it to a Google Artifact Registry repository. - - Args: - name: Name of the image. - gcp_region: Region of the GCP project. - gcp_project: GCP project ID. - repository_id: ID of the artifact registry repository. - source_dir: Path to the source directory. - additional_ignore_patterns: Additional ignore patterns for excluding files or directories when determining - if the source code has changed, and therefore if the image needs to be rebuilt. - platform: Docker platform to build for (e.g., "linux/amd64" or "linux/arm64"). - opts: Pulumi resource options. - """ - opts = ResourceOptions.merge(opts, ResourceOptions(aliases=[Alias(type_="tilebox:LocalBuildTrigger")])) - super().__init__("tilebox:gcp:LocalBuildTrigger", name, opts=opts) - hostname = f"{gcp_region}-docker.pkg.dev" - - ignore = [".venv/*"] + (additional_ignore_patterns or []) - # Include all files so extensionless runtime binaries (for example `dynamic_runner/tilebox`) - # trigger rebuilds reliably. - self.tag = dirhash(source_dir, "sha256", match=["*", "**/*"], ignore=ignore) - - def build_config(repo_id: str) -> str: - build_config = { - "options": { - "machineType": "E2_HIGHCPU_8", - "env": ["DOCKER_BUILDKIT=1"], - }, - "steps": [ - { - "name": "gcr.io/cloud-builders/docker", - "entrypoint": "bash", - "args": [ - "-c", - f"docker pull {hostname}/{gcp_project}/{repo_id}/{name}:latest || true &\nwait", - ], - }, - { - "name": "gcr.io/cloud-builders/docker", - "env": ["DOCKER_BUILDKIT=1"], - "args": [ - "build", - "--platform", - platform, - "-t", - f"{hostname}/{gcp_project}/{repo_id}/{name}:{self.tag}", - "--cache-from", - f"{hostname}/{gcp_project}/{repo_id}/{name}:latest", - "--build-arg", - "BUILDKIT_INLINE_CACHE=1", - ".", - ], - }, - { - "name": "gcr.io/cloud-builders/docker", - "entrypoint": "bash", - "args": [ - "-c", - f"docker push {hostname}/{gcp_project}/{repo_id}/{name}:{self.tag} &\n" - f"docker tag {hostname}/{gcp_project}/{repo_id}/{name}:{self.tag} {hostname}/{gcp_project}/{repo_id}/{name}:latest\n" - f"docker push {hostname}/{gcp_project}/{repo_id}/{name}:latest &\n" - "wait", - ], - }, - ], - "images": [ - f"{hostname}/{gcp_project}/{repo_id}/{name}:{self.tag}", - f"{hostname}/{gcp_project}/{repo_id}/{name}:latest", - ], - "timeout": "600s", - } - return json.dumps(build_config) - - self.cloud_build = Command( - f"{name}-cloud-build-image", - create=f"gcloud builds submit --config=/dev/stdin --project={gcp_project} {source_dir}", - stdin=Output.from_input(repository_id).apply(build_config), - triggers=[self.tag], - opts=ResourceOptions(parent=self), - ) - - self.container_image = Output.concat(hostname, "/", gcp_project, "/", repository_id, "/", name) - - self.register_outputs( - { - "container_image": self.container_image, - "code_hash": self.tag, - } - ) diff --git a/tilebox_iac/release_runner.py b/tilebox_iac/release_runner.py new file mode 100644 index 0000000..a86eaff --- /dev/null +++ b/tilebox_iac/release_runner.py @@ -0,0 +1,5 @@ +from typing import Final + +RUNNER_IMAGE: Final = "ghcr.io/tilebox/runner:latest" + +__all__ = ["RUNNER_IMAGE"] diff --git a/uv.lock b/uv.lock index d26c05e..11e3332 100644 --- a/uv.lock +++ b/uv.lock @@ -63,18 +63,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, ] -[[package]] -name = "dirhash" -version = "0.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "scantree" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/1f/c8bf92552b7f0a13b9f12b85e3de8df6d9814240e0f8ce8f37433df028b3/dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09", size = 13119, upload-time = "2024-08-03T22:14:11.688Z" }, -] - [[package]] name = "googleapis-common-protos" version = "1.73.1" @@ -400,15 +388,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/4c/f98024021bef4d44dce3613feebd702c7ad8883f777ff8488384c59e9774/parver-0.5-py3-none-any.whl", hash = "sha256:2281b187276c8e8e3c15634f62287b2fb6fe0efe3010f739a6bd1e45fa2bf2b2", size = 15172, upload-time = "2023-10-03T21:06:52.796Z" }, ] -[[package]] -name = "pathspec" -version = "1.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, -] - [[package]] name = "protobuf" version = "6.33.6" @@ -461,8 +440,8 @@ wheels = [ ] [[package]] -name = "pulumi-command" -version = "1.2.1" +name = "pulumi-azure-native" +version = "3.18.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "parver" }, @@ -470,9 +449,9 @@ dependencies = [ { name = "semver" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cf/56/b7f66c9d2e4536b24a5bbe4e743093c7cdd689343d86f020419f4897a88d/pulumi_command-1.2.1.tar.gz", hash = "sha256:cbc9281a6571701ac5db598dac0246a058875b62f28c40c7cc0fcdc36898918d", size = 33022, upload-time = "2026-03-04T00:23:25.84Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/70/c5ccbdcf0a1240ae85420354c258a6c2ecad2290eab515b7a5721247a2a2/pulumi_azure_native-3.18.0.tar.gz", hash = "sha256:7cc31cf8e3f790a5544bf0bde772681ffdd8a84fc79d11ca4440e20a6197a7be", size = 12688725, upload-time = "2026-05-09T15:36:46.973Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/ce/1e5f30f868ba0827ca3311b0b171b12606e30c05e864714764d8d943fef1/pulumi_command-1.2.1-py3-none-any.whl", hash = "sha256:c7e465338e4363f114ddff99ee249260c7770ef4451b970ab51f7e8a37c9b030", size = 36861, upload-time = "2026-03-04T00:23:24.571Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/808429be3067a47914e182c8e7738ea9aaee8638bc51a58991f586992d41/pulumi_azure_native-3.18.0-py3-none-any.whl", hash = "sha256:0e7af4e73d1ccf39125a66cd47eca72c8a13f0636e1b6091d5e3a5655d475ccc", size = 21361802, upload-time = "2026-05-09T15:36:43.383Z" }, ] [[package]] @@ -592,19 +571,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/e2/77be4fff062fa78d9b2a4dea85d14785dac5f1d0c1fb58ed52331f0ebe28/ruff-0.15.8-py3-none-win_arm64.whl", hash = "sha256:cf891fa8e3bb430c0e7fac93851a5978fc99c8fa2c053b57b118972866f8e5f2", size = 11048175, upload-time = "2026-03-26T18:40:01.06Z" }, ] -[[package]] -name = "scantree" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "pathspec" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/ce/828467ddfa0d2fe473673026442d2032d552a168e42cfbf25fd0e5264e0c/scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174", size = 20690, upload-time = "2024-08-03T20:08:58.137Z" }, -] - [[package]] name = "semver" version = "3.0.4" @@ -619,12 +585,12 @@ name = "tilebox-iac" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "dirhash" }, { name = "jinja2" }, { name = "pulumi" }, { name = "pulumi-aws" }, - { name = "pulumi-command" }, + { name = "pulumi-azure-native" }, { name = "pulumi-gcp" }, + { name = "typing-extensions" }, ] [package.dev-dependencies] @@ -635,12 +601,12 @@ dev = [ [package.metadata] requires-dist = [ - { name = "dirhash", specifier = ">=0.5.0" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "pulumi", specifier = ">=3.186.0" }, { name = "pulumi-aws", specifier = ">=6.60.0" }, - { name = "pulumi-command", specifier = ">=1.1.0" }, + { name = "pulumi-azure-native", specifier = ">=3.0.0" }, { name = "pulumi-gcp", specifier = ">=8.39.0" }, + { name = "typing-extensions", specifier = ">=4.0" }, ] [package.metadata.requires-dev]